diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..57bc88a --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..3ad93b6 --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +About IndexTank Service +======================= + +IndexTank Service (http://indextank.com) contains the source code that implementing the Search-as-a-Service platform. It contains the components that allows managing user accounts, server instances (worker) and index instances (deploy); depends on IndexTank Engine (https://github.com/linkedin/indextank-engine) binary. + +### Homepage: + +Find out more about at: TBD + +### License: + +Apache Public License (APL) 2.0 + +### Components: + +1. API +2. Backoffice +3. Storefront +4. Nebu + +### Dependencies: + +* Django +* Python tested with 2.6.6 +* Java(TM) SE Runtime Environment tested with build 1.6.0_24-b07 +* nginx +* uWSGI (http://projects.unbit.it/uwsgi/) +* MySQL +* daemontools (http://cr.yp.to/daemontools.html) +* Thrift library (generated sources with version 0.5.0 are provided) + +### Getting started: + +1. Create the database schema. +2. Create an account. +3. Start an index instance (IndexTank Engine). +4. Start API. + +## API + +Django application implementing the REST JSON API, enables index management per account, indexing functions and search. Interacts via Thrift with specific index instances (deploy). + +## Backoffice + +Django application that allows manual administration. + +## Storefront + +Django application with the service web, contains the user registration form (allows creating accounts). + +## Nebu + +Index, deploy & worker manager. A worker (server instance) may contain a deploy (index instance) or many. + diff --git a/api/.project b/api/.project new file mode 100644 index 0000000..21937e5 --- /dev/null +++ b/api/.project @@ -0,0 +1,18 @@ + + + api + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + org.python.pydev.django.djangoNature + + diff --git a/api/.pydevproject b/api/.pydevproject new file mode 100644 index 0000000..5b70c03 --- /dev/null +++ b/api/.pydevproject @@ -0,0 +1,13 @@ + + + + +Default +python 2.6 + +/api/ + + +../ + + diff --git a/api/.settings/org.eclipse.ltk.core.refactoring.prefs b/api/.settings/org.eclipse.ltk.core.refactoring.prefs new file mode 100644 index 0000000..420b4e4 --- /dev/null +++ b/api/.settings/org.eclipse.ltk.core.refactoring.prefs @@ -0,0 +1,3 @@ +#Wed Jun 30 14:24:43 ART 2010 +eclipse.preferences.version=1 +org.eclipse.ltk.core.refactoring.enable.project.refactoring.history=false diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/amazon_credential.py b/api/amazon_credential.py new file mode 100644 index 0000000..c2910f5 --- /dev/null +++ b/api/amazon_credential.py @@ -0,0 +1,2 @@ +AMAZON_USER = "" +AMAZON_PASSWORD = "" diff --git a/api/boto/__init__.py b/api/boto/__init__.py new file mode 100644 index 0000000..fc2e592 --- /dev/null +++ b/api/boto/__init__.py @@ -0,0 +1,263 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.config import Config, BotoConfigLocations +import os, sys +import logging +import logging.config + +Version = '1.9b' +UserAgent = 'Boto/%s (%s)' % (Version, sys.platform) +config = Config() + +def init_logging(): + for file in BotoConfigLocations: + try: + logging.config.fileConfig(os.path.expanduser(file)) + except: + pass + +class NullHandler(logging.Handler): + def emit(self, record): + pass + +log = logging.getLogger('boto') +log.addHandler(NullHandler()) +init_logging() + +# convenience function to set logging to a particular file +def set_file_logger(name, filepath, level=logging.INFO, format_string=None): + global log + if not format_string: + format_string = "%(asctime)s %(name)s [%(levelname)s]:%(message)s" + logger = logging.getLogger(name) + logger.setLevel(level) + fh = logging.FileHandler(filepath) + fh.setLevel(level) + formatter = logging.Formatter(format_string) + fh.setFormatter(formatter) + logger.addHandler(fh) + log = logger + +def set_stream_logger(name, level=logging.DEBUG, format_string=None): + global log + if not format_string: + format_string = "%(asctime)s %(name)s [%(levelname)s]:%(message)s" + logger = logging.getLogger(name) + logger.setLevel(level) + fh = logging.StreamHandler() + fh.setLevel(level) + formatter = logging.Formatter(format_string) + fh.setFormatter(formatter) + logger.addHandler(fh) + log = logger + +def connect_sqs(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.sqs.connection.SQSConnection` + :return: A connection to Amazon's SQS + """ + from boto.sqs.connection import SQSConnection + return SQSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_s3(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.s3.connection.S3Connection` + :return: A connection to Amazon's S3 + """ + from boto.s3.connection import S3Connection + return S3Connection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_ec2(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.connection.EC2Connection` + :return: A connection to Amazon's EC2 + """ + from boto.ec2.connection import EC2Connection + return EC2Connection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_elb(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.elb.ELBConnection` + :return: A connection to Amazon's Load Balancing Service + """ + from boto.ec2.elb import ELBConnection + return ELBConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_autoscale(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.autoscale.AutoScaleConnection` + :return: A connection to Amazon's Auto Scaling Service + """ + from boto.ec2.autoscale import AutoScaleConnection + return AutoScaleConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_cloudwatch(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.cloudwatch.CloudWatchConnection` + :return: A connection to Amazon's EC2 Monitoring service + """ + from boto.ec2.cloudwatch import CloudWatchConnection + return CloudWatchConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_sdb(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.sdb.connection.SDBConnection` + :return: A connection to Amazon's SDB + """ + from boto.sdb.connection import SDBConnection + return SDBConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_fps(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.fps.connection.FPSConnection` + :return: A connection to FPS + """ + from boto.fps.connection import FPSConnection + return FPSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_cloudfront(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.fps.connection.FPSConnection` + :return: A connection to FPS + """ + from boto.cloudfront import CloudFrontConnection + return CloudFrontConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_vpc(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.vpc.VPCConnection` + :return: A connection to VPC + """ + from boto.vpc import VPCConnection + return VPCConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_rds(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.rds.RDSConnection` + :return: A connection to RDS + """ + from boto.rds import RDSConnection + return RDSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def check_extensions(module_name, module_path): + """ + This function checks for extensions to boto modules. It should be called in the + __init__.py file of all boto modules. See: + http://code.google.com/p/boto/wiki/ExtendModules + + for details. + """ + option_name = '%s_extend' % module_name + version = config.get('Boto', option_name, None) + if version: + dirname = module_path[0] + path = os.path.join(dirname, version) + if os.path.isdir(path): + log.info('extending module %s with: %s' % (module_name, path)) + module_path.insert(0, path) + +_aws_cache = {} + +def _get_aws_conn(service): + global _aws_cache + conn = _aws_cache.get(service) + if not conn: + meth = getattr(sys.modules[__name__], 'connect_'+service) + conn = meth() + _aws_cache[service] = conn + return conn + +def lookup(service, name): + global _aws_cache + conn = _get_aws_conn(service) + obj = _aws_cache.get('.'.join((service,name)), None) + if not obj: + obj = conn.lookup(name) + _aws_cache['.'.join((service,name))] = obj + return obj + diff --git a/api/boto/cloudfront/__init__.py b/api/boto/cloudfront/__init__.py new file mode 100644 index 0000000..d03d6eb --- /dev/null +++ b/api/boto/cloudfront/__init__.py @@ -0,0 +1,222 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import xml.sax +import base64 +import time +import boto.utils +from boto.connection import AWSAuthConnection +from boto import handler +from boto.cloudfront.distribution import * +from boto.cloudfront.identity import OriginAccessIdentity +from boto.cloudfront.identity import OriginAccessIdentityConfig +from boto.resultset import ResultSet +from boto.cloudfront.exception import CloudFrontServerError + +class CloudFrontConnection(AWSAuthConnection): + + DefaultHost = 'cloudfront.amazonaws.com' + Version = '2009-12-01' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0): + AWSAuthConnection.__init__(self, host, + aws_access_key_id, aws_secret_access_key, + True, port, proxy, proxy_port, debug=debug) + + def get_etag(self, response): + response_headers = response.msg + for key in response_headers.keys(): + if key.lower() == 'etag': + return response_headers[key] + return None + + def add_aws_auth_header(self, headers, method, path): + if not headers.has_key('Date'): + headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", + time.gmtime()) + + hmac = self.hmac.copy() + hmac.update(headers['Date']) + b64_hmac = base64.encodestring(hmac.digest()).strip() + headers['Authorization'] = "AWS %s:%s" % (self.aws_access_key_id, b64_hmac) + + # Generics + + def _get_all_objects(self, resource, tags): + if not tags: + tags=[('DistributionSummary', DistributionSummary)] + response = self.make_request('GET', '/%s/%s' % (self.Version, resource)) + body = response.read() + if response.status >= 300: + raise CloudFrontServerError(response.status, response.reason, body) + rs = ResultSet(tags) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def _get_info(self, id, resource, dist_class): + uri = '/%s/%s/%s' % (self.Version, resource, id) + response = self.make_request('GET', uri) + body = response.read() + if response.status >= 300: + raise CloudFrontServerError(response.status, response.reason, body) + d = dist_class(connection=self) + response_headers = response.msg + for key in response_headers.keys(): + if key.lower() == 'etag': + d.etag = response_headers[key] + h = handler.XmlHandler(d, self) + xml.sax.parseString(body, h) + return d + + def _get_config(self, id, resource, config_class): + uri = '/%s/%s/%s/config' % (self.Version, resource, id) + response = self.make_request('GET', uri) + body = response.read() + if response.status >= 300: + raise CloudFrontServerError(response.status, response.reason, body) + d = config_class(connection=self) + d.etag = self.get_etag(response) + h = handler.XmlHandler(d, self) + xml.sax.parseString(body, h) + return d + + def _set_config(self, distribution_id, etag, config): + if isinstance(config, StreamingDistributionConfig): + resource = 'streaming-distribution' + else: + resource = 'distribution' + uri = '/%s/%s/%s/config' % (self.Version, resource, distribution_id) + headers = {'If-Match' : etag, 'Content-Type' : 'text/xml'} + response = self.make_request('PUT', uri, headers, config.to_xml()) + body = response.read() + return self.get_etag(response) + if response.status != 200: + raise CloudFrontServerError(response.status, response.reason, body) + + def _create_object(self, config, resource, dist_class): + response = self.make_request('POST', '/%s/%s' % (self.Version, resource), + {'Content-Type' : 'text/xml'}, data=config.to_xml()) + body = response.read() + if response.status == 201: + d = dist_class(connection=self) + h = handler.XmlHandler(d, self) + xml.sax.parseString(body, h) + return d + else: + raise CloudFrontServerError(response.status, response.reason, body) + + def _delete_object(self, id, etag, resource): + uri = '/%s/%s/%s' % (self.Version, resource, id) + response = self.make_request('DELETE', uri, {'If-Match' : etag}) + body = response.read() + if response.status != 204: + raise CloudFrontServerError(response.status, response.reason, body) + + # Distributions + + def get_all_distributions(self): + tags=[('DistributionSummary', DistributionSummary)] + return self._get_all_objects('distribution', tags) + + def get_distribution_info(self, distribution_id): + return self._get_info(distribution_id, 'distribution', Distribution) + + def get_distribution_config(self, distribution_id): + return self._get_config(distribution_id, 'distribution', + DistributionConfig) + + def set_distribution_config(self, distribution_id, etag, config): + return self._set_config(distribution_id, etag, config) + + def create_distribution(self, origin, enabled, caller_reference='', + cnames=None, comment=''): + config = DistributionConfig(origin=origin, enabled=enabled, + caller_reference=caller_reference, + cnames=cnames, comment=comment) + return self._create_object(config, 'distribution', Distribution) + + def delete_distribution(self, distribution_id, etag): + return self._delete_object(distribution_id, etag, 'distribution') + + # Streaming Distributions + + def get_all_streaming_distributions(self): + tags=[('StreamingDistributionSummary', StreamingDistributionSummary)] + return self._get_all_objects('streaming-distribution', tags) + + def get_streaming_distribution_info(self, distribution_id): + return self._get_info(distribution_id, 'streaming-distribution', + StreamingDistribution) + + def get_streaming_distribution_config(self, distribution_id): + return self._get_config(distribution_id, 'streaming-distribution', + StreamingDistributionConfig) + + def set_streaming_distribution_config(self, distribution_id, etag, config): + return self._set_config(distribution_id, etag, config) + + def create_streaming_distribution(self, origin, enabled, + caller_reference='', + cnames=None, comment=''): + config = StreamingDistributionConfig(origin=origin, enabled=enabled, + caller_reference=caller_reference, + cnames=cnames, comment=comment) + return self._create_object(config, 'streaming-distribution', + StreamingDistribution) + + def delete_streaming_distribution(self, distribution_id, etag): + return self._delete_object(distribution_id, etag, 'streaming-distribution') + + # Origin Access Identity + + def get_all_origin_access_identity(self): + tags=[('CloudFrontOriginAccessIdentitySummary', + OriginAccessIdentitySummary)] + return self._get_all_objects('origin-access-identity/cloudfront', tags) + + def get_origin_access_identity_info(self, access_id): + return self._get_info(access_id, 'origin-access-identity/cloudfront', + OriginAccessIdentity) + + def get_origin_access_identity_config(self, access_id): + return self._get_config(access_id, + 'origin-access-identity/cloudfront', + OriginAccessIdentityConfig) + + def set_origin_access_identity_config(self, access_id, + etag, config): + return self._set_config(access_id, etag, config) + + def create_origin_access_identity(self, caller_reference='', comment=''): + config = OriginAccessIdentityConfig(caller_reference=caller_reference, + comment=comment) + return self._create_object(config, 'origin-access-identity/cloudfront', + OriginAccessIdentity) + + def delete_origin_access_identity(self, access_id, etag): + return self._delete_object(access_id, etag, + 'origin-access-identity/cloudfront') + + diff --git a/api/boto/cloudfront/distribution.py b/api/boto/cloudfront/distribution.py new file mode 100644 index 0000000..cd36add --- /dev/null +++ b/api/boto/cloudfront/distribution.py @@ -0,0 +1,470 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import uuid +from boto.cloudfront.identity import OriginAccessIdentity +from boto.cloudfront.object import Object, StreamingObject +from boto.cloudfront.signers import Signer, ActiveTrustedSigners, TrustedSigners +from boto.cloudfront.logging import LoggingInfo +from boto.s3.acl import ACL + +class DistributionConfig: + + def __init__(self, connection=None, origin='', enabled=False, + caller_reference='', cnames=None, comment='', + origin_access_identity=None, trusted_signers=None): + self.connection = connection + self.origin = origin + self.enabled = enabled + if caller_reference: + self.caller_reference = caller_reference + else: + self.caller_reference = str(uuid.uuid4()) + self.cnames = [] + if cnames: + self.cnames = cnames + self.comment = comment + self.origin_access_identity = origin_access_identity + self.trusted_signers = trusted_signers + self.logging = None + + def get_oai_value(self): + if isinstance(self.origin_access_identity, OriginAccessIdentity): + return self.origin_access_identity.uri() + else: + return self.origin_access_identity + + def to_xml(self): + s = '\n' + s += '\n' + s += ' %s\n' % self.origin + s += ' %s\n' % self.caller_reference + for cname in self.cnames: + s += ' %s\n' % cname + if self.comment: + s += ' %s\n' % self.comment + s += ' ' + if self.enabled: + s += 'true' + else: + s += 'false' + s += '\n' + if self.origin_access_identity: + val = self.get_oai_value() + s += '%s\n' % val + if self.trusted_signers: + s += '\n' + for signer in self.trusted_signers: + if signer == 'Self': + s += ' \n' + else: + s += ' %s\n' % signer + s += '\n' + if self.logging: + s += '\n' + s += ' %s\n' % self.logging.bucket + s += ' %s\n' % self.logging.prefix + s += '\n' + s += '\n' + return s + + def startElement(self, name, attrs, connection): + if name == 'TrustedSigners': + self.trusted_signers = TrustedSigners() + return self.trusted_signers + elif name == 'Logging': + self.logging = LoggingInfo() + return self.logging + else: + return None + + def endElement(self, name, value, connection): + if name == 'CNAME': + self.cnames.append(value) + elif name == 'Origin': + self.origin = value + elif name == 'Comment': + self.comment = value + elif name == 'Enabled': + if value.lower() == 'true': + self.enabled = True + else: + self.enabled = False + elif name == 'CallerReference': + self.caller_reference = value + elif name == 'OriginAccessIdentity': + self.origin_access_identity = value + else: + setattr(self, name, value) + +class StreamingDistributionConfig(DistributionConfig): + + def __init__(self, connection=None, origin='', enabled=False, + caller_reference='', cnames=None, comment=''): + DistributionConfig.__init__(self, connection, origin, + enabled, caller_reference, + cnames, comment) + + def to_xml(self): + s = '\n' + s += '\n' + s += ' %s\n' % self.origin + s += ' %s\n' % self.caller_reference + for cname in self.cnames: + s += ' %s\n' % cname + if self.comment: + s += ' %s\n' % self.comment + s += ' ' + if self.enabled: + s += 'true' + else: + s += 'false' + s += '\n' + s += '\n' + return s + + def startElement(self, name, attrs, connection): + pass + +class DistributionSummary: + + def __init__(self, connection=None, domain_name='', id='', + last_modified_time=None, status='', origin='', + cname='', comment='', enabled=False): + self.connection = connection + self.domain_name = domain_name + self.id = id + self.last_modified_time = last_modified_time + self.status = status + self.origin = origin + self.enabled = enabled + self.cnames = [] + if cname: + self.cnames.append(cname) + self.comment = comment + self.trusted_signers = None + self.etag = None + self.streaming = False + + def startElement(self, name, attrs, connection): + if name == 'TrustedSigners': + self.trusted_signers = TrustedSigners() + return self.trusted_signers + return None + + def endElement(self, name, value, connection): + if name == 'Id': + self.id = value + elif name == 'Status': + self.status = value + elif name == 'LastModifiedTime': + self.last_modified_time = value + elif name == 'DomainName': + self.domain_name = value + elif name == 'Origin': + self.origin = value + elif name == 'CNAME': + self.cnames.append(value) + elif name == 'Comment': + self.comment = value + elif name == 'Enabled': + if value.lower() == 'true': + self.enabled = True + else: + self.enabled = False + elif name == 'StreamingDistributionSummary': + self.streaming = True + else: + setattr(self, name, value) + + def get_distribution(self): + return self.connection.get_distribution_info(self.id) + +class StreamingDistributionSummary(DistributionSummary): + + def get_distribution(self): + return self.connection.get_streaming_distribution_info(self.id) + +class Distribution: + + def __init__(self, connection=None, config=None, domain_name='', + id='', last_modified_time=None, status=''): + self.connection = connection + self.config = config + self.domain_name = domain_name + self.id = id + self.last_modified_time = last_modified_time + self.status = status + self.active_signers = None + self.etag = None + self._bucket = None + + def startElement(self, name, attrs, connection): + if name == 'DistributionConfig': + self.config = DistributionConfig() + return self.config + elif name == 'ActiveTrustedSigners': + self.active_signers = ActiveTrustedSigners() + return self.active_signers + else: + return None + + def endElement(self, name, value, connection): + if name == 'Id': + self.id = value + elif name == 'LastModifiedTime': + self.last_modified_time = value + elif name == 'Status': + self.status = value + elif name == 'DomainName': + self.domain_name = value + else: + setattr(self, name, value) + + def update(self, enabled=None, cnames=None, comment=None, + origin_access_identity=None, + trusted_signers=None): + """ + Update the configuration of the Distribution. + + :type enabled: bool + :param enabled: Whether the Distribution is active or not. + + :type cnames: list of str + :param cnames: The DNS CNAME's associated with this + Distribution. Maximum of 10 values. + + :type comment: str or unicode + :param comment: The comment associated with the Distribution. + + :type origin_access_identity: :class:`boto.cloudfront.identity.OriginAccessIdentity` + :param origin_access_identity: The CloudFront origin access identity + associated with the distribution. This + must be provided if you want the + distribution to serve private content. + + :type trusted_signers: :class:`boto.cloudfront.signers.TrustedSigner` + :param trusted_signers: The AWS users who are authorized to sign + URL's for private content in this Distribution. + + """ + new_config = DistributionConfig(self.connection, self.config.origin, + self.config.enabled, self.config.caller_reference, + self.config.cnames, self.config.comment, + self.config.origin_access_identity, + self.config.trusted_signers) + if enabled != None: + new_config.enabled = enabled + if cnames != None: + new_config.cnames = cnames + if comment != None: + new_config.comment = comment + if origin_access_identity != None: + new_config.origin_access_identity = origin_access_identity + if trusted_signers: + new_config.trusted_signers = trusted_signers + self.etag = self.connection.set_distribution_config(self.id, self.etag, new_config) + self.config = new_config + self._object_class = Object + + def enable(self): + """ + Deactivate the Distribution. A convenience wrapper around + the update method. + """ + self.update(enabled=True) + + def disable(self): + """ + Activate the Distribution. A convenience wrapper around + the update method. + """ + self.update(enabled=False) + + def delete(self): + """ + Delete this CloudFront Distribution. The content + associated with the Distribution is not deleted from + the underlying Origin bucket in S3. + """ + self.connection.delete_distribution(self.id, self.etag) + + def _get_bucket(self): + if not self._bucket: + bucket_name = self.config.origin.split('.')[0] + from boto.s3.connection import S3Connection + s3 = S3Connection(self.connection.aws_access_key_id, + self.connection.aws_secret_access_key, + proxy=self.connection.proxy, + proxy_port=self.connection.proxy_port, + proxy_user=self.connection.proxy_user, + proxy_pass=self.connection.proxy_pass) + self._bucket = s3.get_bucket(bucket_name) + self._bucket.distribution = self + self._bucket.set_key_class(self._object_class) + return self._bucket + + def get_objects(self): + """ + Return a list of all content objects in this distribution. + + :rtype: list of :class:`boto.cloudfront.object.Object` + :return: The content objects + """ + bucket = self._get_bucket() + objs = [] + for key in bucket: + objs.append(key) + return objs + + def set_permissions(self, object, replace=False): + """ + Sets the S3 ACL grants for the given object to the appropriate + value based on the type of Distribution. If the Distribution + is serving private content the ACL will be set to include the + Origin Access Identity associated with the Distribution. If + the Distribution is serving public content the content will + be set up with "public-read". + + :type object: :class:`boto.cloudfront.object.Object` + :param enabled: The Object whose ACL is being set + + :type replace: bool + :param replace: If False, the Origin Access Identity will be + appended to the existing ACL for the object. + If True, the ACL for the object will be + completely replaced with one that grants + READ permission to the Origin Access Identity. + + """ + if self.config.origin_access_identity: + id = self.config.origin_access_identity.split('/')[-1] + oai = self.connection.get_origin_access_identity_info(id) + policy = object.get_acl() + if replace: + policy.acl = ACL() + policy.acl.add_user_grant('READ', oai.s3_user_id) + object.set_acl(policy) + else: + object.set_canned_acl('public-read') + + def set_permissions_all(self, replace=False): + """ + Sets the S3 ACL grants for all objects in the Distribution + to the appropriate value based on the type of Distribution. + + :type replace: bool + :param replace: If False, the Origin Access Identity will be + appended to the existing ACL for the object. + If True, the ACL for the object will be + completely replaced with one that grants + READ permission to the Origin Access Identity. + + """ + bucket = self._get_bucket() + for key in bucket: + self.set_permissions(key) + + def add_object(self, name, content, headers=None, replace=True): + """ + Adds a new content object to the Distribution. The content + for the object will be copied to a new Key in the S3 Bucket + and the permissions will be set appropriately for the type + of Distribution. + + :type name: str or unicode + :param name: The name or key of the new object. + + :type content: file-like object + :param content: A file-like object that contains the content + for the new object. + + :type headers: dict + :param headers: A dictionary containing additional headers + you would like associated with the new + object in S3. + + :rtype: :class:`boto.cloudfront.object.Object` + :return: The newly created object. + """ + if self.config.origin_access_identity: + policy = 'private' + else: + policy = 'public-read' + bucket = self._get_bucket() + object = bucket.new_key(name) + object.set_contents_from_file(content, headers=headers, policy=policy) + if self.config.origin_access_identity: + self.set_permissions(object, replace) + return object + +class StreamingDistribution(Distribution): + + def __init__(self, connection=None, config=None, domain_name='', + id='', last_modified_time=None, status=''): + Distribution.__init__(self, connection, config, domain_name, + id, last_modified_time, status) + self._object_class = StreamingObject + + def startElement(self, name, attrs, connection): + if name == 'StreamingDistributionConfig': + self.config = StreamingDistributionConfig() + return self.config + else: + return None + + def update(self, enabled=None, cnames=None, comment=None): + """ + Update the configuration of the Distribution. + + :type enabled: bool + :param enabled: Whether the Distribution is active or not. + + :type cnames: list of str + :param cnames: The DNS CNAME's associated with this + Distribution. Maximum of 10 values. + + :type comment: str or unicode + :param comment: The comment associated with the Distribution. + + """ + new_config = StreamingDistributionConfig(self.connection, + self.config.origin, + self.config.enabled, + self.config.caller_reference, + self.config.cnames, + self.config.comment) + if enabled != None: + new_config.enabled = enabled + if cnames != None: + new_config.cnames = cnames + if comment != None: + new_config.comment = comment + + self.etag = self.connection.set_streaming_distribution_config(self.id, + self.etag, + new_config) + self.config = new_config + + def delete(self): + self.connection.delete_streaming_distribution(self.id, self.etag) + + diff --git a/api/boto/cloudfront/exception.py b/api/boto/cloudfront/exception.py new file mode 100644 index 0000000..7680642 --- /dev/null +++ b/api/boto/cloudfront/exception.py @@ -0,0 +1,26 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import BotoServerError + +class CloudFrontServerError(BotoServerError): + + pass diff --git a/api/boto/cloudfront/identity.py b/api/boto/cloudfront/identity.py new file mode 100644 index 0000000..711b8b7 --- /dev/null +++ b/api/boto/cloudfront/identity.py @@ -0,0 +1,98 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import uuid + +class OriginAccessIdentity: + + def __init__(self, connection=None, config=None, id='', + s3_user_id='', comment=''): + self.connection = connection + self.config = config + self.id = id + self.s3_user_id = s3_user_id + self.comment = comment + self.etag = None + + def startElement(self, name, attrs, connection): + if name == 'CloudFrontOriginAccessIdentityConfig': + self.config = OriginAccessIdentityConfig() + return self.config + else: + return None + + def endElement(self, name, value, connection): + if name == 'Id': + self.id = value + elif name == 'S3CanonicalUserId': + self.s3_user_id = value + elif name == 'Comment': + self.comment = value + else: + setattr(self, name, value) + + def update(self, comment=None): + new_config = OriginAccessIdentifyConfig(self.connection, + self.config.caller_reference, + self.config.comment) + if comment != None: + new_config.comment = comment + self.etag = self.connection.set_origin_identity_config(self.id, self.etag, new_config) + self.config = new_config + + def delete(self): + return self.connection.delete_distribution(self.id, self.etag) + + def uri(self): + return 'origin-access-identity/cloudfront/%s' % id + +class OriginAccessIdentityConfig: + + def __init__(self, connection=None, caller_reference='', comment=''): + self.connection = connection + if caller_reference: + self.caller_reference = caller_reference + else: + self.caller_reference = str(uuid.uuid4()) + self.comment = comment + + def to_xml(self): + s = '\n' + s += '\n' + s += ' %s\n' % self.caller_reference + if self.comment: + s += ' %s\n' % self.comment + s += '\n' + return s + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Comment': + self.comment = value + elif name == 'CallerReference': + self.caller_reference = value + else: + setattr(self, name, value) + + + diff --git a/api/boto/cloudfront/logging.py b/api/boto/cloudfront/logging.py new file mode 100644 index 0000000..6c2f4fd --- /dev/null +++ b/api/boto/cloudfront/logging.py @@ -0,0 +1,38 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class LoggingInfo(object): + + def __init__(self, bucket='', prefix=''): + self.bucket = bucket + self.prefix = prefix + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Bucket': + self.bucket = value + elif name == 'Prefix': + self.prefix = value + else: + setattr(self, name, value) + diff --git a/api/boto/cloudfront/object.py b/api/boto/cloudfront/object.py new file mode 100644 index 0000000..3574d13 --- /dev/null +++ b/api/boto/cloudfront/object.py @@ -0,0 +1,48 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.s3.key import Key + +class Object(Key): + + def __init__(self, bucket, name=None): + Key.__init__(self, bucket, name=name) + self.distribution = bucket.distribution + + def __repr__(self): + return '' % (self.distribution.config.origin, self.name) + + def url(self, scheme='http'): + url = '%s://' % scheme + url += self.distribution.domain_name + if scheme.lower().startswith('rtmp'): + url += '/cfx/st/' + else: + url += '/' + url += self.name + return url + +class StreamingObject(Object): + + def url(self, scheme='rtmp'): + return Object.url(self, scheme) + + diff --git a/api/boto/cloudfront/signers.py b/api/boto/cloudfront/signers.py new file mode 100644 index 0000000..0b0cd50 --- /dev/null +++ b/api/boto/cloudfront/signers.py @@ -0,0 +1,60 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Signer: + + def __init__(self): + self.id = None + self.key_pair_ids = [] + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Self': + self.id = 'Self' + elif name == 'AwsAccountNumber': + self.id = value + elif name == 'KeyPairId': + self.key_pair_ids.append(value) + +class ActiveTrustedSigners(list): + + def startElement(self, name, attrs, connection): + if name == 'Signer': + s = Signer() + self.append(s) + return s + + def endElement(self, name, value, connection): + pass + +class TrustedSigners(list): + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Self': + self.append(name) + elif name == 'AwsAccountNumber': + self.append(value) + diff --git a/api/boto/connection.py b/api/boto/connection.py new file mode 100644 index 0000000..9a443f7 --- /dev/null +++ b/api/boto/connection.py @@ -0,0 +1,648 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2009 The Echo Nest Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# +# Parts of this code were copied or derived from sample code supplied by AWS. +# The following notice applies to that code. +# +# This software code is made available "AS IS" without warranties of any +# kind. You may copy, display, modify and redistribute the software +# code either by itself or as incorporated into your code; provided that +# you do not remove any proprietary notices. Your use of this software +# code is at your own risk and you waive any claim against Amazon +# Digital Services, Inc. or its affiliates with respect to your use of +# this software code. (c) 2006 Amazon Digital Services, Inc. or its +# affiliates. + +""" +Handles basic connections to AWS +""" + +import base64 +import hmac +import httplib +import socket, errno +import re +import sys +import time +import urllib, urlparse +import os +import xml.sax +import Queue +import boto +from boto.exception import AWSConnectionError, BotoClientError, BotoServerError +from boto.resultset import ResultSet +import boto.utils +from boto import config, UserAgent, handler + +# +# the following is necessary because of the incompatibilities +# between Python 2.4, 2.5, and 2.6 as well as the fact that some +# people running 2.4 have installed hashlib as a separate module +# this fix was provided by boto user mccormix. +# see: http://code.google.com/p/boto/issues/detail?id=172 +# for more details. +# +try: + from hashlib import sha1 as sha + from hashlib import sha256 as sha256 + + if sys.version[:3] == "2.4": + # we are using an hmac that expects a .new() method. + class Faker: + def __init__(self, which): + self.which = which + self.digest_size = self.which().digest_size + + def new(self, *args, **kwargs): + return self.which(*args, **kwargs) + + sha = Faker(sha) + sha256 = Faker(sha256) + +except ImportError: + import sha + sha256 = None + +PORTS_BY_SECURITY = { True: 443, False: 80 } + +class ConnectionPool: + def __init__(self, hosts, connections_per_host): + self._hosts = boto.utils.LRUCache(hosts) + self.connections_per_host = connections_per_host + + def __getitem__(self, key): + if key not in self._hosts: + self._hosts[key] = Queue.Queue(self.connections_per_host) + return self._hosts[key] + + def __repr__(self): + return 'ConnectionPool:%s' % ','.join(self._hosts._dict.keys()) + +class AWSAuthConnection: + def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, path='/'): + """ + :type host: string + :param host: The host to make the connection to + + :type aws_access_key_id: string + :param aws_access_key_id: AWS Access Key ID (provided by Amazon) + + :type aws_secret_access_key: string + :param aws_secret_access_key: Secret Access Key (provided by Amazon) + + :type is_secure: boolean + :param is_secure: Whether the connection is over SSL + + :type https_connection_factory: list or tuple + :param https_connection_factory: A pair of an HTTP connection + factory and the exceptions to catch. + The factory should have a similar + interface to L{httplib.HTTPSConnection}. + + :type proxy: + :param proxy: + + :type proxy_port: int + :param proxy_port: The port to use when connecting over a proxy + + :type proxy_user: string + :param proxy_user: The username to connect with on the proxy + + :type proxy_pass: string + :param proxy_pass: The password to use when connection over a proxy. + + :type port: integer + :param port: The port to use to connect + """ + + self.num_retries = 5 + self.is_secure = is_secure + self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) + # define exceptions from httplib that we want to catch and retry + self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror) + # define values in socket exceptions we don't want to catch + self.socket_exception_values = (errno.EINTR,) + if https_connection_factory is not None: + self.https_connection_factory = https_connection_factory[0] + self.http_exceptions += https_connection_factory[1] + else: + self.https_connection_factory = None + if (is_secure): + self.protocol = 'https' + else: + self.protocol = 'http' + self.host = host + self.path = path + if debug: + self.debug = debug + else: + self.debug = config.getint('Boto', 'debug', debug) + if port: + self.port = port + else: + self.port = PORTS_BY_SECURITY[is_secure] + + if aws_access_key_id: + self.aws_access_key_id = aws_access_key_id + elif os.environ.has_key('AWS_ACCESS_KEY_ID'): + self.aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID'] + elif config.has_option('Credentials', 'aws_access_key_id'): + self.aws_access_key_id = config.get('Credentials', 'aws_access_key_id') + + if aws_secret_access_key: + self.aws_secret_access_key = aws_secret_access_key + elif os.environ.has_key('AWS_SECRET_ACCESS_KEY'): + self.aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY'] + elif config.has_option('Credentials', 'aws_secret_access_key'): + self.aws_secret_access_key = config.get('Credentials', 'aws_secret_access_key') + + # initialize an HMAC for signatures, make copies with each request + self.hmac = hmac.new(self.aws_secret_access_key, digestmod=sha) + if sha256: + self.hmac_256 = hmac.new(self.aws_secret_access_key, digestmod=sha256) + else: + self.hmac_256 = None + + # cache up to 20 connections per host, up to 20 hosts + self._pool = ConnectionPool(20, 20) + self._connection = (self.server_name(), self.is_secure) + self._last_rs = None + + def __repr__(self): + return '%s:%s' % (self.__class__.__name__, self.host) + + def _cached_name(self, host, is_secure): + if host is None: + host = self.server_name() + cached_name = is_secure and 'https://' or 'http://' + cached_name += host + return cached_name + + def connection(self): + return self.get_http_connection(*self._connection) + + connection = property(connection) + + def get_path(self, path='/'): + pos = path.find('?') + if pos >= 0: + params = path[pos:] + path = path[:pos] + else: + params = None + if path[-1] == '/': + need_trailing = True + else: + need_trailing = False + path_elements = self.path.split('/') + path_elements.extend(path.split('/')) + path_elements = [p for p in path_elements if p] + path = '/' + '/'.join(path_elements) + if path[-1] != '/' and need_trailing: + path += '/' + if params: + path = path + params + return path + + def server_name(self, port=None): + if not port: + port = self.port + if port == 80: + signature_host = self.host + else: + # This unfortunate little hack can be attributed to + # a difference in the 2.6 version of httplib. In old + # versions, it would append ":443" to the hostname sent + # in the Host header and so we needed to make sure we + # did the same when calculating the V2 signature. In 2.6 + # it no longer does that. Hence, this kludge. + if sys.version[:3] == "2.6" and port == 443: + signature_host = self.host + else: + signature_host = '%s:%d' % (self.host, port) + return signature_host + + def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): + self.proxy = proxy + self.proxy_port = proxy_port + self.proxy_user = proxy_user + self.proxy_pass = proxy_pass + if os.environ.has_key('http_proxy') and not self.proxy: + pattern = re.compile( + '(?:http://)?' \ + '(?:(?P\w+):(?P.*)@)?' \ + '(?P[\w\-\.]+)' \ + '(?::(?P\d+))?' + ) + match = pattern.match(os.environ['http_proxy']) + if match: + self.proxy = match.group('host') + self.proxy_port = match.group('port') + self.proxy_user = match.group('user') + self.proxy_pass = match.group('pass') + else: + if not self.proxy: + self.proxy = config.get_value('Boto', 'proxy', None) + if not self.proxy_port: + self.proxy_port = config.get_value('Boto', 'proxy_port', None) + if not self.proxy_user: + self.proxy_user = config.get_value('Boto', 'proxy_user', None) + if not self.proxy_pass: + self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) + + if not self.proxy_port and self.proxy: + print "http_proxy environment variable does not specify " \ + "a port, using default" + self.proxy_port = self.port + self.use_proxy = (self.proxy != None) + + def get_http_connection(self, host, is_secure): + queue = self._pool[self._cached_name(host, is_secure)] + try: + return queue.get_nowait() + except Queue.Empty: + return self.new_http_connection(host, is_secure) + + def new_http_connection(self, host, is_secure): + if self.use_proxy: + host = '%s:%d' % (self.proxy, int(self.proxy_port)) + if host is None: + host = self.server_name() + boto.log.debug('establishing HTTP connection') + if is_secure: + if self.use_proxy: + connection = self.proxy_ssl() + elif self.https_connection_factory: + connection = self.https_connection_factory(host) + else: + connection = httplib.HTTPSConnection(host) + else: + connection = httplib.HTTPConnection(host) + if self.debug > 1: + connection.set_debuglevel(self.debug) + # self.connection must be maintained for backwards-compatibility + # however, it must be dynamically pulled from the connection pool + # set a private variable which will enable that + if host.split(':')[0] == self.host and is_secure == self.is_secure: + self._connection = (host, is_secure) + return connection + + def put_http_connection(self, host, is_secure, connection): + try: + self._pool[self._cached_name(host, is_secure)].put_nowait(connection) + except Queue.Full: + # gracefully fail in case of pool overflow + connection.close() + + def proxy_ssl(self): + host = '%s:%d' % (self.host, self.port) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.connect((self.proxy, int(self.proxy_port))) + except: + raise + sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) + sock.sendall("User-Agent: %s\r\n" % UserAgent) + if self.proxy_user and self.proxy_pass: + for k, v in self.get_proxy_auth_header().items(): + sock.sendall("%s: %s\r\n" % (k, v)) + sock.sendall("\r\n") + resp = httplib.HTTPResponse(sock, strict=True) + resp.begin() + + if resp.status != 200: + # Fake a socket error, use a code that make it obvious it hasn't + # been generated by the socket library + raise socket.error(-71, + "Error talking to HTTP proxy %s:%s: %s (%s)" % + (self.proxy, self.proxy_port, resp.status, resp.reason)) + + # We can safely close the response, it duped the original socket + resp.close() + + h = httplib.HTTPConnection(host) + + # Wrap the socket in an SSL socket + if hasattr(httplib, 'ssl'): + sslSock = httplib.ssl.SSLSocket(sock) + else: # Old Python, no ssl module + sslSock = socket.ssl(sock, None, None) + sslSock = httplib.FakeSocket(sock, sslSock) + # This is a bit unclean + h.sock = sslSock + return h + + def prefix_proxy_to_path(self, path, host=None): + path = self.protocol + '://' + (host or self.server_name()) + path + return path + + def get_proxy_auth_header(self): + auth = base64.encodestring(self.proxy_user+':'+self.proxy_pass) + return {'Proxy-Authorization': 'Basic %s' % auth} + + def _mexe(self, method, path, data, headers, host=None, sender=None): + """ + mexe - Multi-execute inside a loop, retrying multiple times to handle + transient Internet errors by simply trying again. + Also handles redirects. + + This code was inspired by the S3Utils classes posted to the boto-users + Google group by Larry Bates. Thanks! + """ + boto.log.debug('Method: %s' % method) + boto.log.debug('Path: %s' % path) + boto.log.debug('Data: %s' % data) + boto.log.debug('Headers: %s' % headers) + boto.log.debug('Host: %s' % host) + response = None + body = None + e = None + num_retries = config.getint('Boto', 'num_retries', self.num_retries) + i = 0 + connection = self.get_http_connection(host, self.is_secure) + while i <= num_retries: + try: + if callable(sender): + response = sender(connection, method, path, data, headers) + else: + connection.request(method, path, data, headers) + response = connection.getresponse() + location = response.getheader('location') + # -- gross hack -- + # httplib gets confused with chunked responses to HEAD requests + # so I have to fake it out + if method == 'HEAD' and getattr(response, 'chunked', False): + response.chunked = 0 + if response.status == 500 or response.status == 503: + boto.log.debug('received %d response, retrying in %d seconds' % (response.status, 2**i)) + body = response.read() + elif response.status == 408: + body = response.read() + print '-------------------------' + print ' 4 0 8 ' + print 'path=%s' % path + print body + print '-------------------------' + elif response.status < 300 or response.status >= 400 or \ + not location: + self.put_http_connection(host, self.is_secure, connection) + return response + else: + scheme, host, path, params, query, fragment = \ + urlparse.urlparse(location) + if query: + path += '?' + query + boto.log.debug('Redirecting: %s' % scheme + '://' + host + path) + connection = self.get_http_connection(host, + scheme == 'https') + continue + except KeyboardInterrupt: + sys.exit('Keyboard Interrupt') + except self.http_exceptions, e: + boto.log.debug('encountered %s exception, reconnecting' % \ + e.__class__.__name__) + connection = self.new_http_connection(host, self.is_secure) + time.sleep(2**i) + i += 1 + # If we made it here, it's because we have exhausted our retries and stil haven't + # succeeded. So, if we have a response object, use it to raise an exception. + # Otherwise, raise the exception that must have already happened. + if response: + raise BotoServerError(response.status, response.reason, body) + elif e: + raise e + else: + raise BotoClientError('Please report this exception as a Boto Issue!') + + def make_request(self, method, path, headers=None, data='', host=None, + auth_path=None, sender=None): + path = self.get_path(path) + if headers == None: + headers = {} + else: + headers = headers.copy() + headers['User-Agent'] = UserAgent + if not headers.has_key('Content-Length'): + headers['Content-Length'] = str(len(data)) + if self.use_proxy: + path = self.prefix_proxy_to_path(path, host) + if self.proxy_user and self.proxy_pass and not self.is_secure: + # If is_secure, we don't have to set the proxy authentication + # header here, we did that in the CONNECT to the proxy. + headers.update(self.get_proxy_auth_header()) + request_string = auth_path or path + self.add_aws_auth_header(headers, method, request_string) + return self._mexe(method, path, data, headers, host, sender) + + def add_aws_auth_header(self, headers, method, path): + path = self.get_path(path) + if not headers.has_key('Date'): + headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", + time.gmtime()) + + c_string = boto.utils.canonical_string(method, path, headers) + boto.log.debug('Canonical: %s' % c_string) + hmac = self.hmac.copy() + hmac.update(c_string) + b64_hmac = base64.encodestring(hmac.digest()).strip() + headers['Authorization'] = "AWS %s:%s" % (self.aws_access_key_id, b64_hmac) + + def close(self): + """(Optional) Close any open HTTP connections. This is non-destructive, + and making a new request will open a connection again.""" + + boto.log.debug('closing all HTTP connections') + self.connection = None # compat field + hosts = list(self._cache.keys()) + for host in hosts: + conn = self._cache[host] + conn.close() + del self._cache[host] + +class AWSQueryConnection(AWSAuthConnection): + + APIVersion = '' + SignatureVersion = '1' + ResponseError = BotoServerError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=None, debug=0, + https_connection_factory=None, path='/'): + AWSAuthConnection.__init__(self, host, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + debug, https_connection_factory, path) + + def get_utf8_value(self, value): + if not isinstance(value, str) and not isinstance(value, unicode): + value = str(value) + if isinstance(value, unicode): + return value.encode('utf-8') + else: + return value + + def calc_signature_0(self, params): + boto.log.debug('using calc_signature_0') + hmac = self.hmac.copy() + s = params['Action'] + params['Timestamp'] + hmac.update(s) + keys = params.keys() + keys.sort(cmp = lambda x, y: cmp(x.lower(), y.lower())) + pairs = [] + for key in keys: + val = self.get_utf8_value(params[key]) + pairs.append(key + '=' + urllib.quote(val)) + qs = '&'.join(pairs) + return (qs, base64.b64encode(hmac.digest())) + + def calc_signature_1(self, params): + boto.log.debug('using calc_signature_1') + hmac = self.hmac.copy() + keys = params.keys() + keys.sort(cmp = lambda x, y: cmp(x.lower(), y.lower())) + pairs = [] + for key in keys: + hmac.update(key) + val = self.get_utf8_value(params[key]) + hmac.update(val) + pairs.append(key + '=' + urllib.quote(val)) + qs = '&'.join(pairs) + return (qs, base64.b64encode(hmac.digest())) + + def calc_signature_2(self, params, verb, path): + boto.log.debug('using calc_signature_2') + string_to_sign = '%s\n%s\n%s\n' % (verb, self.server_name().lower(), path) + if self.hmac_256: + hmac = self.hmac_256.copy() + params['SignatureMethod'] = 'HmacSHA256' + else: + hmac = self.hmac.copy() + params['SignatureMethod'] = 'HmacSHA1' + keys = params.keys() + keys.sort() + pairs = [] + for key in keys: + val = self.get_utf8_value(params[key]) + pairs.append(urllib.quote(key, safe='') + '=' + urllib.quote(val, safe='-_~')) + qs = '&'.join(pairs) + boto.log.debug('query string: %s' % qs) + string_to_sign += qs + boto.log.debug('string_to_sign: %s' % string_to_sign) + hmac.update(string_to_sign) + b64 = base64.b64encode(hmac.digest()) + boto.log.debug('len(b64)=%d' % len(b64)) + boto.log.debug('base64 encoded digest: %s' % b64) + return (qs, b64) + + def get_signature(self, params, verb, path): + if self.SignatureVersion == '0': + t = self.calc_signature_0(params) + elif self.SignatureVersion == '1': + t = self.calc_signature_1(params) + elif self.SignatureVersion == '2': + t = self.calc_signature_2(params, verb, path) + else: + raise BotoClientError('Unknown Signature Version: %s' % self.SignatureVersion) + return t + + def make_request(self, action, params=None, path='/', verb='GET'): + headers = {} + if params == None: + params = {} + params['Action'] = action + params['Version'] = self.APIVersion + params['AWSAccessKeyId'] = self.aws_access_key_id + params['SignatureVersion'] = self.SignatureVersion + params['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + qs, signature = self.get_signature(params, verb, self.get_path(path)) + if verb == 'POST': + headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8' + request_body = qs + '&Signature=' + urllib.quote(signature) + qs = path + else: + request_body = '' + qs = path + '?' + qs + '&Signature=' + urllib.quote(signature) + return AWSAuthConnection.make_request(self, verb, qs, + data=request_body, + headers=headers) + + def build_list_params(self, params, items, label): + if isinstance(items, str): + items = [items] + for i in range(1, len(items)+1): + params['%s.%d' % (label, i)] = items[i-1] + + # generics + + def get_list(self, action, params, markers, path='/', parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if response.status == 200: + rs = ResultSet(markers) + h = handler.XmlHandler(rs, parent) + xml.sax.parseString(body, h) + return rs + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_object(self, action, params, cls, path='/', parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if response.status == 200: + obj = cls(parent) + h = handler.XmlHandler(obj, parent) + xml.sax.parseString(body, h) + return obj + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_status(self, action, params, path='/', parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, parent) + xml.sax.parseString(body, h) + return rs.status + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + diff --git a/api/boto/contrib/__init__.py b/api/boto/contrib/__init__.py new file mode 100644 index 0000000..303dbb6 --- /dev/null +++ b/api/boto/contrib/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/api/boto/contrib/m2helpers.py b/api/boto/contrib/m2helpers.py new file mode 100644 index 0000000..82d2730 --- /dev/null +++ b/api/boto/contrib/m2helpers.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006,2007 Jon Colverson +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module was contributed by Jon Colverson. It provides a couple of helper +functions that allow you to use M2Crypto's implementation of HTTPSConnection +rather than the default version in httplib.py. The main benefit is that +M2Crypto's version verifies the certificate of the server. + +To use this feature, do something like this: + +from boto.ec2.connection import EC2Connection + +ec2 = EC2Connection(ACCESS_KEY_ID, SECRET_ACCESS_KEY, + https_connection_factory=https_connection_factory(cafile=CA_FILE)) + +See http://code.google.com/p/boto/issues/detail?id=57 for more details. +""" +from M2Crypto import SSL +from M2Crypto.httpslib import HTTPSConnection + +def secure_context(cafile=None, capath=None): + ctx = SSL.Context() + ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, depth=9) + if ctx.load_verify_locations(cafile=cafile, capath=capath) != 1: + raise Exception("Couldn't load certificates") + return ctx + +def https_connection_factory(cafile=None, capath=None): + def factory(*args, **kwargs): + return HTTPSConnection( + ssl_context=secure_context(cafile=cafile, capath=capath), + *args, **kwargs) + return (factory, (SSL.SSLError,)) diff --git a/api/boto/contrib/ymlmessage.py b/api/boto/contrib/ymlmessage.py new file mode 100644 index 0000000..22e5c62 --- /dev/null +++ b/api/boto/contrib/ymlmessage.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006,2007 Chris Moyer +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module was contributed by Chris Moyer. It provides a subclass of the +SQS Message class that supports YAML as the body of the message. + +This module requires the yaml module. +""" +from boto.sqs.message import Message +import yaml + +class YAMLMessage(Message): + """ + The YAMLMessage class provides a YAML compatible message. Encoding and + decoding are handled automaticaly. + + Access this message data like such: + + m.data = [ 1, 2, 3] + m.data[0] # Returns 1 + + This depends on the PyYAML package + """ + + def __init__(self, queue=None, body='', xml_attrs=None): + self.data = None + Message.__init__(self, queue, body) + + def set_body(self, body): + self.data = yaml.load(body) + + def get_body(self): + return yaml.dump(self.data) diff --git a/api/boto/ec2/__init__.py b/api/boto/ec2/__init__.py new file mode 100644 index 0000000..8bb3f53 --- /dev/null +++ b/api/boto/ec2/__init__.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +service from AWS. +""" +from boto.ec2.connection import EC2Connection + +def regions(**kw_params): + """ + Get all available regions for the EC2 service. + You may pass any of the arguments accepted by the EC2Connection + object's constructor as keyword arguments and they will be + passed along to the EC2Connection object. + + :rtype: list + :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` + """ + c = EC2Connection(**kw_params) + return c.get_all_regions() + +def connect_to_region(region_name, **kw_params): + for region in regions(**kw_params): + if region.name == region_name: + return region.connect(**kw_params) + return None + +def get_region(region_name, **kw_params): + for region in regions(**kw_params): + if region.name == region_name: + return region + return None + diff --git a/api/boto/ec2/address.py b/api/boto/ec2/address.py new file mode 100644 index 0000000..b2af107 --- /dev/null +++ b/api/boto/ec2/address.py @@ -0,0 +1,51 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Elastic IP Address +""" + +from boto.ec2.ec2object import EC2Object + +class Address(EC2Object): + + def __init__(self, connection=None, public_ip=None, instance_id=None): + EC2Object.__init__(self, connection) + self.connection = connection + self.public_ip = public_ip + self.instance_id = instance_id + + def __repr__(self): + return 'Address:%s' % self.public_ip + + def endElement(self, name, value, connection): + if name == 'publicIp': + self.public_ip = value + elif name == 'instanceId': + self.instance_id = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_address(self.public_ip) + + + diff --git a/api/boto/ec2/autoscale/__init__.py b/api/boto/ec2/autoscale/__init__.py new file mode 100644 index 0000000..d7c5946 --- /dev/null +++ b/api/boto/ec2/autoscale/__init__.py @@ -0,0 +1,206 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +Auto Scaling service. +""" + +import boto +from boto import config +from boto.connection import AWSQueryConnection +from boto.resultset import ResultSet +from boto.ec2.regioninfo import RegionInfo +from boto.ec2.autoscale.request import Request +from boto.ec2.autoscale.trigger import Trigger +from boto.ec2.autoscale.launchconfig import LaunchConfiguration +from boto.ec2.autoscale.group import AutoScalingGroup +from boto.ec2.autoscale.activity import Activity + + +class AutoScaleConnection(AWSQueryConnection): + APIVersion = boto.config.get('Boto', 'autoscale_version', '2009-05-15') + Endpoint = boto.config.get('Boto', 'autoscale_endpoint', + 'autoscaling.amazonaws.com') + SignatureVersion = '2' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=Endpoint, debug=1, + https_connection_factory=None, region=None, path='/'): + """ + Init method to create a new connection to the AutoScaling service. + + B{Note:} The host argument is overridden by the host specified in the + boto configuration file. + """ + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, host, debug, + https_connection_factory, path=path) + + def build_list_params(self, params, items, label): + """ items is a list of dictionaries or strings: + [{'Protocol' : 'HTTP', + 'LoadBalancerPort' : '80', + 'InstancePort' : '80'},..] etc. + or + ['us-east-1b',...] + """ + # different from EC2 list params + for i in xrange(1, len(items)+1): + if isinstance(items[i-1], dict): + for k, v in items[i-1].iteritems(): + params['%s.member.%d.%s' % (label, i, k)] = v + elif isinstance(items[i-1], basestring): + params['%s.member.%d' % (label, i)] = items[i-1] + + def _update_group(self, op, as_group): + params = { + 'AutoScalingGroupName' : as_group.name, + 'Cooldown' : as_group.cooldown, + 'LaunchConfigurationName' : as_group.launch_config_name, + 'MinSize' : as_group.min_size, + 'MaxSize' : as_group.max_size, + } + if op.startswith('Create'): + if as_group.availability_zones: + zones = self.availability_zones + else: + zones = [as_group.availability_zone] + self.build_list_params(params, as_group.load_balancers, + 'LoadBalancerNames') + self.build_list_params(params, zones, + 'AvailabilityZones') + return self.get_object(op, params, Request) + + def create_auto_scaling_group(self, as_group): + """ + Create auto scaling group. + """ + return self._update_group('CreateAutoScalingGroup', as_group) + + def create_launch_configuration(self, launch_config): + """ + Creates a new Launch Configuration. + + :type launch_config: boto.ec2.autoscale.launchconfig.LaunchConfiguration + :param launch_config: LaunchConfiguraiton object. + + """ + params = { + 'ImageId' : launch_config.image_id, + 'KeyName' : launch_config.key_name, + 'LaunchConfigurationName' : launch_config.name, + 'InstanceType' : launch_config.instance_type, + } + if launch_config.user_data: + params['UserData'] = launch_config.user_data + if launch_config.kernel_id: + params['KernelId'] = launch_config.kernel_id + if launch_config.ramdisk_id: + params['RamdiskId'] = launch_config.ramdisk_id + if launch_config.block_device_mappings: + self.build_list_params(params, launch_config.block_device_mappings, + 'BlockDeviceMappings') + self.build_list_params(params, launch_config.security_groups, + 'SecurityGroups') + return self.get_object('CreateLaunchConfiguration', params, + Request) + + def create_trigger(self, trigger): + """ + + """ + params = {'TriggerName' : trigger.name, + 'AutoScalingGroupName' : trigger.autoscale_group.name, + 'MeasureName' : trigger.measure_name, + 'Statistic' : trigger.statistic, + 'Period' : trigger.period, + 'Unit' : trigger.unit, + 'LowerThreshold' : trigger.lower_threshold, + 'LowerBreachScaleIncrement' : trigger.lower_breach_scale_increment, + 'UpperThreshold' : trigger.upper_threshold, + 'UpperBreachScaleIncrement' : trigger.upper_breach_scale_increment, + 'BreachDuration' : trigger.breach_duration} + # dimensions should be a list of tuples + dimensions = [] + for dim in trigger.dimensions: + name, value = dim + dimensions.append(dict(Name=name, Value=value)) + self.build_list_params(params, dimensions, 'Dimensions') + + req = self.get_object('CreateOrUpdateScalingTrigger', params, + Request) + return req + + def get_all_groups(self, names=None): + """ + """ + params = {} + if names: + self.build_list_params(params, names, 'AutoScalingGroupNames') + return self.get_list('DescribeAutoScalingGroups', params, + [('member', AutoScalingGroup)]) + + def get_all_launch_configurations(self, names=None): + """ + """ + params = {} + if names: + self.build_list_params(params, names, 'LaunchConfigurationNames') + return self.get_list('DescribeLaunchConfigurations', params, + [('member', LaunchConfiguration)]) + + def get_all_activities(self, autoscale_group, + activity_ids=None, + max_records=100): + """ + Get all activities for the given autoscaling group. + + :type autoscale_group: str or AutoScalingGroup object + :param autoscale_group: The auto scaling group to get activities on. + + @max_records: int + :param max_records: Maximum amount of activities to return. + """ + name = autoscale_group + if isinstance(autoscale_group, AutoScalingGroup): + name = autoscale_group.name + params = {'AutoScalingGroupName' : name} + if activity_ids: + self.build_list_params(params, activity_ids, 'ActivityIds') + return self.get_list('DescribeScalingActivities', params, + [('member', Activity)]) + + def get_all_triggers(self, autoscale_group): + params = {'AutoScalingGroupName' : autoscale_group} + return self.get_list('DescribeTriggers', params, + [('member', Trigger)]) + + def terminate_instance(self, instance_id, decrement_capacity=True): + params = { + 'InstanceId' : instance_id, + 'ShouldDecrementDesiredCapacity' : decrement_capacity + } + return self.get_object('TerminateInstanceInAutoScalingGroup', params, + Activity) + diff --git a/api/boto/ec2/autoscale/activity.py b/api/boto/ec2/autoscale/activity.py new file mode 100644 index 0000000..f895d65 --- /dev/null +++ b/api/boto/ec2/autoscale/activity.py @@ -0,0 +1,55 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +class Activity(object): + def __init__(self, connection=None): + self.connection = connection + self.start_time = None + self.activity_id = None + self.progress = None + self.status_code = None + self.cause = None + self.description = None + + def __repr__(self): + return 'Activity:%s status:%s progress:%s' % (self.description, + self.status_code, + self.progress) + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'ActivityId': + self.activity_id = value + elif name == 'StartTime': + self.start_time = value + elif name == 'Progress': + self.progress = value + elif name == 'Cause': + self.cause = value + elif name == 'Description': + self.description = value + elif name == 'StatusCode': + self.status_code = value + else: + setattr(self, name, value) + diff --git a/api/boto/ec2/autoscale/group.py b/api/boto/ec2/autoscale/group.py new file mode 100644 index 0000000..d9df39f --- /dev/null +++ b/api/boto/ec2/autoscale/group.py @@ -0,0 +1,190 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import weakref + +from boto.ec2.zone import Zone +from boto.ec2.elb.listelement import ListElement +from boto.resultset import ResultSet +from boto.ec2.autoscale.trigger import Trigger +from boto.ec2.autoscale.request import Request + +class Instance(object): + def __init__(self, connection=None): + self.connection = connection + self.instance_id = '' + + def __repr__(self): + return 'Instance:%s' % self.instance_id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'InstanceId': + self.instance_id = value + else: + setattr(self, name, value) + + +class AutoScalingGroup(object): + def __init__(self, connection=None, group_name=None, + availability_zone=None, launch_config=None, + availability_zones=None, + load_balancers=None, cooldown=0, + min_size=None, max_size=None): + """ + Creates a new AutoScalingGroup with the specified name. + + You must not have already used up your entire quota of + AutoScalingGroups in order for this call to be successful. Once the + creation request is completed, the AutoScalingGroup is ready to be + used in other calls. + + :type name: str + :param name: Name of autoscaling group. + + :type availability_zone: str + :param availability_zone: An availability zone. DEPRECATED - use the + availability_zones parameter, which expects + a list of availability zone + strings + + :type availability_zone: list + :param availability_zone: List of availability zones. + + :type launch_config: str + :param launch_config: Name of launch configuration name. + + :type load_balancers: list + :param load_balancers: List of load balancers. + + :type minsize: int + :param minsize: Minimum size of group + + :type maxsize: int + :param maxsize: Maximum size of group + + :type cooldown: int + :param cooldown: Amount of time after a Scaling Activity completes + before any further scaling activities can start. + + :rtype: tuple + :return: Updated healthcheck for the instances. + """ + self.name = group_name + self.connection = connection + self.min_size = min_size + self.max_size = max_size + self.created_time = None + self.cooldown = cooldown + self.launch_config = launch_config + if self.launch_config: + self.launch_config_name = self.launch_config.name + else: + self.launch_config_name = None + self.desired_capacity = None + lbs = load_balancers or [] + self.load_balancers = ListElement(lbs) + zones = availability_zones or [] + self.availability_zone = availability_zone + self.availability_zones = ListElement(zones) + self.instances = None + + def __repr__(self): + return 'AutoScalingGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'Instances': + self.instances = ResultSet([('member', Instance)]) + return self.instances + elif name == 'LoadBalancerNames': + return self.load_balancers + elif name == 'AvailabilityZones': + return self.availability_zones + else: + return + + def endElement(self, name, value, connection): + if name == 'MinSize': + self.min_size = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'Cooldown': + self.cooldown = value + elif name == 'LaunchConfigurationName': + self.launch_config_name = value + elif name == 'DesiredCapacity': + self.desired_capacity = value + elif name == 'MaxSize': + self.max_size = value + elif name == 'AutoScalingGroupName': + self.name = value + else: + setattr(self, name, value) + + def set_capacity(self, capacity): + """ Set the desired capacity for the group. """ + params = { + 'AutoScalingGroupName' : self.name, + 'DesiredCapacity' : capacity, + } + req = self.connection.get_object('SetDesiredCapacity', params, + Request) + self.connection.last_request = req + return req + + def update(self): + """ Sync local changes with AutoScaling group. """ + return self.connection._update_group('UpdateAutoScalingGroup', self) + + def shutdown_instances(self): + """ Convenience method which shuts down all instances associated with + this group. + """ + self.min_size = 0 + self.max_size = 0 + self.update() + + def get_all_triggers(self): + """ Get all triggers for this auto scaling group. """ + params = {'AutoScalingGroupName' : self.name} + triggers = self.connection.get_list('DescribeTriggers', params, + [('member', Trigger)]) + + # allow triggers to be able to access the autoscale group + for tr in triggers: + tr.autoscale_group = weakref.proxy(self) + + return triggers + + def delete(self): + """ Delete this auto-scaling group. """ + params = {'AutoScalingGroupName' : self.name} + return self.connection.get_object('DeleteAutoScalingGroup', params, + Request) + + def get_activities(self, activity_ids=None, max_records=100): + """ + Get all activies for this group. + """ + return self.connection.get_all_activities(self, activity_ids, max_records) + diff --git a/api/boto/ec2/autoscale/instance.py b/api/boto/ec2/autoscale/instance.py new file mode 100644 index 0000000..33f2ae6 --- /dev/null +++ b/api/boto/ec2/autoscale/instance.py @@ -0,0 +1,51 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +class Instance(object): + def __init__(self, connection=None): + self.connection = connection + self.instance_id = '' + self.lifecycle_state = None + self.availability_zone = '' + + def __repr__(self): + return 'Instance:%s' % self.instance_id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'InstanceId': + self.instance_id = value + elif name == 'LifecycleState': + self.lifecycle_state = value + elif name == 'AvailabilityZone': + self.availability_zone = value + else: + setattr(self, name, value) + + def terminate(self): + """ Terminate this instance. """ + params = {'LaunchConfigurationName' : self.instance_id} + return self.get_object('DeleteLaunchConfiguration', params, + Request) + diff --git a/api/boto/ec2/autoscale/launchconfig.py b/api/boto/ec2/autoscale/launchconfig.py new file mode 100644 index 0000000..7587cb6 --- /dev/null +++ b/api/boto/ec2/autoscale/launchconfig.py @@ -0,0 +1,98 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +from boto.ec2.autoscale.request import Request +from boto.ec2.elb.listelement import ListElement + + +class LaunchConfiguration(object): + def __init__(self, connection=None, name=None, image_id=None, + key_name=None, security_groups=None, user_data=None, + instance_type='m1.small', kernel_id=None, + ramdisk_id=None, block_device_mappings=None): + """ + A launch configuration. + + :type name: str + :param name: Name of the launch configuration to create. + + :type image_id: str + :param image_id: Unique ID of the Amazon Machine Image (AMI) which was + assigned during registration. + + :type key_name: str + :param key_name: The name of the EC2 key pair. + + :type security_groups: list + :param security_groups: Names of the security groups with which to + associate the EC2 instances. + + """ + self.connection = connection + self.name = name + self.instance_type = instance_type + self.block_device_mappings = block_device_mappings + self.key_name = key_name + sec_groups = security_groups or [] + self.security_groups = ListElement(sec_groups) + self.image_id = image_id + self.ramdisk_id = ramdisk_id + self.created_time = None + self.kernel_id = kernel_id + self.user_data = user_data + self.created_time = None + + def __repr__(self): + return 'LaunchConfiguration:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'SecurityGroups': + return self.security_groups + else: + return + + def endElement(self, name, value, connection): + if name == 'InstanceType': + self.instance_type = value + elif name == 'LaunchConfigurationName': + self.name = value + elif name == 'KeyName': + self.key_name = value + elif name == 'ImageId': + self.image_id = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'KernelId': + self.kernel_id = value + elif name == 'RamdiskId': + self.ramdisk_id = value + elif name == 'UserData': + self.user_data = value + else: + setattr(self, name, value) + + def delete(self): + """ Delete this launch configuration. """ + params = {'LaunchConfigurationName' : self.name} + return self.connection.get_object('DeleteLaunchConfiguration', params, + Request) + diff --git a/api/boto/ec2/autoscale/request.py b/api/boto/ec2/autoscale/request.py new file mode 100644 index 0000000..c066dff --- /dev/null +++ b/api/boto/ec2/autoscale/request.py @@ -0,0 +1,38 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Request(object): + def __init__(self, connection=None): + self.connection = connection + self.request_id = '' + + def __repr__(self): + return 'Request:%s' % self.request_id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'RequestId': + self.request_id = value + else: + setattr(self, name, value) + diff --git a/api/boto/ec2/autoscale/trigger.py b/api/boto/ec2/autoscale/trigger.py new file mode 100644 index 0000000..197803d --- /dev/null +++ b/api/boto/ec2/autoscale/trigger.py @@ -0,0 +1,137 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import weakref + +from boto.ec2.autoscale.request import Request + + +class Trigger(object): + """ + An auto scaling trigger. + """ + + def __init__(self, connection=None, name=None, autoscale_group=None, + dimensions=None, measure_name=None, + statistic=None, unit=None, period=60, + lower_threshold=None, + lower_breach_scale_increment=None, + upper_threshold=None, + upper_breach_scale_increment=None, + breach_duration=None): + """ + Initialize an auto-scaling trigger object. + + :type name: str + :param name: The name for this trigger + + :type autoscale_group: str + :param autoscale_group: The name of the AutoScalingGroup that will be + associated with the trigger. The AutoScalingGroup + that will be affected by the trigger when it is + activated. + + :type dimensions: list + :param dimensions: List of tuples, i.e. + ('ImageId', 'i-13lasde') etc. + + :type measure_name: str + :param measure_name: The measure name associated with the metric used by + the trigger to determine when to activate, for + example, CPU, network I/O, or disk I/O. + + :type statistic: str + :param statistic: The particular statistic used by the trigger when + fetching metric statistics to examine. + + :type period: int + :param period: The period associated with the metric statistics in + seconds. Valid Values: 60 or a multiple of 60. + + :type unit: + :param unit + + :type lower_threshold: + :param lower_threshold + """ + self.name = name + self.connection = connection + self.dimensions = dimensions + self.breach_duration = breach_duration + self.upper_breach_scale_increment = upper_breach_scale_increment + self.created_time = None + self.upper_threshold = upper_threshold + self.status = None + self.lower_threshold = lower_threshold + self.period = period + self.lower_breach_scale_increment = lower_breach_scale_increment + self.statistic = statistic + self.unit = unit + self.namespace = None + if autoscale_group: + self.autoscale_group = weakref.proxy(autoscale_group) + else: + self.autoscale_group = None + self.measure_name = measure_name + + def __repr__(self): + return 'Trigger:%s' % (self.name) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'BreachDuration': + self.breach_duration = value + elif name == 'TriggerName': + self.name = value + elif name == 'Period': + self.period = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'Statistic': + self.statistic = value + elif name == 'Unit': + self.unit = value + elif name == 'Namespace': + self.namespace = value + elif name == 'AutoScalingGroupName': + self.autoscale_group_name = value + elif name == 'MeasureName': + self.measure_name = value + else: + setattr(self, name, value) + + def update(self): + """ Write out differences to trigger. """ + self.connection.create_trigger(self) + + def delete(self): + """ Delete this trigger. """ + params = { + 'TriggerName' : self.name, + 'AutoScalingGroupName' : self.autoscale_group_name, + } + req =self.connection.get_object('DeleteTrigger', params, + Request) + self.connection.last_request = req + return req + diff --git a/api/boto/ec2/blockdevicemapping.py b/api/boto/ec2/blockdevicemapping.py new file mode 100644 index 0000000..ef7163a --- /dev/null +++ b/api/boto/ec2/blockdevicemapping.py @@ -0,0 +1,89 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +class EBSBlockDeviceType(object): + + def __init__(self, connection=None): + self.connection = connection + self.volume_id = None + self.snapshot_id = None + self.status = None + self.attach_time = None + self.delete_on_termination = False + self.size = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name =='volumeId': + self.volume_id = value + elif name =='snapshotId': + self.snapshot_id = value + elif name == 'volumeSize': + self.size = int(value) + elif name == 'status': + self.status = value + elif name == 'attachTime': + self.attach_time = value + elif name == 'deleteOnTermination': + if value == 'true': + self.delete_on_termination = True + else: + self.delete_on_termination = False + else: + setattr(self, name, value) + +class BlockDeviceMapping(dict): + + def __init__(self, connection=None): + dict.__init__(self) + self.connection = connection + self.current_name = None + self.current_value = None + + def startElement(self, name, attrs, connection): + if name == 'ebs': + self.current_value = EBSBlockDeviceType(self) + return self.current_value + + def endElement(self, name, value, connection): + if name == 'device' or name == 'deviceName': + self.current_name = value + elif name == 'item': + self[self.current_name] = self.current_value + + def build_list_params(self, params, prefix=''): + i = 1 + for dev_name in self: + pre = '%sBlockDeviceMapping.%d' % (pre, i) + params['%s.DeviceName' % pre] = dev_name + ebs = self[dev_name] + if ebs.snapshot_id: + params['%s.Ebs.SnapshotId' % pre] = ebs.snapshot_id + if ebs.size: + params['%s.Ebs.VolumeSize' % pre] = ebs.size + if ebs.delete_on_termination: + params['%s.Ebs.DeleteOnTermination' % pre] = 'true' + else: + params['%s.Ebs.DeleteOnTermination' % pre] = 'false' + i += 1 diff --git a/api/boto/ec2/buyreservation.py b/api/boto/ec2/buyreservation.py new file mode 100644 index 0000000..ba65590 --- /dev/null +++ b/api/boto/ec2/buyreservation.py @@ -0,0 +1,81 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto.ec2 +from boto.sdb.db.property import * +from boto.manage import propget + +InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', 'c1.medium', 'c1.xlarge'] + +class BuyReservation(object): + + def get_region(self, params): + if not params.get('region', None): + prop = StringProperty(name='region', verbose_name='EC2 Region', + choices=boto.ec2.regions) + params['region'] = propget.get(prop, choices=boto.ec2.regions) + + def get_instance_type(self, params): + if not params.get('instance_type', None): + prop = StringProperty(name='instance_type', verbose_name='Instance Type', + choices=InstanceTypes) + params['instance_type'] = propget.get(prop) + + def get_quantity(self, params): + if not params.get('quantity', None): + prop = IntegerProperty(name='quantity', verbose_name='Number of Instances') + params['quantity'] = propget.get(prop) + + def get_zone(self, params): + if not params.get('zone', None): + prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', + choices=self.ec2.get_all_zones) + params['zone'] = propget.get(prop) + + def get(self, params): + self.get_region(params) + self.ec2 = params['region'].connect() + self.get_instance_type(params) + self.get_zone(params) + self.get_quantity(params) + +if __name__ == "__main__": + obj = BuyReservation() + params = {} + obj.get(params) + offerings = obj.ec2.get_all_reserved_instances_offerings(instance_type=params['instance_type'], + availability_zone=params['zone'].name) + print '\nThe following Reserved Instances Offerings are available:\n' + for offering in offerings: + offering.describe() + prop = StringProperty(name='offering', verbose_name='Offering', + choices=offerings) + offering = propget.get(prop) + print '\nYou have chosen this offering:' + offering.describe() + unit_price = float(offering.fixed_price) + total_price = unit_price * params['quantity'] + print '!!! You are about to purchase %d of these offerings for a total of $%.2f !!!' % (params['quantity'], total_price) + answer = raw_input('Are you sure you want to do this? If so, enter YES: ') + if answer.strip().lower() == 'yes': + offering.purchase(params['quantity']) + else: + print 'Purchase cancelled' diff --git a/api/boto/ec2/cloudwatch/__init__.py b/api/boto/ec2/cloudwatch/__init__.py new file mode 100644 index 0000000..1c606a1 --- /dev/null +++ b/api/boto/ec2/cloudwatch/__init__.py @@ -0,0 +1,206 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +CloudWatch service from AWS. + +The 5 Minute How-To Guide +------------------------- +First, make sure you have something to monitor. You can either create a +LoadBalancer or enable monitoring on an existing EC2 instance. To enable +monitoring, you can either call the monitor_instance method on the +EC2Connection object or call the monitor method on the Instance object. + +It takes a while for the monitoring data to start accumulating but once +it does, you can do this: + +>>> import boto +>>> c = boto.connect_cloudwatch() +>>> metrics = c.list_metrics() +>>> metrics +[Metric:NetworkIn, + Metric:NetworkOut, + Metric:NetworkOut(InstanceType,m1.small), + Metric:NetworkIn(InstanceId,i-e573e68c), + Metric:CPUUtilization(InstanceId,i-e573e68c), + Metric:DiskWriteBytes(InstanceType,m1.small), + Metric:DiskWriteBytes(ImageId,ami-a1ffb63), + Metric:NetworkOut(ImageId,ami-a1ffb63), + Metric:DiskWriteOps(InstanceType,m1.small), + Metric:DiskReadBytes(InstanceType,m1.small), + Metric:DiskReadOps(ImageId,ami-a1ffb63), + Metric:CPUUtilization(InstanceType,m1.small), + Metric:NetworkIn(ImageId,ami-a1ffb63), + Metric:DiskReadOps(InstanceType,m1.small), + Metric:DiskReadBytes, + Metric:CPUUtilization, + Metric:DiskWriteBytes(InstanceId,i-e573e68c), + Metric:DiskWriteOps(InstanceId,i-e573e68c), + Metric:DiskWriteOps, + Metric:DiskReadOps, + Metric:CPUUtilization(ImageId,ami-a1ffb63), + Metric:DiskReadOps(InstanceId,i-e573e68c), + Metric:NetworkOut(InstanceId,i-e573e68c), + Metric:DiskReadBytes(ImageId,ami-a1ffb63), + Metric:DiskReadBytes(InstanceId,i-e573e68c), + Metric:DiskWriteBytes, + Metric:NetworkIn(InstanceType,m1.small), + Metric:DiskWriteOps(ImageId,ami-a1ffb63)] + +The list_metrics call will return a list of all of the available metrics +that you can query against. Each entry in the list is a Metric object. +As you can see from the list above, some of the metrics are generic metrics +and some have Dimensions associated with them (e.g. InstanceType=m1.small). +The Dimension can be used to refine your query. So, for example, I could +query the metric Metric:CPUUtilization which would create the desired statistic +by aggregating cpu utilization data across all sources of information available +or I could refine that by querying the metric +Metric:CPUUtilization(InstanceId,i-e573e68c) which would use only the data +associated with the instance identified by the instance ID i-e573e68c. + +Because for this example, I'm only monitoring a single instance, the set +of metrics available to me are fairly limited. If I was monitoring many +instances, using many different instance types and AMI's and also several +load balancers, the list of available metrics would grow considerably. + +Once you have the list of available metrics, you can actually +query the CloudWatch system for that metric. Let's choose the CPU utilization +metric for our instance. + +>>> m = metrics[5] +>>> m +Metric:CPUUtilization(InstanceId,i-e573e68c) + +The Metric object has a query method that lets us actually perform +the query against the collected data in CloudWatch. To call that, +we need a start time and end time to control the time span of data +that we are interested in. For this example, let's say we want the +data for the previous hour: + +>>> import datetime +>>> end = datetime.datetime.now() +>>> start = end - datetime.timedelta(hours=1) + +We also need to supply the Statistic that we want reported and +the Units to use for the results. The Statistic can be one of these +values: + +['Minimum', 'Maximum', 'Sum', 'Average', 'Samples'] + +And Units must be one of the following: + +['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', +'Bytes/Second', 'Bits/Second', 'Count/Second'] + +The query method also takes an optional parameter, period. This +parameter controls the granularity (in seconds) of the data returned. +The smallest period is 60 seconds and the value must be a multiple +of 60 seconds. So, let's ask for the average as a percent: + +>>> datapoints = m.query(start, end, 'Average', 'Percent') +>>> len(datapoints) +60 + +Our period was 60 seconds and our duration was one hour so +we should get 60 data points back and we can see that we did. +Each element in the datapoints list is a DataPoint object +which is a simple subclass of a Python dict object. Each +Datapoint object contains all of the information available +about that particular data point. + +>>> d = datapoints[0] +>>> d +{u'Average': 0.0, + u'Samples': 1.0, + u'Timestamp': u'2009-05-21T19:55:00Z', + u'Unit': u'Percent'} + +My server obviously isn't very busy right now! +""" +from boto.connection import AWSQueryConnection +from boto.ec2.cloudwatch.metric import Metric +from boto.ec2.cloudwatch.datapoint import Datapoint +import boto +import datetime + +class CloudWatchConnection(AWSQueryConnection): + + APIVersion = boto.config.get('Boto', 'cloudwatch_version', '2009-05-15') + Endpoint = boto.config.get('Boto', 'cloudwatch_endpoint', 'monitoring.amazonaws.com') + SignatureVersion = '2' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=Endpoint, debug=0, + https_connection_factory=None, path='/'): + """ + Init method to create a new connection to EC2 Monitoring Service. + + B{Note:} The host argument is overridden by the host specified in the boto configuration file. + """ + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory, path) + + def build_list_params(self, params, items, label): + if isinstance(items, str): + items = [items] + for i in range(1, len(items)+1): + params[label % i] = items[i-1] + + def get_metric_statistics(self, period, start_time, end_time, measure_name, + namespace, statistics=None, dimensions=None, unit=None): + """ + Get time-series data for one or more statistics of a given metric. + + :type measure_name: string + :param measure_name: CPUUtilization|NetworkIO-in|NetworkIO-out|DiskIO-ALL-read| + DiskIO-ALL-write|DiskIO-ALL-read-bytes|DiskIO-ALL-write-bytes + + :rtype: list + :return: A list of :class:`boto.ec2.image.Image` + """ + params = {'Period' : period, + 'MeasureName' : measure_name, + 'Namespace' : namespace, + 'StartTime' : start_time.isoformat(), + 'EndTime' : end_time.isoformat()} + if dimensions: + i = 1 + for name in dimensions: + params['Dimensions.member.%d.Name' % i] = name + params['Dimensions.member.%d.Value' % i] = dimensions[name] + i += 1 + if statistics: + self.build_list_params(params, statistics, 'Statistics.member.%d') + return self.get_list('GetMetricStatistics', params, [('member', Datapoint)]) + + def list_metrics(self): + """ + Returns a list of the valid metrics for which there is recorded data available. + """ + response = self.make_request('ListMetrics') + body = response.read() + return self.get_list('ListMetrics', None, [('member', Metric)]) + + + diff --git a/api/boto/ec2/cloudwatch/datapoint.py b/api/boto/ec2/cloudwatch/datapoint.py new file mode 100644 index 0000000..1860f4a --- /dev/null +++ b/api/boto/ec2/cloudwatch/datapoint.py @@ -0,0 +1,37 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +class Datapoint(dict): + + def __init__(self, connection=None): + dict.__init__(self) + self.connection = connection + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name in ['Average', 'Maximum', 'Minimum', 'Samples', 'Sum']: + self[name] = float(value) + elif name != 'member': + self[name] = value + diff --git a/api/boto/ec2/cloudwatch/metric.py b/api/boto/ec2/cloudwatch/metric.py new file mode 100644 index 0000000..e4661f4 --- /dev/null +++ b/api/boto/ec2/cloudwatch/metric.py @@ -0,0 +1,71 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +class Dimensions(dict): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'Name': + self._name = value + elif name == 'Value': + self[self._name] = value + elif name != 'Dimensions' and name != 'member': + self[name] = value + +class Metric(object): + + Statistics = ['Minimum', 'Maximum', 'Sum', 'Average', 'Samples'] + Units = ['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', + 'Bytes/Second', 'Bits/Second', 'Count/Second'] + + def __init__(self, connection=None): + self.connection = connection + self.name = None + self.namespace = None + self.dimensions = None + + def __repr__(self): + s = 'Metric:%s' % self.name + if self.dimensions: + for name,value in self.dimensions.items(): + s += '(%s,%s)' % (name, value) + return s + + def startElement(self, name, attrs, connection): + if name == 'Dimensions': + self.dimensions = Dimensions() + return self.dimensions + + def endElement(self, name, value, connection): + if name == 'MeasureName': + self.name = value + elif name == 'Namespace': + self.namespace = value + else: + setattr(self, name, value) + + def query(self, start_time, end_time, statistic, unit, period=60): + return self.connection.get_metric_statistics(period, start_time, end_time, + self.name, self.namespace, [statistic], + self.dimensions, unit) diff --git a/api/boto/ec2/connection.py b/api/boto/ec2/connection.py new file mode 100644 index 0000000..9574986 --- /dev/null +++ b/api/boto/ec2/connection.py @@ -0,0 +1,1520 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a connection to the EC2 service. +""" + +import urllib +import xml.sax +import base64 +import boto +from boto import config +from boto.connection import AWSQueryConnection +from boto.resultset import ResultSet +from boto.ec2.image import Image, ImageAttribute +from boto.ec2.instance import Reservation, Instance, ConsoleOutput, InstanceAttribute +from boto.ec2.keypair import KeyPair +from boto.ec2.address import Address +from boto.ec2.volume import Volume +from boto.ec2.snapshot import Snapshot +from boto.ec2.snapshot import SnapshotAttribute +from boto.ec2.zone import Zone +from boto.ec2.securitygroup import SecurityGroup +from boto.ec2.regioninfo import RegionInfo +from boto.ec2.instanceinfo import InstanceInfo +from boto.ec2.reservedinstance import ReservedInstancesOffering, ReservedInstance +from boto.ec2.spotinstancerequest import SpotInstanceRequest +from boto.ec2.spotpricehistory import SpotPriceHistory +from boto.ec2.spotdatafeedsubscription import SpotDatafeedSubscription +from boto.ec2.launchspecification import LaunchSpecification +from boto.exception import EC2ResponseError + +#boto.set_stream_logger('ec2') + +class EC2Connection(AWSQueryConnection): + + APIVersion = boto.config.get('Boto', 'ec2_version', '2009-11-30') + DefaultRegionName = boto.config.get('Boto', 'ec2_region_name', 'us-east-1') + DefaultRegionEndpoint = boto.config.get('Boto', 'ec2_region_endpoint', + 'ec2.amazonaws.com') + SignatureVersion = '2' + ResponseError = EC2ResponseError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, host=None, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/'): + """ + Init method to create a new connection to EC2. + + B{Note:} The host argument is overridden by the host specified in the boto configuration file. + """ + if not region: + region = RegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, + self.region.endpoint, debug, + https_connection_factory, path) + + def get_params(self): + """ + Returns a dictionary containing the value of of all of the keyword + arguments passed when constructing this connection. + """ + param_names = ['aws_access_key_id', 'aws_secret_access_key', 'is_secure', + 'port', 'proxy', 'proxy_port', 'proxy_user', 'proxy_pass', + 'debug', 'https_connection_factory'] + params = {} + for name in param_names: + params[name] = getattr(self, name) + return params + + # Image methods + + def get_all_images(self, image_ids=None, owners=None, executable_by=None): + """ + Retrieve all the EC2 images available on your account. + + :type image_ids: list + :param image_ids: A list of strings with the image IDs wanted + + :type owners: list + :param owners: A list of owner IDs + + :type executable_by: + :param executable_by: + + :rtype: list + :return: A list of :class:`boto.ec2.image.Image` + """ + params = {} + if image_ids: + self.build_list_params(params, image_ids, 'ImageId') + if owners: + self.build_list_params(params, owners, 'Owner') + if executable_by: + self.build_list_params(params, executable_by, 'ExecutableBy') + return self.get_list('DescribeImages', params, [('item', Image)]) + + def get_all_kernels(self, kernel_ids=None, owners=None): + """ + Retrieve all the EC2 kernels available on your account. Simply filters the list returned + by get_all_images because EC2 does not provide a way to filter server-side. + + :type kernel_ids: list + :param kernel_ids: A list of strings with the image IDs wanted + + :type owners: list + :param owners: A list of owner IDs + + :rtype: list + :return: A list of :class:`boto.ec2.image.Image` + """ + rs = self.get_all_images(kernel_ids, owners) + kernels = [] + for image in rs: + if image.type == 'kernel': + kernels.append(image) + return kernels + + def get_all_ramdisks(self, ramdisk_ids=None, owners=None): + """ + Retrieve all the EC2 ramdisks available on your account. + Simply filters the list returned by get_all_images because + EC2 does not provide a way to filter server-side. + + :type ramdisk_ids: list + :param ramdisk_ids: A list of strings with the image IDs wanted + + :type owners: list + :param owners: A list of owner IDs + + :rtype: list + :return: A list of :class:`boto.ec2.image.Image` + """ + rs = self.get_all_images(ramdisk_ids, owners) + ramdisks = [] + for image in rs: + if image.type == 'ramdisk': + ramdisks.append(image) + return ramdisks + + def get_image(self, image_id): + """ + Shortcut method to retrieve a specific image (AMI). + + :type image_id: string + :param image_id: the ID of the Image to retrieve + + :rtype: :class:`boto.ec2.image.Image` + :return: The EC2 Image specified or None if the image is not found + """ + try: + return self.get_all_images(image_ids=[image_id])[0] + except IndexError: # None of those images available + return None + + def register_image(self, name, description=None, image_location=None, + architecture=None, kernel_id=None, ramdisk_id=None, + root_device_name=None, block_device_map=None): + """ + Register an image. + + :type name: string + :param name: The name of the AMI. + + :type description: string + :param description: The description of the AMI. + + :type image_location: string + :param image_location: Full path to your AMI manifest in Amazon S3 storage. + Only used for S3-based AMI's. + + :type architecture: string + :param architecture: The architecture of the AMI. Valid choices are: + i386 | x86_64 + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the instances + + :type root_device_name: string + :param root_device_name: The root device name (e.g. /dev/sdh) + + :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` + :param block_device_map: A BlockDeviceMapping data structure + describing the EBS volumes associated + with the Image. + + :rtype: string + :return: The new image id + """ + params = {'Name': name} + if description: + params['Description'] = description + if architecture: + params['Architecture'] = architecture + if kernel_id: + params['KernelId'] = kernel_id + if ramdisk_id: + params['RamdiskId'] = ramdisk_id + if image_location: + params['Location'] = image_location + if root_device_name: + params['RootDeviceName'] = root_device_name + if block_device_map: + block_device_map.build_list_params(params) + rs = self.get_object('RegisterImage', params, ResultSet) + image_id = getattr(rs, 'imageId', None) + return image_id + + def deregister_image(self, image_id): + """ + Unregister an AMI. + + :type image_id: string + :param image_id: the ID of the Image to unregister + + :rtype: bool + :return: True if successful + """ + return self.get_status('DeregisterImage', {'ImageId':image_id}) + + def create_image(self, instance_id, name, description=None, no_reboot=False): + """ + Will create an AMI from the instance in the running or stopped + state. + + :type instance_id: string + :param instance_id: the ID of the instance to image. + + :type name: string + :param name: The name of the new image + + :type description: string + :param description: An optional human-readable string describing + the contents and purpose of the AMI. + + :type no_reboot: bool + :param no_reboot: An optional flag indicating that the bundling process + should not attempt to shutdown the instance before + bundling. If this flag is True, the responsibility + of maintaining file system integrity is left to the + owner of the instance. + + :rtype: string + :return: The new image id + """ + params = {'InstanceId' : instance_id, + 'Name' : name} + if description: + params['Description'] = description + if no_reboot: + params['NoReboot'] = 'true' + rs = self.get_object('CreateImage', params, Image) + image_id = getattr(rs, 'imageId', None) + if not image_id: + image_id = getattr(rs, 'ImageId', None) + return image_id + + # ImageAttribute methods + + def get_image_attribute(self, image_id, attribute='launchPermission'): + """ + Gets an attribute from an image. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-DescribeImageAttribute.html + + :type image_id: string + :param image_id: The Amazon image id for which you want info about + + :type attribute: string + :param attribute: The attribute you need information about. + Valid choices are: + * launchPermission + * productCodes + * blockDeviceMapping + + :rtype: :class:`boto.ec2.image.ImageAttribute` + :return: An ImageAttribute object representing the value of the attribute requested + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute} + return self.get_object('DescribeImageAttribute', params, ImageAttribute) + + def modify_image_attribute(self, image_id, attribute='launchPermission', + operation='add', user_ids=None, groups=None, + product_codes=None): + """ + Changes an attribute of an image. + See http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-ModifyImageAttribute.html + + :type image_id: string + :param image_id: The image id you wish to change + + :type attribute: string + :param attribute: The attribute you wish to change + + :type operation: string + :param operation: Either add or remove (this is required for changing launchPermissions) + + :type user_ids: list + :param user_ids: The Amazon IDs of users to add/remove attributes + + :type groups: list + :param groups: The groups to add/remove attributes + + :type product_codes: list + :param product_codes: Amazon DevPay product code. Currently only one + product code can be associated with an AMI. Once + set, the product code cannot be changed or reset. + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute, + 'OperationType' : operation} + if user_ids: + self.build_list_params(params, user_ids, 'UserId') + if groups: + self.build_list_params(params, groups, 'UserGroup') + if product_codes: + self.build_list_params(params, product_codes, 'ProductCode') + return self.get_status('ModifyImageAttribute', params) + + def reset_image_attribute(self, image_id, attribute='launchPermission'): + """ + Resets an attribute of an AMI to its default value. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-ResetImageAttribute.html + + :type image_id: string + :param image_id: ID of the AMI for which an attribute will be described + + :type attribute: string + :param attribute: The attribute to reset + + :rtype: bool + :return: Whether the operation succeeded or not + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute} + return self.get_status('ResetImageAttribute', params) + + # Instance methods + + def get_all_instances(self, instance_ids=None): + """ + Retrieve all the instances associated with your account. + + :type instance_ids: list + :param instance_ids: A list of strings of instance IDs + + :rtype: list + :return: A list of :class:`boto.ec2.instance.Reservation` + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('DescribeInstances', params, [('item', Reservation)]) + + def run_instances(self, image_id, min_count=1, max_count=1, + key_name=None, security_groups=None, + user_data=None, addressing_type=None, + instance_type='m1.small', placement=None, + kernel_id=None, ramdisk_id=None, + monitoring_enabled=False, subnet_id=None, + block_device_map=None): + """ + Runs an image on EC2. + + :type image_id: string + :param image_id: The ID of the image to run + + :type min_count: int + :param min_count: The minimum number of instances to launch + + :type max_count: int + :param max_count: The maximum number of instances to launch + + :type key_name: string + :param key_name: The name of the key pair with which to launch instances + + :type security_groups: list of strings + :param security_groups: The names of the security groups with which to associate instances + + :type user_data: string + :param user_data: The user data passed to the launched instances + + :type instance_type: string + :param instance_type: The type of instance to run (m1.small, m1.large, m1.xlarge) + + :type placement: string + :param placement: The availability zone in which to launch the instances + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the instances + + :type ramdisk_id: string + :param ramdisk_id: The ID of the RAM disk with which to launch the instances + + :type monitoring_enabled: bool + :param monitoring_enabled: Enable CloudWatch monitoring on the instance. + + :type subnet_id: string + :param subnet_id: The subnet ID within which to launch the instances for VPC. + + :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` + :param block_device_map: A BlockDeviceMapping data structure + describing the EBS volumes associated + with the Image. + + :rtype: Reservation + :return: The :class:`boto.ec2.instance.Reservation` associated with the request for machines + """ + params = {'ImageId':image_id, + 'MinCount':min_count, + 'MaxCount': max_count} + if key_name: + params['KeyName'] = key_name + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, SecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, 'SecurityGroup') + if user_data: + params['UserData'] = base64.b64encode(user_data) + if addressing_type: + params['AddressingType'] = addressing_type + if instance_type: + params['InstanceType'] = instance_type + if placement: + params['Placement.AvailabilityZone'] = placement + if kernel_id: + params['KernelId'] = kernel_id + if ramdisk_id: + params['RamdiskId'] = ramdisk_id + if monitoring_enabled: + params['Monitoring.Enabled'] = 'true' + if subnet_id: + params['SubnetId'] = subnet_id + if block_device_map: + block_device_map.build_list_params(params) + return self.get_object('RunInstances', params, Reservation, verb='POST') + + def terminate_instances(self, instance_ids=None): + """ + Terminate the instances specified + + :type instance_ids: list + :param instance_ids: A list of strings of the Instance IDs to terminate + + :rtype: list + :return: A list of the instances terminated + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('TerminateInstances', params, [('item', Instance)]) + + def stop_instances(self, instance_ids=None): + """ + Stop the instances specified + + :type instance_ids: list + :param instance_ids: A list of strings of the Instance IDs to stop + + :rtype: list + :return: A list of the instances stopped + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('StopInstances', params, [('item', Instance)]) + + def start_instances(self, instance_ids=None): + """ + Start the instances specified + + :type instance_ids: list + :param instance_ids: A list of strings of the Instance IDs to start + + :rtype: list + :return: A list of the instances started + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('StartInstances', params, [('item', Instance)]) + + def get_console_output(self, instance_id): + """ + Retrieves the console output for the specified instance. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-GetConsoleOutput.html + + :type instance_id: string + :param instance_id: The instance ID of a running instance on the cloud. + + :rtype: :class:`boto.ec2.instance.ConsoleOutput` + :return: The console output as a ConsoleOutput object + """ + params = {} + self.build_list_params(params, [instance_id], 'InstanceId') + return self.get_object('GetConsoleOutput', params, ConsoleOutput) + + def reboot_instances(self, instance_ids=None): + """ + Reboot the specified instances. + + :type instance_ids: list + :param instance_ids: The instances to terminate and reboot + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_status('RebootInstances', params) + + def confirm_product_instance(self, product_code, instance_id): + params = {'ProductCode' : product_code, + 'InstanceId' : instance_id} + rs = self.get_object('ConfirmProductInstance', params, ResultSet) + return (rs.status, rs.ownerId) + + # InstanceAttribute methods + + def get_instance_attribute(self, instance_id, attribute): + """ + Gets an attribute from an instance. + + :type instance_id: string + :param instance_id: The Amazon id of the instance + + :type attribute: string + :param attribute: The attribute you need information about + Valid choices are: + instanceType|kernel|ramdisk|userData| + disableApiTermination| + instanceInitiatedShutdownBehavior| + rootDeviceName|blockDeviceMapping + + :rtype: :class:`boto.ec2.image.ImageAttribute` + :return: An ImageAttribute object representing the value of the attribute requested + """ + params = {'InstanceId' : instance_id} + if attribute: + params['Attribute'] = attribute + return self.get_object('DescribeInstanceAttribute', params, InstanceAttribute) + + def modify_image_attribute(self, image_id, attribute='launchPermission', + operation='add', user_ids=None, groups=None, + product_codes=None): + """ + Changes an attribute of an image. + See http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-ModifyImageAttribute.html + + :type image_id: string + :param image_id: The image id you wish to change + + :type attribute: string + :param attribute: The attribute you wish to change + + :type operation: string + :param operation: Either add or remove (this is required for changing launchPermissions) + + :type user_ids: list + :param user_ids: The Amazon IDs of users to add/remove attributes + + :type groups: list + :param groups: The groups to add/remove attributes + + :type product_codes: list + :param product_codes: Amazon DevPay product code. Currently only one + product code can be associated with an AMI. Once + set, the product code cannot be changed or reset. + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute, + 'OperationType' : operation} + if user_ids: + self.build_list_params(params, user_ids, 'UserId') + if groups: + self.build_list_params(params, groups, 'UserGroup') + if product_codes: + self.build_list_params(params, product_codes, 'ProductCode') + return self.get_status('ModifyImageAttribute', params) + + def reset_image_attribute(self, image_id, attribute='launchPermission'): + """ + Resets an attribute of an AMI to its default value. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-ResetImageAttribute.html + + :type image_id: string + :param image_id: ID of the AMI for which an attribute will be described + + :type attribute: string + :param attribute: The attribute to reset + + :rtype: bool + :return: Whether the operation succeeded or not + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute} + return self.get_status('ResetImageAttribute', params) + + # Spot Instances + + def get_all_spot_instance_requests(self, request_ids=None): + """ + Retrieve all the spot instances requests associated with your account. + + @type request_ids: list + @param request_ids: A list of strings of spot instance request IDs + + @rtype: list + @return: A list of + :class:`boto.ec2.spotinstancerequest.SpotInstanceRequest` + """ + params = {} + if request_ids: + self.build_list_params(params, request_ids, 'SpotInstanceRequestId') + return self.get_list('DescribeSpotInstanceRequests', params, + [('item', SpotInstanceRequest)]) + + def get_spot_price_history(self, start_time=None, end_time=None, + instance_type=None, product_description=None): + """ + Retrieve the recent history of spot instances pricing. + + @type start_time: str + @param start_time: An indication of how far back to provide price + changes for. An ISO8601 DateTime string. + + @type end_time: str + @param end_time: An indication of how far forward to provide price + changes for. An ISO8601 DateTime string. + + @type instance_type: str + @param instance_type: Filter responses to a particular instance type. + + @type product_description: str + @param product_descripton: Filter responses to a particular platform. + Valid values are currently: Linux + + @rtype: list + @return: A list tuples containing price and timestamp. + """ + params = {} + if start_time: + params['StartTime'] = start_time + if end_time: + params['EndTime'] = end_time + if instance_type: + params['InstanceType'] = instance_type + if product_description: + params['ProductDescription'] = product_description + return self.get_list('DescribeSpotPriceHistory', params, [('item', SpotPriceHistory)]) + + def request_spot_instances(self, price, image_id, count=1, type=None, + valid_from=None, valid_until=None, + launch_group=None, availability_zone_group=None, + key_name=None, security_groups=None, + user_data=None, addressing_type=None, + instance_type='m1.small', placement=None, + kernel_id=None, ramdisk_id=None, + monitoring_enabled=False, subnet_id=None, + block_device_map=None): + """ + Request instances on the spot market at a particular price. + + :type price: str + :param price: The maximum price of your bid + + :type image_id: string + :param image_id: The ID of the image to run + + :type count: int + :param count: The of instances to requested + + :type type: str + :param type: Type of request. Can be 'one-time' or 'persistent'. + Default is one-time. + + :type valid_from: str + :param valid_from: Start date of the request. An ISO8601 time string. + + :type valid_until: str + :param valid_until: End date of the request. An ISO8601 time string. + + :type launch_group: str + :param launch_group: If supplied, all requests will be fulfilled + as a group. + + :type availability_zone_group: str + :param availability_zone_group: If supplied, all requests will be fulfilled + within a single availability zone. + + :type key_name: string + :param key_name: The name of the key pair with which to launch instances + + :type security_groups: list of strings + :param security_groups: The names of the security groups with which to associate instances + + :type user_data: string + :param user_data: The user data passed to the launched instances + + :type instance_type: string + :param instance_type: The type of instance to run (m1.small, m1.large, m1.xlarge) + + :type placement: string + :param placement: The availability zone in which to launch the instances + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the instances + + :type ramdisk_id: string + :param ramdisk_id: The ID of the RAM disk with which to launch the instances + + :type monitoring_enabled: bool + :param monitoring_enabled: Enable CloudWatch monitoring on the instance. + + :type subnet_id: string + :param subnet_id: The subnet ID within which to launch the instances for VPC. + + :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` + :param block_device_map: A BlockDeviceMapping data structure + describing the EBS volumes associated + with the Image. + + :rtype: Reservation + :return: The :class:`boto.ec2.instance.Reservation` associated with the request for machines + """ + params = {'LaunchSpecification.ImageId':image_id, + 'SpotPrice' : price} + if count: + params['InstanceCount'] = count + if valid_from: + params['ValidFrom'] = valid_from + if valid_until: + params['ValidUntil'] = valid_until + if launch_group: + params['LaunchGroup'] = launch_group + if availability_zone_group: + params['AvailabilityZoneGroup'] = availability_zone_group + if key_name: + params['LaunchSpecification.KeyName'] = key_name + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, SecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, + 'LaunchSpecification.SecurityGroup') + if user_data: + params['LaunchSpecification.UserData'] = base64.b64encode(user_data) + if addressing_type: + params['LaunchSpecification.AddressingType'] = addressing_type + if instance_type: + params['LaunchSpecification.InstanceType'] = instance_type + if placement: + params['LaunchSpecification.Placement.AvailabilityZone'] = placement + if kernel_id: + params['LaunchSpecification.KernelId'] = kernel_id + if ramdisk_id: + params['LaunchSpecification.RamdiskId'] = ramdisk_id + if monitoring_enabled: + params['LaunchSpecification.Monitoring.Enabled'] = 'true' + if subnet_id: + params['LaunchSpecification.SubnetId'] = subnet_id + if block_device_map: + block_device_map.build_list_params(params, 'LaunchSpecification.') + return self.get_list('RequestSpotInstances', params, + [('item', SpotInstanceRequest)], + verb='POST') + + + def cancel_spot_instance_requests(self, request_ids): + """ + Cancel the specified Spot Instance Requests. + + :type request_ids: list + :param request_ids: A list of strings of the Request IDs to terminate + + :rtype: list + :return: A list of the instances terminated + """ + params = {} + if request_ids: + self.build_list_params(params, request_ids, 'SpotInstanceRequestId') + return self.get_list('CancelSpotInstanceRequests', params, [('item', Instance)]) + + def get_spot_datafeed_subscription(self): + """ + Return the current spot instance data feed subscription + associated with this account, if any. + + :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription + :return: The datafeed subscription object or None + """ + return self.get_object('DescribeSpotDatafeedSubscription', + None, SpotDatafeedSubscription) + + def create_spot_datafeed_subscription(self, bucket, prefix): + """ + Create a spot instance datafeed subscription for this account. + + :type bucket: str or unicode + :param bucket: The name of the bucket where spot instance data + will be written. + + :type prefix: str or unicode + :param prefix: An optional prefix that will be pre-pended to all + data files written to the bucket. + + :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription + :return: The datafeed subscription object or None + """ + params = {'Bucket' : bucket} + if prefix: + params['Prefix'] = prefix + return self.get_object('CreateSpotDatafeedSubscription', + params, SpotDatafeedSubscription) + + def delete_spot_datafeed_subscription(self): + """ + Delete the current spot instance data feed subscription + associated with this account + + :rtype: bool + :return: True if successful + """ + return self.get_status('DeleteSpotDatafeedSubscription', None) + + # Zone methods + + def get_all_zones(self, zones=None): + """ + Get all Availability Zones associated with the current region. + + :type zones: list + :param zones: Optional list of zones. If this list is present, + only the Zones associated with these zone names + will be returned. + + :rtype: list of L{boto.ec2.zone.Zone} + :return: The requested Zone objects + """ + params = {} + if zones: + self.build_list_params(params, zones, 'ZoneName') + return self.get_list('DescribeAvailabilityZones', params, [('item', Zone)]) + + # Address methods + + def get_all_addresses(self, addresses=None): + """ + Get all EIP's associated with the current credentials. + + :type addresses: list + :param addresses: Optional list of addresses. If this list is present, + only the Addresses associated with these addresses + will be returned. + + :rtype: list of L{boto.ec2.address.Address} + :return: The requested Address objects + """ + params = {} + if addresses: + self.build_list_params(params, addresses, 'PublicIp') + return self.get_list('DescribeAddresses', params, [('item', Address)]) + + def allocate_address(self): + """ + Allocate a new Elastic IP address and associate it with your account. + + :rtype: L{boto.ec2.address.Address} + :return: The newly allocated Address + """ + return self.get_object('AllocateAddress', None, Address) + + def associate_address(self, instance_id, public_ip): + """ + Associate an Elastic IP address with a currently running instance. + + :type instance_id: string + :param instance_id: The ID of the instance + + :type public_ip: string + :param public_ip: The public IP address + + :rtype: bool + :return: True if successful + """ + params = {'InstanceId' : instance_id, 'PublicIp' : public_ip} + return self.get_status('AssociateAddress', params) + + def disassociate_address(self, public_ip): + """ + Disassociate an Elastic IP address from a currently running instance. + + :type public_ip: string + :param public_ip: The public IP address + + :rtype: bool + :return: True if successful + """ + params = {'PublicIp' : public_ip} + return self.get_status('DisassociateAddress', params) + + def release_address(self, public_ip): + """ + Free up an Elastic IP address + + :type public_ip: string + :param public_ip: The public IP address + + :rtype: bool + :return: True if successful + """ + params = {'PublicIp' : public_ip} + return self.get_status('ReleaseAddress', params) + + # Volume methods + + def get_all_volumes(self, volume_ids=None): + """ + Get all Volumes associated with the current credentials. + + :type volume_ids: list + :param volume_ids: Optional list of volume ids. If this list is present, + only the volumes associated with these volume ids + will be returned. + + :rtype: list of L{boto.ec2.volume.Volume} + :return: The requested Volume objects + """ + params = {} + if volume_ids: + self.build_list_params(params, volume_ids, 'VolumeId') + return self.get_list('DescribeVolumes', params, [('item', Volume)]) + + def create_volume(self, size, zone, snapshot=None): + """ + Create a new EBS Volume. + + :type size: int + :param size: The size of the new volume, in GiB + + :type zone: string or L{boto.ec2.zone.Zone} + :param zone: The availability zone in which the Volume will be created. + + :type snapshot: string or L{boto.ec2.snapshot.Snapshot} + :param snapshot: The snapshot from which the new Volume will be created. + """ + if isinstance(zone, Zone): + zone = zone.name + params = {'AvailabilityZone' : zone} + if size: + params['Size'] = size + if snapshot: + if isinstance(snapshot, Snapshot): + snapshot = snapshot.id + params['SnapshotId'] = snapshot + return self.get_object('CreateVolume', params, Volume) + + def delete_volume(self, volume_id): + """ + Delete an EBS volume. + + :type volume_id: str + :param volume_id: The ID of the volume to be delete. + + :rtype: bool + :return: True if successful + """ + params = {'VolumeId': volume_id} + return self.get_status('DeleteVolume', params) + + def attach_volume(self, volume_id, instance_id, device): + """ + Attach an EBS volume to an EC2 instance. + + :type volume_id: str + :param volume_id: The ID of the EBS volume to be attached. + + :type instance_id: str + :param instance_id: The ID of the EC2 instance to which it will + be attached. + + :type device: str + :param device: The device on the instance through which the + volume will be exposted (e.g. /dev/sdh) + + :rtype: bool + :return: True if successful + """ + params = {'InstanceId' : instance_id, + 'VolumeId' : volume_id, + 'Device' : device} + return self.get_status('AttachVolume', params) + + def detach_volume(self, volume_id, instance_id=None, device=None, force=False): + """ + Detach an EBS volume from an EC2 instance. + + :type volume_id: str + :param volume_id: The ID of the EBS volume to be attached. + + :type instance_id: str + :param instance_id: The ID of the EC2 instance from which it will + be detached. + + :type device: str + :param device: The device on the instance through which the + volume is exposted (e.g. /dev/sdh) + + :type force: bool + :param force: Forces detachment if the previous detachment attempt did + not occur cleanly. This option can lead to data loss or + a corrupted file system. Use this option only as a last + resort to detach a volume from a failed instance. The + instance will not have an opportunity to flush file system + caches nor file system meta data. If you use this option, + you must perform file system check and repair procedures. + + :rtype: bool + :return: True if successful + """ + params = {'VolumeId' : volume_id} + if instance_id: + params['InstanceId'] = instance_id + if device: + params['Device'] = device + if force: + params['Force'] = 'true' + return self.get_status('DetachVolume', params) + + # Snapshot methods + + def get_all_snapshots(self, snapshot_ids=None, owner=None, restorable_by=None): + """ + Get all EBS Snapshots associated with the current credentials. + + :type snapshot_ids: list + :param snapshot_ids: Optional list of snapshot ids. If this list is present, + only the Snapshots associated with these snapshot ids + will be returned. + + :type owner: str + :param owner: If present, only the snapshots owned by the specified user + will be returned. Valid values are: + self | amazon | AWS Account ID + + :type restorable_by: str + :param restorable_by: If present, only the snapshots that are restorable + by the specified account id will be returned. + + :rtype: list of L{boto.ec2.snapshot.Snapshot} + :return: The requested Snapshot objects + """ + params = {} + if snapshot_ids: + self.build_list_params(params, snapshot_ids, 'SnapshotId') + if owner: + params['Owner'] = owner + if restorable_by: + params['RestorableBy'] = restorable_by + return self.get_list('DescribeSnapshots', params, [('item', Snapshot)]) + + def create_snapshot(self, volume_id, description=None): + """ + Create a snapshot of an existing EBS Volume. + + :type volume_id: str + :param volume_id: The ID of the volume to be snapshot'ed + + :type description: str + :param description: A description of the snapshot. Limited to 255 characters. + + :rtype: bool + :return: True if successful + """ + params = {'VolumeId' : volume_id} + if description: + params['Description'] = description[0:255] + return self.get_object('CreateSnapshot', params, Snapshot) + + def delete_snapshot(self, snapshot_id): + params = {'SnapshotId': snapshot_id} + return self.get_status('DeleteSnapshot', params) + + def get_snapshot_attribute(self, snapshot_id, attribute='createVolumePermission'): + """ + Get information about an attribute of a snapshot. Only one attribute can be + specified per call. + + :type snapshot_id: str + :param snapshot_id: The ID of the snapshot. + + :type attribute: str + :param attribute: The requested attribute. Valid values are: + createVolumePermission + + :rtype: list of L{boto.ec2.snapshotattribute.SnapshotAttribute} + :return: The requested Snapshot attribute + """ + params = {'Attribute' : attribute} + if snapshot_id: + params['SnapshotId'] = snapshot_id + return self.get_object('DescribeSnapshotAttribute', params, SnapshotAttribute) + + def modify_snapshot_attribute(self, snapshot_id, attribute='createVolumePermission', + operation='add', user_ids=None, groups=None): + """ + Changes an attribute of an image. + + :type snapshot_id: string + :param snapshot_id: The snapshot id you wish to change + + :type attribute: string + :param attribute: The attribute you wish to change. Valid values are: + createVolumePermission + + :type operation: string + :param operation: Either add or remove (this is required for changing + snapshot ermissions) + + :type user_ids: list + :param user_ids: The Amazon IDs of users to add/remove attributes + + :type groups: list + :param groups: The groups to add/remove attributes. The only valid + value at this time is 'all'. + + """ + params = {'SnapshotId' : snapshot_id, + 'Attribute' : attribute, + 'OperationType' : operation} + if user_ids: + self.build_list_params(params, user_ids, 'UserId') + if groups: + self.build_list_params(params, groups, 'UserGroup') + return self.get_status('ModifySnapshotAttribute', params) + + def reset_snapshot_attribute(self, snapshot_id, attribute='createVolumePermission'): + """ + Resets an attribute of a snapshot to its default value. + + :type snapshot_id: string + :param snapshot_id: ID of the snapshot + + :type attribute: string + :param attribute: The attribute to reset + + :rtype: bool + :return: Whether the operation succeeded or not + """ + params = {'SnapshotId' : snapshot_id, + 'Attribute' : attribute} + return self.get_status('ResetSnapshotAttribute', params) + + # Keypair methods + + def get_all_key_pairs(self, keynames=None): + """ + Get all key pairs associated with your account. + + :type keynames: list + :param keynames: A list of the names of keypairs to retrieve. + If not provided, all key pairs will be returned. + + :rtype: list + :return: A list of :class:`boto.ec2.keypair.KeyPair` + """ + params = {} + if keynames: + self.build_list_params(params, keynames, 'KeyName') + return self.get_list('DescribeKeyPairs', params, [('item', KeyPair)]) + + def get_key_pair(self, keyname): + """ + Convenience method to retrieve a specific keypair (KeyPair). + + :type image_id: string + :param image_id: the ID of the Image to retrieve + + :rtype: :class:`boto.ec2.keypair.KeyPair` + :return: The KeyPair specified or None if it is not found + """ + try: + return self.get_all_key_pairs(keynames=[keyname])[0] + except IndexError: # None of those key pairs available + return None + + def create_key_pair(self, key_name): + """ + Create a new key pair for your account. + This will create the key pair within the region you + are currently connected to. + + :type key_name: string + :param key_name: The name of the new keypair + + :rtype: :class:`boto.ec2.keypair.KeyPair` + :return: The newly created :class:`boto.ec2.keypair.KeyPair`. + The material attribute of the new KeyPair object + will contain the the unencrypted PEM encoded RSA private key. + """ + params = {'KeyName':key_name} + return self.get_object('CreateKeyPair', params, KeyPair) + + def delete_key_pair(self, key_name): + """ + Delete a key pair from your account. + + :type key_name: string + :param key_name: The name of the keypair to delete + """ + params = {'KeyName':key_name} + return self.get_status('DeleteKeyPair', params) + + # SecurityGroup methods + + def get_all_security_groups(self, groupnames=None): + """ + Get all security groups associated with your account in a region. + + :type groupnames: list + :param groupnames: A list of the names of security groups to retrieve. + If not provided, all security groups will be returned. + + :rtype: list + :return: A list of :class:`boto.ec2.securitygroup.SecurityGroup` + """ + params = {} + if groupnames: + self.build_list_params(params, groupnames, 'GroupName') + return self.get_list('DescribeSecurityGroups', params, [('item', SecurityGroup)]) + + def create_security_group(self, name, description): + """ + Create a new security group for your account. + This will create the security group within the region you + are currently connected to. + + :type name: string + :param name: The name of the new security group + + :type description: string + :param description: The description of the new security group + + :rtype: :class:`boto.ec2.securitygroup.SecurityGroup` + :return: The newly created :class:`boto.ec2.keypair.KeyPair`. + """ + params = {'GroupName':name, 'GroupDescription':description} + group = self.get_object('CreateSecurityGroup', params, SecurityGroup) + group.name = name + group.description = description + return group + + def delete_security_group(self, name): + """ + Delete a security group from your account. + + :type key_name: string + :param key_name: The name of the keypair to delete + """ + params = {'GroupName':name} + return self.get_status('DeleteSecurityGroup', params) + + def authorize_security_group(self, group_name, src_security_group_name=None, + src_security_group_owner_id=None, + ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None): + """ + Add a new rule to an existing security group. + You need to pass in either src_security_group_name and + src_security_group_owner_id OR ip_protocol, from_port, to_port, + and cidr_ip. In other words, either you are authorizing another + group or you are authorizing some ip-based rule. + + :type group_name: string + :param group_name: The name of the security group you are adding + the rule to. + + :type src_security_group_name: string + :param src_security_group_name: The name of the security group you are + granting access to. + + :type src_security_group_owner_id: string + :param src_security_group_owner_id: The ID of the owner of the security group you are + granting access to. + + :type ip_protocol: string + :param ip_protocol: Either tcp | udp | icmp + + :type from_port: int + :param from_port: The beginning port number you are enabling + + :type to_port: int + :param to_port: The ending port number you are enabling + + :type to_port: string + :param to_port: The CIDR block you are providing access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + params = {'GroupName':group_name} + if src_security_group_name: + params['SourceSecurityGroupName'] = src_security_group_name + if src_security_group_owner_id: + params['SourceSecurityGroupOwnerId'] = src_security_group_owner_id + if ip_protocol: + params['IpProtocol'] = ip_protocol + if from_port: + params['FromPort'] = from_port + if to_port: + params['ToPort'] = to_port + if cidr_ip: + params['CidrIp'] = urllib.quote(cidr_ip) + return self.get_status('AuthorizeSecurityGroupIngress', params) + + def revoke_security_group(self, group_name, src_security_group_name=None, + src_security_group_owner_id=None, + ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None): + """ + Remove an existing rule from an existing security group. + You need to pass in either src_security_group_name and + src_security_group_owner_id OR ip_protocol, from_port, to_port, + and cidr_ip. In other words, either you are revoking another + group or you are revoking some ip-based rule. + + :type group_name: string + :param group_name: The name of the security group you are removing + the rule from. + + :type src_security_group_name: string + :param src_security_group_name: The name of the security group you are + revoking access to. + + :type src_security_group_owner_id: string + :param src_security_group_owner_id: The ID of the owner of the security group you are + revoking access to. + + :type ip_protocol: string + :param ip_protocol: Either tcp | udp | icmp + + :type from_port: int + :param from_port: The beginning port number you are disabling + + :type to_port: int + :param to_port: The ending port number you are disabling + + :type to_port: string + :param to_port: The CIDR block you are revoking access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + params = {'GroupName':group_name} + if src_security_group_name: + params['SourceSecurityGroupName'] = src_security_group_name + if src_security_group_owner_id: + params['SourceSecurityGroupOwnerId'] = src_security_group_owner_id + if ip_protocol: + params['IpProtocol'] = ip_protocol + if from_port: + params['FromPort'] = from_port + if to_port: + params['ToPort'] = to_port + if cidr_ip: + params['CidrIp'] = cidr_ip + return self.get_status('RevokeSecurityGroupIngress', params) + + # + # Regions + # + + def get_all_regions(self): + """ + Get all available regions for the EC2 service. + + :rtype: list + :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` + """ + return self.get_list('DescribeRegions', None, [('item', RegionInfo)]) + + # + # Reservation methods + # + + def get_all_reserved_instances_offerings(self, reserved_instances_id=None, + instance_type=None, + availability_zone=None, + product_description=None): + """ + Describes Reserved Instance offerings that are available for purchase. + + :type reserved_instances_id: str + :param reserved_instances_id: Displays Reserved Instances with the specified offering IDs. + + :type instance_type: str + :param instance_type: Displays Reserved Instances of the specified instance type. + + :type availability_zone: str + :param availability_zone: Displays Reserved Instances within the specified Availability Zone. + + :type product_description: str + :param product_description: Displays Reserved Instances with the specified product description. + + :rtype: list + :return: A list of :class:`boto.ec2.reservedinstance.ReservedInstancesOffering` + """ + params = {} + if reserved_instances_id: + params['ReservedInstancesId'] = reserved_instances_id + if instance_type: + params['InstanceType'] = instance_type + if availability_zone: + params['AvailabilityZone'] = availability_zone + if product_description: + params['ProductDescription'] = product_description + + return self.get_list('DescribeReservedInstancesOfferings', + params, [('item', ReservedInstancesOffering)]) + + def get_all_reserved_instances(self, reserved_instances_id=None): + """ + Describes Reserved Instance offerings that are available for purchase. + + :type reserved_instance_ids: list + :param reserved_instance_ids: A list of the reserved instance ids that will be returned. + If not provided, all reserved instances will be returned. + + :rtype: list + :return: A list of :class:`boto.ec2.reservedinstance.ReservedInstance` + """ + params = {} + if reserved_instances_id: + self.build_list_params(params, reserved_instances_id, 'ReservedInstancesId') + return self.get_list('DescribeReservedInstances', + params, [('item', ReservedInstance)]) + + def purchase_reserved_instance_offering(self, reserved_instances_offering_id, + instance_count=1): + """ + Purchase a Reserved Instance for use with your account. + ** CAUTION ** + This request can result in large amounts of money being charged to your + AWS account. Use with caution! + + :type reserved_instances_offering_id: string + :param reserved_instances_offering_id: The offering ID of the Reserved + Instance to purchase + + :type instance_count: int + :param instance_count: The number of Reserved Instances to purchase. + Default value is 1. + + :rtype: :class:`boto.ec2.reservedinstance.ReservedInstance` + :return: The newly created Reserved Instance + """ + params = {'ReservedInstancesOfferingId' : reserved_instances_offering_id, + 'InstanceCount' : instance_count} + return self.get_object('PurchaseReservedInstancesOffering', params, ReservedInstance) + + # + # Monitoring + # + + def monitor_instance(self, instance_id): + """ + Enable CloudWatch monitoring for the supplied instance. + + :type instance_id: string + :param instance_id: The instance id + + :rtype: list + :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` + """ + params = {'InstanceId' : instance_id} + return self.get_list('MonitorInstances', params, [('item', InstanceInfo)]) + + def unmonitor_instance(self, instance_id): + """ + Disable CloudWatch monitoring for the supplied instance. + + :type instance_id: string + :param instance_id: The instance id + + :rtype: list + :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` + """ + params = {'InstanceId' : instance_id} + return self.get_list('UnmonitorInstances', params, [('item', InstanceInfo)]) + diff --git a/api/boto/ec2/ec2object.py b/api/boto/ec2/ec2object.py new file mode 100644 index 0000000..9ffab5d --- /dev/null +++ b/api/boto/ec2/ec2object.py @@ -0,0 +1,41 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Object +""" + +class EC2Object(object): + + def __init__(self, connection=None): + self.connection = connection + if self.connection and hasattr(self.connection, 'region'): + self.region = connection.region + else: + self.region = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + setattr(self, name, value) + + diff --git a/api/boto/ec2/elb/__init__.py b/api/boto/ec2/elb/__init__.py new file mode 100644 index 0000000..55e846f --- /dev/null +++ b/api/boto/ec2/elb/__init__.py @@ -0,0 +1,238 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +load balancing service from AWS. +""" +from boto.connection import AWSQueryConnection +from boto.ec2.instanceinfo import InstanceInfo +from boto.ec2.elb.loadbalancer import LoadBalancer +from boto.ec2.elb.instancestate import InstanceState +from boto.ec2.elb.healthcheck import HealthCheck +import boto + +class ELBConnection(AWSQueryConnection): + + APIVersion = boto.config.get('Boto', 'elb_version', '2009-05-15') + Endpoint = boto.config.get('Boto', 'elb_endpoint', 'elasticloadbalancing.amazonaws.com') + SignatureVersion = '1' + #ResponseError = EC2ResponseError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=Endpoint, debug=0, + https_connection_factory=None, path='/'): + """ + Init method to create a new connection to EC2 Load Balancing Service. + + B{Note:} The host argument is overridden by the host specified in the boto configuration file. + """ + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory, path) + + def build_list_params(self, params, items, label): + if isinstance(items, str): + items = [items] + for i in range(1, len(items)+1): + params[label % i] = items[i-1] + + def get_all_load_balancers(self, load_balancer_name=None): + """ + Retrieve all load balancers associated with your account. + + :type load_balancer_names: str + :param load_balancer_names: An optional filter string to get only one ELB + + :rtype: list + :return: A list of :class:`boto.ec2.elb.loadbalancer.LoadBalancer` + """ + params = {} + if load_balancer_name: + #self.build_list_params(params, load_balancer_names, 'LoadBalancerName.%d') + params['LoadBalancerName'] = load_balancer_name + return self.get_list('DescribeLoadBalancers', params, [('member', LoadBalancer)]) + + + def create_load_balancer(self, name, zones, listeners): + """ + Create a new load balancer for your account. + + :type name: string + :param name: The mnemonic name associated with the new load balancer + + :type zones: List of strings + :param zones: The names of the availability zone(s) to add. + + :type listeners: List of tuples + :param listeners: Each tuple contains three values. + (LoadBalancerPortNumber, InstancePortNumber, Protocol) + where LoadBalancerPortNumber and InstancePortNumber are + integer values between 1 and 65535 and Protocol is a + string containing either 'TCP' or 'HTTP'. + + :rtype: :class:`boto.ec2.elb.loadbalancer.LoadBalancer` + :return: The newly created :class:`boto.ec2.elb.loadbalancer.LoadBalancer` + """ + params = {'LoadBalancerName' : name} + for i in range(0, len(listeners)): + params['Listeners.member.%d.LoadBalancerPort' % (i+1)] = listeners[i][0] + params['Listeners.member.%d.InstancePort' % (i+1)] = listeners[i][1] + params['Listeners.member.%d.Protocol' % (i+1)] = listeners[i][2] + self.build_list_params(params, zones, 'AvailabilityZones.member.%d') + load_balancer = self.get_object('CreateLoadBalancer', params, LoadBalancer) + load_balancer.name = name + load_balancer.listeners = listeners + load_balancer.availability_zones = zones + return load_balancer + + def delete_load_balancer(self, name): + """ + Delete a Load Balancer from your account. + + :type name: string + :param name: The name of the Load Balancer to delete + """ + params = {'LoadBalancerName': name} + return self.get_status('DeleteLoadBalancer', params) + + def enable_availability_zones(self, load_balancer_name, zones_to_add): + """ + Add availability zones to an existing Load Balancer + All zones must be in the same region as the Load Balancer + Adding zones that are already registered with the Load Balancer + has no effect. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type zones: List of strings + :param zones: The name of the zone(s) to add. + + :rtype: List of strings + :return: An updated list of zones for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, zones_to_add, 'AvailabilityZones.member.%d') + return self.get_list('EnableAvailabilityZonesForLoadBalancer', params, None) + + def disable_availability_zones(self, load_balancer_name, zones_to_remove): + """ + Remove availability zones from an existing Load Balancer. + All zones must be in the same region as the Load Balancer. + Removing zones that are not registered with the Load Balancer + has no effect. + You cannot remove all zones from an Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type zones: List of strings + :param zones: The name of the zone(s) to remove. + + :rtype: List of strings + :return: An updated list of zones for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, zones_to_remove, 'AvailabilityZones.member.%d') + return self.get_list('DisableAvailabilityZonesForLoadBalancer', params, None) + + def register_instances(self, load_balancer_name, instances): + """ + Add new Instances to an existing Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type instances: List of strings + :param instances: The instance ID's of the EC2 instances to add. + + :rtype: List of strings + :return: An updated list of instances for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, instances, 'Instances.member.%d.InstanceId') + return self.get_list('RegisterInstancesWithLoadBalancer', params, [('member', InstanceInfo)]) + + def deregister_instances(self, load_balancer_name, instances): + """ + Remove Instances from an existing Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type instances: List of strings + :param instances: The instance ID's of the EC2 instances to remove. + + :rtype: List of strings + :return: An updated list of instances for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, instances, 'Instances.member.%d.InstanceId') + return self.get_list('DeregisterInstancesFromLoadBalancer', params, [('member', InstanceInfo)]) + + def describe_instance_health(self, load_balancer_name, instances=None): + """ + Get current state of all Instances registered to an Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type instances: List of strings + :param instances: The instance ID's of the EC2 instances + to return status for. If not provided, + the state of all instances will be returned. + + :rtype: List of :class:`boto.ec2.elb.instancestate.InstanceState` + :return: list of state info for instances in this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + if instances: + self.build_list_params(params, instances, 'instances.member.%d') + return self.get_list('DescribeInstanceHealth', params, [('member', InstanceState)]) + + def configure_health_check(self, name, health_check): + """ + Define a health check for the EndPoints. + + :type name: string + :param name: The mnemonic name associated with the new access point + + :type health_check: :class:`boto.ec2.elb.healthcheck.HealthCheck` + :param health_check: A HealthCheck object populated with the desired + values. + + :rtype: :class:`boto.ec2.elb.healthcheck.HealthCheck` + :return: The updated :class:`boto.ec2.elb.healthcheck.HealthCheck` + """ + params = {'LoadBalancerName' : name, + 'HealthCheck.Timeout' : health_check.timeout, + 'HealthCheck.Target' : health_check.target, + 'HealthCheck.Interval' : health_check.interval, + 'HealthCheck.UnhealthyThreshold' : health_check.unhealthy_threshold, + 'HealthCheck.HealthyThreshold' : health_check.healthy_threshold} + return self.get_object('ConfigureHealthCheck', params, HealthCheck) diff --git a/api/boto/ec2/elb/healthcheck.py b/api/boto/ec2/elb/healthcheck.py new file mode 100644 index 0000000..5a3edbc --- /dev/null +++ b/api/boto/ec2/elb/healthcheck.py @@ -0,0 +1,68 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class HealthCheck(object): + """ + Represents an EC2 Access Point Health Check + """ + + def __init__(self, access_point=None, interval=30, target=None, + healthy_threshold=3, timeout=5, unhealthy_threshold=5): + self.access_point = access_point + self.interval = interval + self.target = target + self.healthy_threshold = healthy_threshold + self.timeout = timeout + self.unhealthy_threshold = unhealthy_threshold + + def __repr__(self): + return 'HealthCheck:%s' % self.target + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Interval': + self.interval = int(value) + elif name == 'Target': + self.target = value + elif name == 'HealthyThreshold': + self.healthy_threshold = int(value) + elif name == 'Timeout': + self.timeout = int(value) + elif name == 'UnhealthyThreshold': + self.unhealthy_threshold = int(value) + else: + setattr(self, name, value) + + def update(self): + if not self.access_point: + return + + new_hc = self.connection.configure_health_check(self.access_point, + self) + self.interval = new_hc.interval + self.target = new_hc.target + self.healthy_threshold = new_hc.healthy_threshold + self.unhealthy_threshold = new_hc.unhealthy_threshold + self.timeout = new_hc.timeout + + diff --git a/api/boto/ec2/elb/instancestate.py b/api/boto/ec2/elb/instancestate.py new file mode 100644 index 0000000..4a9b0d4 --- /dev/null +++ b/api/boto/ec2/elb/instancestate.py @@ -0,0 +1,54 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class InstanceState(object): + """ + Represents the state of an EC2 Load Balancer Instance + """ + + def __init__(self, load_balancer=None, description=None, + state=None, instance_id=None, reason_code=None): + self.load_balancer = load_balancer + self.description = description + self.state = state + self.instance_id = instance_id + self.reason_code = reason_code + + def __repr__(self): + return 'InstanceState:(%s,%s)' % (self.instance_id, self.state) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Description': + self.description = value + elif name == 'State': + self.state = value + elif name == 'InstanceId': + self.instance_id = value + elif name == 'ReasonCode': + self.reason_code = value + else: + setattr(self, name, value) + + + diff --git a/api/boto/ec2/elb/listelement.py b/api/boto/ec2/elb/listelement.py new file mode 100644 index 0000000..5be4599 --- /dev/null +++ b/api/boto/ec2/elb/listelement.py @@ -0,0 +1,31 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class ListElement(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'member': + self.append(value) + + diff --git a/api/boto/ec2/elb/listener.py b/api/boto/ec2/elb/listener.py new file mode 100644 index 0000000..ab482c2 --- /dev/null +++ b/api/boto/ec2/elb/listener.py @@ -0,0 +1,64 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Listener(object): + """ + Represents an EC2 Load Balancer Listener tuple + """ + + def __init__(self, load_balancer=None, load_balancer_port=0, + instance_port=0, protocol=''): + self.load_balancer = load_balancer + self.load_balancer_port = load_balancer_port + self.instance_port = instance_port + self.protocol = protocol + + def __repr__(self): + return "(%d, %d, '%s')" % (self.load_balancer_port, self.instance_port, self.protocol) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'LoadBalancerPort': + self.load_balancer_port = int(value) + elif name == 'InstancePort': + self.instance_port = int(value) + elif name == 'Protocol': + self.protocol = value + else: + setattr(self, name, value) + + def get_tuple(self): + return self.load_balancer_port, self.instance_port, self.protocol + + def __getitem__(self, key): + if key == 0: + return self.load_balancer_port + if key == 1: + return self.instance_port + if key == 2: + return self.protocol + raise KeyError + + + + diff --git a/api/boto/ec2/elb/loadbalancer.py b/api/boto/ec2/elb/loadbalancer.py new file mode 100644 index 0000000..2902107 --- /dev/null +++ b/api/boto/ec2/elb/loadbalancer.py @@ -0,0 +1,144 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.ec2.elb.healthcheck import HealthCheck +from boto.ec2.elb.instancestate import InstanceState +from boto.ec2.elb.listener import Listener +from boto.ec2.elb.listelement import ListElement +from boto.ec2.zone import Zone +from boto.ec2.instanceinfo import InstanceInfo +from boto.resultset import ResultSet + +class LoadBalancer(object): + """ + Represents an EC2 Load Balancer + """ + + def __init__(self, connection=None, name=None, endpoints=None): + self.connection = connection + self.name = name + self.listeners = None + self.health_check = None + self.dns_name = None + self.created_time = None + self.instances = None + self.availability_zones = ListElement() + + def __repr__(self): + return 'LoadBalancer:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'HealthCheck': + self.health_check = HealthCheck(self) + return self.health_check + elif name == 'Listeners': + self.listeners = ResultSet([('member', Listener)]) + return self.listeners + elif name == 'AvailabilityZones': + return self.availability_zones + elif name == 'Instances': + self.instances = ResultSet([('member', InstanceInfo)]) + return self.instances + else: + return None + + def endElement(self, name, value, connection): + if name == 'LoadBalancerName': + self.name = value + elif name == 'DNSName': + self.dns_name = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'InstanceId': + self.instances.append(value) + else: + setattr(self, name, value) + + def enable_zones(self, zones): + """ + Enable availability zones to this Access Point. + All zones must be in the same region as the Access Point. + + :type zones: string or List of strings + :param zones: The name of the zone(s) to add. + + """ + if isinstance(zones, str) or isinstance(zones, unicode): + zones = [zones] + new_zones = self.connection.enable_availability_zones(self.name, zones) + self.availability_zones = new_zones + + def disable_zones(self, zones): + """ + Disable availability zones from this Access Point. + + :type zones: string or List of strings + :param zones: The name of the zone(s) to add. + + """ + if isinstance(zones, str) or isinstance(zones, unicode): + zones = [zones] + new_zones = self.connection.disable_availability_zones(self.name, zones) + self.availability_zones = new_zones + + def register_instances(self, instances): + """ + Add instances to this Load Balancer + All instances must be in the same region as the Load Balancer. + Adding endpoints that are already registered with the Load Balancer + has no effect. + + :type zones: string or List of instance id's + :param zones: The name of the endpoint(s) to add. + + """ + if isinstance(instances, str) or isinstance(instances, unicode): + instances = [instances] + new_instances = self.connection.register_instances(self.name, instances) + self.instances = new_instances + + def deregister_instances(self, instances): + """ + Remove instances from this Load Balancer. + Removing instances that are not registered with the Load Balancer + has no effect. + + :type zones: string or List of instance id's + :param zones: The name of the endpoint(s) to add. + + """ + if isinstance(instances, str) or isinstance(instances, unicode): + instances = [instances] + new_instances = self.connection.deregister_instances(self.name, instances) + self.instances = new_instances + + def delete(self): + """ + Delete this load balancer + """ + return self.connection.delete_load_balancer(self.name) + + def configure_health_check(self, health_check): + self.connection.configure_health_check(self.name, health_check) + + def get_instance_health(self, instances=None): + self.connection.describe_instance_health(self.name, instances) + diff --git a/api/boto/ec2/image.py b/api/boto/ec2/image.py new file mode 100644 index 0000000..8ef2513 --- /dev/null +++ b/api/boto/ec2/image.py @@ -0,0 +1,243 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.ec2.ec2object import EC2Object +from boto.ec2.blockdevicemapping import BlockDeviceMapping + +class ProductCodes(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'productCode': + self.append(value) + +class Image(EC2Object): + """ + Represents an EC2 Image + """ + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.location = None + self.state = None + self.ownerId = None + self.owner_alias = None + self.is_public = False + self.architecture = None + self.platform = None + self.type = None + self.kernel_id = None + self.ramdisk_id = None + self.name = None + self.description = None + self.product_codes = ProductCodes() + self.block_device_mapping = None + self.root_device_type = None + self.root_device_name = None + + def __repr__(self): + return 'Image:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'blockDeviceMapping': + self.block_device_mapping = BlockDeviceMapping() + return self.block_device_mapping + elif name == 'productCodes': + return self.product_codes + else: + return None + + def endElement(self, name, value, connection): + if name == 'imageId': + self.id = value + elif name == 'imageLocation': + self.location = value + elif name == 'imageState': + self.state = value + elif name == 'imageOwnerId': + self.ownerId = value + elif name == 'isPublic': + if value == 'false': + self.is_public = False + elif value == 'true': + self.is_public = True + else: + raise Exception( + 'Unexpected value of isPublic %s for image %s'%( + value, + self.id + ) + ) + elif name == 'architecture': + self.architecture = value + elif name == 'imageType': + self.type = value + elif name == 'kernelId': + self.kernel_id = value + elif name == 'ramdiskId': + self.ramdisk_id = value + elif name == 'imageOwnerAlias': + self.owner_alias = value + elif name == 'platform': + self.platform = value + elif name == 'name': + self.name = value + elif name == 'description': + self.description = value + elif name == 'rootDeviceType': + self.root_device_type = value + elif name == 'rootDeviceName': + self.root_device_name = value + else: + setattr(self, name, value) + + def run(self, min_count=1, max_count=1, key_name=None, + security_groups=None, user_data=None, + addressing_type=None, instance_type='m1.small', placement=None, + kernel_id=None, ramdisk_id=None, + monitoring_enabled=False, subnet_id=None): + """ + Runs this instance. + + :type min_count: int + :param min_count: The minimum number of instances to start + + :type max_count: int + :param max_count: The maximum number of instances to start + + :type key_name: string + :param key_name: The keypair to run this instance with. + + :type security_groups: + :param security_groups: + + :type user_data: + :param user_data: + + :type addressing_type: + :param daddressing_type: + + :type instance_type: string + :param instance_type: The type of instance to run (m1.small, m1.large, m1.xlarge) + + :type placement: + :param placement: + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the instances + + :type ramdisk_id: string + :param ramdisk_id: The ID of the RAM disk with which to launch the instances + + :type monitoring_enabled: bool + :param monitoring_enabled: Enable CloudWatch monitoring on the instance. + + :type subnet_id: string + :param subnet_id: The subnet ID within which to launch the instances for VPC. + + :rtype: Reservation + :return: The :class:`boto.ec2.instance.Reservation` associated with the request for machines + """ + return self.connection.run_instances(self.id, min_count, max_count, + key_name, security_groups, + user_data, addressing_type, + instance_type, placement, + kernel_id, ramdisk_id, + monitoring_enabled, subnet_id) + + def deregister(self): + return self.connection.deregister_image(self.id) + + def get_launch_permissions(self): + img_attrs = self.connection.get_image_attribute(self.id, + 'launchPermission') + return img_attrs.attrs + + def set_launch_permissions(self, user_ids=None, group_names=None): + return self.connection.modify_image_attribute(self.id, + 'launchPermission', + 'add', + user_ids, + group_names) + + def remove_launch_permissions(self, user_ids=None, group_names=None): + return self.connection.modify_image_attribute(self.id, + 'launchPermission', + 'remove', + user_ids, + group_names) + + def reset_launch_attributes(self): + return self.connection.reset_image_attribute(self.id, + 'launchPermission') + + def get_kernel(self): + img_attrs =self.connection.get_image_attribute(self.id, 'kernel') + return img_attrs.kernel + + def get_ramdisk(self): + img_attrs = self.connection.get_image_attribute(self.id, 'ramdisk') + return img_attrs.ramdisk + +class ImageAttribute: + + def __init__(self, parent=None): + self.name = None + self.kernel = None + self.ramdisk = None + self.attrs = {} + + def startElement(self, name, attrs, connection): + if name == 'blockDeviceMapping': + self.attrs['block_device_mapping'] = BlockDeviceMapping() + return self.attrs['block_device_mapping'] + else: + return None + + def endElement(self, name, value, connection): + if name == 'launchPermission': + self.name = 'launch_permission' + elif name == 'group': + if self.attrs.has_key('groups'): + self.attrs['groups'].append(value) + else: + self.attrs['groups'] = [value] + elif name == 'userId': + if self.attrs.has_key('user_ids'): + self.attrs['user_ids'].append(value) + else: + self.attrs['user_ids'] = [value] + elif name == 'productCode': + if self.attrs.has_key('product_codes'): + self.attrs['product_codes'].append(value) + else: + self.attrs['product_codes'] = [value] + elif name == 'imageId': + self.image_id = value + elif name == 'kernel': + self.kernel = value + elif name == 'ramdisk': + self.ramdisk = value + else: + setattr(self, name, value) diff --git a/api/boto/ec2/instance.py b/api/boto/ec2/instance.py new file mode 100644 index 0000000..5932c4e --- /dev/null +++ b/api/boto/ec2/instance.py @@ -0,0 +1,280 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Instance +""" +import boto +from boto.ec2.ec2object import EC2Object +from boto.resultset import ResultSet +from boto.ec2.address import Address +from boto.ec2.blockdevicemapping import BlockDeviceMapping +from boto.ec2.image import ProductCodes +import base64 + +class Reservation(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.owner_id = None + self.groups = [] + self.instances = [] + + def __repr__(self): + return 'Reservation:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'instancesSet': + self.instances = ResultSet([('item', Instance)]) + return self.instances + elif name == 'groupSet': + self.groups = ResultSet([('item', Group)]) + return self.groups + else: + return None + + def endElement(self, name, value, connection): + if name == 'reservationId': + self.id = value + elif name == 'ownerId': + self.owner_id = value + else: + setattr(self, name, value) + + def stop_all(self): + for instance in self.instances: + instance.stop() + +class Instance(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.dns_name = None + self.public_dns_name = None + self.private_dns_name = None + self.state = None + self.state_code = None + self.key_name = None + self.shutdown_state = None + self.previous_state = None + self.instance_type = None + self.instance_class = None + self.launch_time = None + self.image_id = None + self.placement = None + self.kernel = None + self.ramdisk = None + self.product_codes = ProductCodes() + self.ami_launch_index = None + self.monitored = False + self.instance_class = None + self.spot_instance_request_id = None + self.subnet_id = None + self.vpc_id = None + self.private_ip_address = None + self.ip_address = None + self.requester_id = None + self._in_monitoring_element = False + self.persistent = False + self.root_device_name = None + self.block_device_mapping = None + + def __repr__(self): + return 'Instance:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'monitoring': + self._in_monitoring_element = True + elif name == 'blockDeviceMapping': + self.block_device_mapping = BlockDeviceMapping() + return self.block_device_mapping + elif name == 'productCodes': + return self.product_codes + return None + + def endElement(self, name, value, connection): + if name == 'instanceId': + self.id = value + elif name == 'imageId': + self.image_id = value + elif name == 'dnsName' or name == 'publicDnsName': + self.dns_name = value # backwards compatibility + self.public_dns_name = value + elif name == 'privateDnsName': + self.private_dns_name = value + elif name == 'keyName': + self.key_name = value + elif name == 'amiLaunchIndex': + self.ami_launch_index = value + elif name == 'shutdownState': + self.shutdown_state = value + elif name == 'previousState': + self.previous_state = value + elif name == 'name': + self.state = value + elif name == 'code': + try: + self.state_code = int(value) + except ValueError: + boto.log.warning('Error converting code (%s) to int' % value) + self.state_code = value + elif name == 'instanceType': + self.instance_type = value + elif name == 'instanceClass': + self.instance_class = value + elif name == 'rootDeviceName': + self.root_device_name = value + elif name == 'launchTime': + self.launch_time = value + elif name == 'availabilityZone': + self.placement = value + elif name == 'placement': + pass + elif name == 'kernelId': + self.kernel = value + elif name == 'ramdiskId': + self.ramdisk = value + elif name == 'state': + if self._in_monitoring_element: + if value == 'enabled': + self.monitored = True + self._in_monitoring_element = False + elif name == 'instanceClass': + self.instance_class = value + elif name == 'spotInstanceRequestId': + self.spot_instance_request_id = value + elif name == 'subnetId': + self.subnet_id = value + elif name == 'vpcId': + self.vpc_id = value + elif name == 'privateIpAddress': + self.private_ip_address = value + elif name == 'ipAddress': + self.ip_address = value + elif name == 'requesterId': + self.requester_id = value + elif name == 'persistent': + if value == 'true': + self.persistent = True + else: + self.persistent = False + else: + setattr(self, name, value) + + def _update(self, updated): + self.updated = updated + if hasattr(updated, 'dns_name'): + self.dns_name = updated.dns_name + self.public_dns_name = updated.dns_name + if hasattr(updated, 'private_dns_name'): + self.private_dns_name = updated.private_dns_name + if hasattr(updated, 'ami_launch_index'): + self.ami_launch_index = updated.ami_launch_index + self.shutdown_state = updated.shutdown_state + self.previous_state = updated.previous_state + if hasattr(updated, 'state'): + self.state = updated.state + else: + self.state = None + if hasattr(updated, 'state_code'): + self.state_code = updated.state_code + else: + self.state_code = None + + def update(self): + rs = self.connection.get_all_instances([self.id]) + if len(rs) > 0: + self._update(rs[0].instances[0]) + return self.state + + def stop(self): + rs = self.connection.terminate_instances([self.id]) + self._update(rs[0]) + + def reboot(self): + return self.connection.reboot_instances([self.id]) + + def get_console_output(self): + return self.connection.get_console_output(self.id) + + def confirm_product(self, product_code): + return self.connection.confirm_product_instance(self.id, product_code) + + def use_ip(self, ip_address): + if isinstance(ip_address, Address): + ip_address = ip_address.public_ip + return self.connection.associate_address(self.id, ip_address) + + def monitor(self): + return self.connection.monitor_instance(self.id) + + def unmonitor(self): + return self.connection.unmonitor_instance(self.id) + +class Group: + + def __init__(self, parent=None): + self.id = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'groupId': + self.id = value + else: + setattr(self, name, value) + +class ConsoleOutput: + + def __init__(self, parent=None): + self.parent = parent + self.instance_id = None + self.timestamp = None + self.comment = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'instanceId': + self.instance_id = value + elif name == 'output': + self.output = base64.b64decode(value) + else: + setattr(self, name, value) + +class InstanceAttribute(dict): + + def __init__(self, parent=None): + dict.__init__(self) + self._current_value = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'value': + self._current_value = value + else: + self[name] = self._current_value diff --git a/api/boto/ec2/instanceinfo.py b/api/boto/ec2/instanceinfo.py new file mode 100644 index 0000000..6efbaed --- /dev/null +++ b/api/boto/ec2/instanceinfo.py @@ -0,0 +1,47 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class InstanceInfo(object): + """ + Represents an EC2 Instance status response from CloudWatch + """ + + def __init__(self, connection=None, id=None, state=None): + self.connection = connection + self.id = id + self.state = state + + def __repr__(self): + return 'InstanceInfo:%s' % self.id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'instanceId' or name == 'InstanceId': + self.id = value + elif name == 'state': + self.state = value + else: + setattr(self, name, value) + + + diff --git a/api/boto/ec2/keypair.py b/api/boto/ec2/keypair.py new file mode 100644 index 0000000..d08e5ce --- /dev/null +++ b/api/boto/ec2/keypair.py @@ -0,0 +1,111 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Keypair +""" + +import os +from boto.ec2.ec2object import EC2Object +from boto.exception import BotoClientError + +class KeyPair(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.name = None + self.fingerprint = None + self.material = None + + def __repr__(self): + return 'KeyPair:%s' % self.name + + def endElement(self, name, value, connection): + if name == 'keyName': + self.name = value + elif name == 'keyFingerprint': + self.fingerprint = value + elif name == 'keyMaterial': + self.material = value + else: + setattr(self, name, value) + + def delete(self): + """ + Delete the KeyPair. + + :rtype: bool + :return: True if successful, otherwise False. + """ + return self.connection.delete_key_pair(self.name) + + def save(self, directory_path): + """ + Save the material (the unencrypted PEM encoded RSA private key) + of a newly created KeyPair to a local file. + + :type directory_path: string + :param directory_path: The fully qualified path to the directory + in which the keypair will be saved. The + keypair file will be named using the name + of the keypair as the base name and .pem + for the file extension. If a file of that + name already exists in the directory, an + exception will be raised and the old file + will not be overwritten. + + :rtype: bool + :return: True if successful. + """ + if self.material: + file_path = os.path.join(directory_path, '%s.pem' % self.name) + if os.path.exists(file_path): + raise BotoClientError('%s already exists, it will not be overwritten' % file_path) + fp = open(file_path, 'wb') + fp.write(self.material) + fp.close() + return True + else: + raise BotoClientError('KeyPair contains no material') + + def copy_to_region(self, region): + """ + Create a new key pair of the same new in another region. + Note that the new key pair will use a different ssh + cert than the this key pair. After doing the copy, + you will need to save the material associated with the + new key pair (use the save method) to a local file. + + :type region: :class:`boto.ec2.regioninfo.RegionInfo` + :param region: The region to which this security group will be copied. + + :rtype: :class:`boto.ec2.keypair.KeyPair` + :return: The new key pair + """ + if region.name == self.region: + raise BotoClientError('Unable to copy to the same Region') + conn_params = self.connection.get_params() + rconn = region.connect(**conn_params) + kp = rconn.create_key_pair(self.name) + return kp + + + diff --git a/api/boto/ec2/launchspecification.py b/api/boto/ec2/launchspecification.py new file mode 100644 index 0000000..a574a38 --- /dev/null +++ b/api/boto/ec2/launchspecification.py @@ -0,0 +1,96 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a launch specification for Spot instances. +""" + +from boto.ec2.ec2object import EC2Object +from boto.resultset import ResultSet +from boto.ec2.blockdevicemapping import BlockDeviceMapping +from boto.ec2.instance import Group + +class GroupList(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'groupId': + self.append(value) + +class LaunchSpecification(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.key_name = None + self.instance_type = None + self.image_id = None + self.groups = [] + self.placement = None + self.kernel = None + self.ramdisk = None + self.monitored = False + self.subnet_id = None + self._in_monitoring_element = False + self.block_device_mapping = None + + def __repr__(self): + return 'LaunchSpecification(%s)' % self.image_id + + def startElement(self, name, attrs, connection): + if name == 'groupSet': + self.groups = ResultSet([('item', Group)]) + return self.groups + elif name == 'monitoring': + self._in_monitoring_element = True + elif name == 'blockDeviceMapping': + self.block_device_mapping = BlockDeviceMapping() + return self.block_device_mapping + else: + return None + + def endElement(self, name, value, connection): + if name == 'imageId': + self.image_id = value + elif name == 'keyName': + self.key_name = value + elif name == 'instanceType': + self.instance_type = value + elif name == 'availabilityZone': + self.placement = value + elif name == 'placement': + pass + elif name == 'kernelId': + self.kernel = value + elif name == 'ramdiskId': + self.ramdisk = value + elif name == 'subnetId': + self.subnet_id = value + elif name == 'state': + if self._in_monitoring_element: + if value == 'enabled': + self.monitored = True + self._in_monitoring_element = False + else: + setattr(self, name, value) + + diff --git a/api/boto/ec2/regioninfo.py b/api/boto/ec2/regioninfo.py new file mode 100644 index 0000000..ab61703 --- /dev/null +++ b/api/boto/ec2/regioninfo.py @@ -0,0 +1,60 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class RegionInfo(object): + """ + Represents an EC2 Region + """ + + def __init__(self, connection=None, name=None, endpoint=None): + self.connection = connection + self.name = name + self.endpoint = endpoint + + def __repr__(self): + return 'RegionInfo:%s' % self.name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'regionName': + self.name = value + elif name == 'regionEndpoint': + self.endpoint = value + else: + setattr(self, name, value) + + def connect(self, **kw_params): + """ + Connect to this Region's endpoint. Returns an EC2Connection + object pointing to the endpoint associated with this region. + You may pass any of the arguments accepted by the EC2Connection + object's constructor as keyword arguments and they will be + passed along to the EC2Connection object. + + :rtype: :class:`boto.ec2.connection.EC2Connection` + :return: The connection to this regions endpoint + """ + from boto.ec2.connection import EC2Connection + return EC2Connection(region=self, **kw_params) + + diff --git a/api/boto/ec2/reservedinstance.py b/api/boto/ec2/reservedinstance.py new file mode 100644 index 0000000..1d35c1d --- /dev/null +++ b/api/boto/ec2/reservedinstance.py @@ -0,0 +1,97 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.ec2.ec2object import EC2Object + +class ReservedInstancesOffering(EC2Object): + + def __init__(self, connection=None, id=None, instance_type=None, + availability_zone=None, duration=None, fixed_price=None, + usage_price=None, description=None): + EC2Object.__init__(self, connection) + self.id = id + self.instance_type = instance_type + self.availability_zone = availability_zone + self.duration = duration + self.fixed_price = fixed_price + self.usage_price = usage_price + self.description = description + + def __repr__(self): + return 'ReservedInstanceOffering:%s' % self.id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'reservedInstancesOfferingId': + self.id = value + elif name == 'instanceType': + self.instance_type = value + elif name == 'availabilityZone': + self.availability_zone = value + elif name == 'duration': + self.duration = value + elif name == 'fixedPrice': + self.fixed_price = value + elif name == 'usagePrice': + self.usage_price = value + elif name == 'productDescription': + self.description = value + else: + setattr(self, name, value) + + def describe(self): + print 'ID=%s' % self.id + print '\tInstance Type=%s' % self.instance_type + print '\tZone=%s' % self.availability_zone + print '\tDuration=%s' % self.duration + print '\tFixed Price=%s' % self.fixed_price + print '\tUsage Price=%s' % self.usage_price + print '\tDescription=%s' % self.description + + def purchase(self, instance_count=1): + return self.connection.purchase_reserved_instance_offering(self.id, instance_count) + +class ReservedInstance(ReservedInstancesOffering): + + def __init__(self, connection=None, id=None, instance_type=None, + availability_zone=None, duration=None, fixed_price=None, + usage_price=None, description=None, + instance_count=None, state=None): + ReservedInstancesOffering.__init__(self, connection, id, instance_type, + availability_zone, duration, fixed_price, + usage_price, description) + self.instance_count = instance_count + self.state = state + + def __repr__(self): + return 'ReservedInstance:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'reservedInstancesId': + self.id = value + if name == 'instanceCount': + self.instance_count = int(value) + elif name == 'state': + self.state = value + else: + ReservedInstancesOffering.endElement(self, name, value, connection) diff --git a/api/boto/ec2/securitygroup.py b/api/boto/ec2/securitygroup.py new file mode 100644 index 0000000..6f17ad3 --- /dev/null +++ b/api/boto/ec2/securitygroup.py @@ -0,0 +1,281 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Security Group +""" +from boto.ec2.ec2object import EC2Object + +class SecurityGroup(EC2Object): + + def __init__(self, connection=None, owner_id=None, + name=None, description=None): + EC2Object.__init__(self, connection) + self.owner_id = owner_id + self.name = name + self.description = description + self.rules = [] + + def __repr__(self): + return 'SecurityGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'item': + self.rules.append(IPPermissions(self)) + return self.rules[-1] + else: + return None + + def endElement(self, name, value, connection): + if name == 'ownerId': + self.owner_id = value + elif name == 'groupName': + self.name = value + elif name == 'groupDescription': + self.description = value + elif name == 'ipRanges': + pass + elif name == 'return': + if value == 'false': + self.status = False + elif value == 'true': + self.status = True + else: + raise Exception( + 'Unexpected value of status %s for image %s'%( + value, + self.id + ) + ) + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_security_group(self.name) + + def add_rule(self, ip_protocol, from_port, to_port, + src_group_name, src_group_owner_id, cidr_ip): + rule = IPPermissions(self) + rule.ip_protocol = ip_protocol + rule.from_port = from_port + rule.to_port = to_port + self.rules.append(rule) + rule.add_grant(src_group_name, src_group_owner_id, cidr_ip) + + def remove_rule(self, ip_protocol, from_port, to_port, + src_group_name, src_group_owner_id, cidr_ip): + target_rule = None + for rule in self.rules: + if rule.ip_protocol == ip_protocol: + if rule.from_port == from_port: + if rule.to_port == to_port: + target_rule = rule + target_grant = None + for grant in rule.grants: + if grant.name == src_group_name: + if grant.owner_id == src_group_owner_id: + if grant.cidr_ip == cidr_ip: + target_grant = grant + if target_grant: + rule.grants.remove(target_grant) + if len(rule.grants) == 0: + self.rules.remove(target_rule) + + def authorize(self, ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None, src_group=None): + """ + Add a new rule to this security group. + You need to pass in either src_group_name + OR ip_protocol, from_port, to_port, + and cidr_ip. In other words, either you are authorizing another + group or you are authorizing some ip-based rule. + + :type ip_protocol: string + :param ip_protocol: Either tcp | udp | icmp + + :type from_port: int + :param from_port: The beginning port number you are enabling + + :type to_port: int + :param to_port: The ending port number you are enabling + + :type to_port: string + :param to_port: The CIDR block you are providing access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :type src_group: :class:`boto.ec2.securitygroup.SecurityGroup` or + :class:`boto.ec2.securitygroup.GroupOrCIDR` + + :rtype: bool + :return: True if successful. + """ + if src_group: + from_port = None + to_port = None + cidr_ip = None + ip_protocol = None + src_group_name = src_group.name + src_group_owner_id = src_group.owner_id + else: + src_group_name = None + src_group_owner_id = None + status = self.connection.authorize_security_group(self.name, + src_group_name, + src_group_owner_id, + ip_protocol, + from_port, + to_port, + cidr_ip) + if status: + self.add_rule(ip_protocol, from_port, to_port, src_group_name, + src_group_owner_id, cidr_ip) + return status + + def revoke(self, ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None, src_group=None): + if src_group: + from_port=None + to_port=None + cidr_ip=None + ip_protocol = None + src_group_name = src_group.name + src_group_owner_id = src_group.owner_id + else: + src_group_name = None + src_group_owner_id = None + status = self.connection.revoke_security_group(self.name, + src_group_name, + src_group_owner_id, + ip_protocol, + from_port, + to_port, + cidr_ip) + if status: + self.remove_rule(ip_protocol, from_port, to_port, src_group_name, + src_group_owner_id, cidr_ip) + return status + + def copy_to_region(self, region, name=None): + """ + Create a copy of this security group in another region. + Note that the new security group will be a separate entity + and will not stay in sync automatically after the copy + operation. + + :type region: :class:`boto.ec2.regioninfo.RegionInfo` + :param region: The region to which this security group will be copied. + + :type name: string + :param name: The name of the copy. If not supplied, the copy + will have the same name as this security group. + + :rtype: :class:`boto.ec2.securitygroup.SecurityGroup` + :return: The new security group. + """ + if region.name == self.region: + raise BotoClientError('Unable to copy to the same Region') + conn_params = self.connection.get_params() + rconn = region.connect(**conn_params) + sg = rconn.create_security_group(name or self.name, self.description) + source_groups = [] + for rule in self.rules: + grant = rule.grants[0] + if grant.name: + if grant.name not in source_groups: + source_groups.append(grant.name) + sg.authorize(None, None, None, None, grant) + else: + sg.authorize(rule.ip_protocol, rule.from_port, rule.to_port, + grant.cidr_ip) + return sg + + def instances(self): + instances = [] + rs = self.connection.get_all_instances() + for reservation in rs: + uses_group = [g.id for g in reservation.groups if g.id == self.name] + if uses_group: + instances.extend(reservation.instances) + return instances + +class IPPermissions: + + def __init__(self, parent=None): + self.parent = parent + self.ip_protocol = None + self.from_port = None + self.to_port = None + self.grants = [] + + def __repr__(self): + return 'IPPermissions:%s(%s-%s)' % (self.ip_protocol, + self.from_port, self.to_port) + + def startElement(self, name, attrs, connection): + if name == 'item': + self.grants.append(GroupOrCIDR(self)) + return self.grants[-1] + return None + + def endElement(self, name, value, connection): + if name == 'ipProtocol': + self.ip_protocol = value + elif name == 'fromPort': + self.from_port = value + elif name == 'toPort': + self.to_port = value + else: + setattr(self, name, value) + + def add_grant(self, owner_id=None, name=None, cidr_ip=None): + grant = GroupOrCIDR(self) + grant.owner_id = owner_id + grant.name = name + grant.cidr_ip = cidr_ip + self.grants.append(grant) + return grant + +class GroupOrCIDR: + + def __init__(self, parent=None): + self.owner_id = None + self.name = None + self.cidr_ip = None + + def __repr__(self): + if self.cidr_ip: + return '%s' % self.cidr_ip + else: + return '%s-%s' % (self.name, self.owner_id) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'userId': + self.owner_id = value + elif name == 'groupName': + self.name = value + if name == 'cidrIp': + self.cidr_ip = value + else: + setattr(self, name, value) + diff --git a/api/boto/ec2/snapshot.py b/api/boto/ec2/snapshot.py new file mode 100644 index 0000000..33b53b0 --- /dev/null +++ b/api/boto/ec2/snapshot.py @@ -0,0 +1,124 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Elastic IP Snapshot +""" +from boto.ec2.ec2object import EC2Object + +class Snapshot(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.volume_id = None + self.status = None + self.progress = None + self.start_time = None + self.owner_id = None + self.volume_size = None + self.description = None + + def __repr__(self): + return 'Snapshot:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'snapshotId': + self.id = value + elif name == 'volumeId': + self.volume_id = value + elif name == 'status': + self.status = value + elif name == 'startTime': + self.start_time = value + elif name == 'ownerId': + self.owner_id = value + elif name == 'volumeSize': + self.volume_size = int(value) + elif name == 'description': + self.description = value + else: + setattr(self, name, value) + + def _update(self, updated): + self.progress = updated.progress + self.status = updated.status + + def update(self): + rs = self.connection.get_all_snapshots([self.id]) + if len(rs) > 0: + self._update(rs[0]) + return self.progress + + def delete(self): + return self.connection.delete_snapshot(self.id) + + def get_permissions(self): + attrs = self.connection.get_snapshot_attribute(self.id, + attribute='createVolumePermission') + return attrs.attrs + + def share(self, user_ids=None, groups=None): + return self.connection.modify_snapshot_attribute(self.id, + 'createVolumePermission', + 'add', + user_ids, + groups) + + def unshare(self, user_ids=None, groups=None): + return self.connection.modify_snapshot_attribute(self.id, + 'createVolumePermission', + 'remove', + user_ids, + groups) + + def reset_permissions(self): + return self.connection.reset_snapshot_attribute(self.id, 'createVolumePermission') + +class SnapshotAttribute: + + def __init__(self, parent=None): + self.snapshot_id = None + self.attrs = {} + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'createVolumePermission': + self.name = 'create_volume_permission' + elif name == 'group': + if self.attrs.has_key('groups'): + self.attrs['groups'].append(value) + else: + self.attrs['groups'] = [value] + elif name == 'userId': + if self.attrs.has_key('user_ids'): + self.attrs['user_ids'].append(value) + else: + self.attrs['user_ids'] = [value] + elif name == 'snapshotId': + self.snapshot_id = value + else: + setattr(self, name, value) + + + diff --git a/api/boto/ec2/spotdatafeedsubscription.py b/api/boto/ec2/spotdatafeedsubscription.py new file mode 100644 index 0000000..9b820a3 --- /dev/null +++ b/api/boto/ec2/spotdatafeedsubscription.py @@ -0,0 +1,63 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Spot Instance Datafeed Subscription +""" +from boto.ec2.ec2object import EC2Object +from boto.ec2.spotinstancerequest import SpotInstanceStateFault + +class SpotDatafeedSubscription(EC2Object): + + def __init__(self, connection=None, owner_id=None, + bucket=None, prefix=None, state=None,fault=None): + EC2Object.__init__(self, connection) + self.owner_id = owner_id + self.bucket = bucket + self.prefix = prefix + self.state = state + self.fault = fault + + def __repr__(self): + return 'SpotDatafeedSubscription:%s' % self.bucket + + def startElement(self, name, attrs, connection): + if name == 'fault': + self.fault = SpotInstanceStateFault() + return self.fault + else: + return None + + def endElement(self, name, value, connection): + if name == 'ownerId': + self.owner_id = value + elif name == 'bucket': + self.bucket = value + elif name == 'prefix': + self.prefix = value + elif name == 'state': + self.state = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_spot_datafeed_subscription() + diff --git a/api/boto/ec2/spotinstancerequest.py b/api/boto/ec2/spotinstancerequest.py new file mode 100644 index 0000000..5b1d7ce --- /dev/null +++ b/api/boto/ec2/spotinstancerequest.py @@ -0,0 +1,106 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Spot Instance Request +""" + +from boto.ec2.ec2object import EC2Object +from boto.ec2.launchspecification import LaunchSpecification + +class SpotInstanceStateFault(object): + + def __init__(self, code=None, message=None): + self.code = code + self.message = message + + def __repr__(self): + return '(%s, %s)' % (self.code, self.message) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'code': + self.code = code + elif name == 'message': + self.message = message + setattr(self, name, value) + +class SpotInstanceRequest(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.price = None + self.type = None + self.state = None + self.fault = None + self.valid_from = None + self.valid_until = None + self.launch_group = None + self.product_description = None + self.availability_zone_group = None + self.create_time = None + self.launch_specification = None + + def __repr__(self): + return 'SpotInstanceRequest:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'launchSpecification': + self.launch_specification = LaunchSpecification(connection) + return self.launch_specification + elif name == 'fault': + self.fault = SpotInstanceStateFault() + return self.fault + else: + return None + + def endElement(self, name, value, connection): + if name == 'spotInstanceRequestId': + self.id = value + elif name == 'spotPrice': + self.price = float(value) + elif name == 'type': + self.type = value + elif name == 'state': + self.state = value + elif name == 'productDescription': + self.product_description = value + elif name == 'validFrom': + self.valid_from = value + elif name == 'validUntil': + self.valid_until = value + elif name == 'launchGroup': + self.launch_group = value + elif name == 'availabilityZoneGroup': + self.availability_zone_group = value + elif name == 'createTime': + self.create_time = value + else: + setattr(self, name, value) + + def cancel(self): + self.connection.cancel_spot_instance_requests([self.id]) + + + diff --git a/api/boto/ec2/spotpricehistory.py b/api/boto/ec2/spotpricehistory.py new file mode 100644 index 0000000..d4e1711 --- /dev/null +++ b/api/boto/ec2/spotpricehistory.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Spot Instance Request +""" + +from boto.ec2.ec2object import EC2Object + +class SpotPriceHistory(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.price = 0.0 + self.instance_type = None + self.product_description = None + self.timestamp = None + + def __repr__(self): + return 'SpotPriceHistory(%s):%2f' % (self.instance_type, self.price) + + def endElement(self, name, value, connection): + if name == 'instanceType': + self.instance_type = value + elif name == 'spotPrice': + self.price = float(value) + elif name == 'productDescription': + self.product_description = value + elif name == 'timestamp': + self.timestamp = value + else: + setattr(self, name, value) + + diff --git a/api/boto/ec2/volume.py b/api/boto/ec2/volume.py new file mode 100644 index 0000000..200ca90 --- /dev/null +++ b/api/boto/ec2/volume.py @@ -0,0 +1,229 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Elastic IP Volume +""" +from boto.ec2.ec2object import EC2Object + +class Volume(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.create_time = None + self.status = None + self.size = None + self.snapshot_id = None + self.attach_data = None + self.zone = None + + def __repr__(self): + return 'Volume:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'attachmentSet': + self.attach_data = AttachmentSet() + return self.attach_data + else: + return None + + def endElement(self, name, value, connection): + if name == 'volumeId': + self.id = value + elif name == 'createTime': + self.create_time = value + elif name == 'status': + if value != '': + self.status = value + elif name == 'size': + self.size = int(value) + elif name == 'snapshotId': + self.snapshot_id = value + elif name == 'availabilityZone': + self.zone = value + else: + setattr(self, name, value) + + def _update(self, updated): + self.updated = updated + if hasattr(updated, 'create_time'): + self.create_time = updated.create_time + if hasattr(updated, 'status'): + self.status = updated.status + else: + self.status = None + if hasattr(updated, 'size'): + self.size = updated.size + if hasattr(updated, 'snapshot_id'): + self.snapshot_id = updated.snapshot_id + if hasattr(updated, 'attach_data'): + self.attach_data = updated.attach_data + if hasattr(updated, 'zone'): + self.zone = updated.zone + + def update(self): + rs = self.connection.get_all_volumes([self.id]) + if len(rs) > 0: + self._update(rs[0]) + return self.status + + def delete(self): + """ + Delete this EBS volume. + + :rtype: bool + :return: True if successful + """ + return self.connection.delete_volume(self.id) + + def attach(self, instance_id, device): + """ + Attach this EBS volume to an EC2 instance. + + :type instance_id: str + :param instance_id: The ID of the EC2 instance to which it will + be attached. + + :type device: str + :param device: The device on the instance through which the + volume will be exposted (e.g. /dev/sdh) + + :rtype: bool + :return: True if successful + """ + return self.connection.attach_volume(self.id, instance_id, device) + + def detach(self, force=False): + """ + Detach this EBS volume from an EC2 instance. + + :type force: bool + :param force: Forces detachment if the previous detachment attempt did + not occur cleanly. This option can lead to data loss or + a corrupted file system. Use this option only as a last + resort to detach a volume from a failed instance. The + instance will not have an opportunity to flush file system + caches nor file system meta data. If you use this option, + you must perform file system check and repair procedures. + + :rtype: bool + :return: True if successful + """ + instance_id = None + if self.attach_data: + instance_id = self.attach_data.instance_id + device = None + if self.attach_data: + device = self.attach_data.device + return self.connection.detach_volume(self.id, instance_id, device, force) + + def create_snapshot(self, description=None): + """ + Create a snapshot of this EBS Volume. + + :type description: str + :param description: A description of the snapshot. Limited to 256 characters. + + :rtype: bool + :return: True if successful + """ + return self.connection.create_snapshot(self.id, description) + + def volume_state(self): + """ + Returns the state of the volume. Same value as the status attribute. + """ + return self.status + + def attachment_state(self): + """ + Get the attachmentSet information for the volume. This info is stored + in a dictionary object and contains at least the following info: + + - volumeId + - instanceId + - device + - status + - attachTime + """ + state = None + if self.attach_data: + state = self.attach_data.status + return state + + def snapshots(self, owner=None, restorable_by=None): + """ + Get all snapshots related to this volume. Note that this requires + that all available snapshots for the account be retrieved from EC2 + first and then the list is filtered client-side to contain only + those for this volume. + + :type owner: str + :param owner: If present, only the snapshots owned by the specified user + will be returned. Valid values are: + self | amazon | AWS Account ID + + :type restorable_by: str + :param restorable_by: If present, only the snapshots that are restorable + by the specified account id will be returned. + + :rtype: list of L{boto.ec2.snapshot.Snapshot} + :return: The requested Snapshot objects + + """ + rs = self.connection.get_all_snapshots(owner=owner, + restorable_by=restorable_by) + mine = [] + for snap in rs: + if snap.volume_id == self.id: + mine.append(snap) + return mine + +class AttachmentSet(object): + + def __init__(self): + self.id = None + self.instance_id = None + self.status = None + self.attach_time = None + self.device = None + + def __repr__(self): + return 'AttachmentSet:%s' % self.id + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'volumeId': + self.id = value + elif name == 'instanceId': + self.instance_id = value + elif name == 'status': + self.status = value + elif name == 'attachTime': + self.attach_time = value + elif name == 'device': + self.device = value + else: + setattr(self, name, value) + diff --git a/api/boto/ec2/zone.py b/api/boto/ec2/zone.py new file mode 100644 index 0000000..aec79b2 --- /dev/null +++ b/api/boto/ec2/zone.py @@ -0,0 +1,47 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Availability Zone +""" +from boto.ec2.ec2object import EC2Object + +class Zone(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.name = None + self.state = None + + def __repr__(self): + return 'Zone:%s' % self.name + + def endElement(self, name, value, connection): + if name == 'zoneName': + self.name = value + elif name == 'zoneState': + self.state = value + else: + setattr(self, name, value) + + + + diff --git a/api/boto/exception.py b/api/boto/exception.py new file mode 100644 index 0000000..ba65694 --- /dev/null +++ b/api/boto/exception.py @@ -0,0 +1,284 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Exception classes - Subclassing allows you to check for specific errors +""" + +from boto import handler +from boto.resultset import ResultSet + +import xml.sax + +class BotoClientError(StandardError): + """ + General Boto Client error (error accessing AWS) + """ + + def __init__(self, reason): + self.reason = reason + + def __repr__(self): + return 'S3Error: %s' % self.reason + + def __str__(self): + return 'S3Error: %s' % self.reason + +class SDBPersistenceError(StandardError): + + pass + +class S3PermissionsError(BotoClientError): + """ + Permissions error when accessing a bucket or key on S3. + """ + pass + +class BotoServerError(StandardError): + + def __init__(self, status, reason, body=None): + self.status = status + self.reason = reason + self.body = body or '' + self.request_id = None + self.error_code = None + self.error_message = None + self.box_usage = None + + # Attempt to parse the error response. If body isn't present, + # then just ignore the error response. + if self.body: + try: + h = handler.XmlHandler(self, self) + xml.sax.parseString(self.body, h) + except xml.sax.SAXParseException, pe: + # Go ahead and clean up anything that may have + # managed to get into the error data so we + # don't get partial garbage. + print "Warning: failed to parse error message from AWS: %s" % pe + self._cleanupParsedProperties() + + def __getattr__(self, name): + if name == 'message': + return self.error_message + if name == 'code': + return self.error_code + raise AttributeError + + def __repr__(self): + return '%s: %s %s\n%s' % (self.__class__.__name__, + self.status, self.reason, self.body) + + def __str__(self): + return '%s: %s %s\n%s' % (self.__class__.__name__, + self.status, self.reason, self.body) + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name in ('RequestId', 'RequestID'): + self.request_id = value + elif name == 'Code': + self.error_code = value + elif name == 'Message': + self.error_message = value + elif name == 'BoxUsage': + self.box_usage = value + return None + + def _cleanupParsedProperties(self): + self.request_id = None + self.error_code = None + self.error_message = None + self.box_usage = None + +class ConsoleOutput: + + def __init__(self, parent=None): + self.parent = parent + self.instance_id = None + self.timestamp = None + self.comment = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'instanceId': + self.instance_id = value + elif name == 'output': + self.output = base64.b64decode(value) + else: + setattr(self, name, value) + +class S3CreateError(BotoServerError): + """ + Error creating a bucket or key on S3. + """ + def __init__(self, status, reason, body=None): + self.bucket = None + BotoServerError.__init__(self, status, reason, body) + + def endElement(self, name, value, connection): + if name == 'BucketName': + self.bucket = value + else: + return BotoServerError.endElement(self, name, value, connection) + +class S3CopyError(BotoServerError): + """ + Error copying a key on S3. + """ + pass + +class SQSError(BotoServerError): + """ + General Error on Simple Queue Service. + """ + def __init__(self, status, reason, body=None): + self.detail = None + self.type = None + BotoServerError.__init__(self, status, reason, body) + + def startElement(self, name, attrs, connection): + return BotoServerError.startElement(self, name, attrs, connection) + + def endElement(self, name, value, connection): + if name == 'Detail': + self.detail = value + elif name == 'Type': + self.type = value + else: + return BotoServerError.endElement(self, name, value, connection) + + def _cleanupParsedProperties(self): + BotoServerError._cleanupParsedProperties(self) + for p in ('detail', 'type'): + setattr(self, p, None) + +class SQSDecodeError(BotoClientError): + """ + Error when decoding an SQS message. + """ + def __init__(self, reason, message): + self.reason = reason + self.message = message + + def __repr__(self): + return 'SQSDecodeError: %s' % self.reason + + def __str__(self): + return 'SQSDecodeError: %s' % self.reason + +class S3ResponseError(BotoServerError): + """ + Error in response from S3. + """ + def __init__(self, status, reason, body=None): + self.resource = None + BotoServerError.__init__(self, status, reason, body) + + def startElement(self, name, attrs, connection): + return BotoServerError.startElement(self, name, attrs, connection) + + def endElement(self, name, value, connection): + if name == 'Resource': + self.resource = value + else: + return BotoServerError.endElement(self, name, value, connection) + + def _cleanupParsedProperties(self): + BotoServerError._cleanupParsedProperties(self) + for p in ('resource'): + setattr(self, p, None) + +class EC2ResponseError(BotoServerError): + """ + Error in response from EC2. + """ + + def __init__(self, status, reason, body=None): + self.errors = None + self._errorResultSet = [] + BotoServerError.__init__(self, status, reason, body) + self.errors = [ (e.error_code, e.error_message) \ + for e in self._errorResultSet ] + if len(self.errors): + self.error_code, self.error_message = self.errors[0] + + def startElement(self, name, attrs, connection): + if name == 'Errors': + self._errorResultSet = ResultSet([('Error', _EC2Error)]) + return self._errorResultSet + else: + return None + + def endElement(self, name, value, connection): + if name == 'RequestID': + self.request_id = value + else: + return None # don't call subclass here + + def _cleanupParsedProperties(self): + BotoServerError._cleanupParsedProperties(self) + self._errorResultSet = [] + for p in ('errors'): + setattr(self, p, None) + +class _EC2Error: + + def __init__(self, connection=None): + self.connection = connection + self.error_code = None + self.error_message = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Code': + self.error_code = value + elif name == 'Message': + self.error_message = value + else: + return None + +class SDBResponseError(BotoServerError): + """ + Error in respones from SDB. + """ + pass + +class AWSConnectionError(BotoClientError): + """ + General error connecting to Amazon Web Services. + """ + pass + +class S3DataError(BotoClientError): + """ + Error receiving data from S3. + """ + pass + +class FPSResponseError(BotoServerError): + pass diff --git a/api/boto/fps/__init__.py b/api/boto/fps/__init__.py new file mode 100644 index 0000000..2f44483 --- /dev/null +++ b/api/boto/fps/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2008, Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/api/boto/fps/connection.py b/api/boto/fps/connection.py new file mode 100644 index 0000000..0f14775 --- /dev/null +++ b/api/boto/fps/connection.py @@ -0,0 +1,173 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import urllib +import xml.sax +import uuid +import boto +import boto.utils +import urllib +from boto import handler +from boto.connection import AWSQueryConnection +from boto.resultset import ResultSet +from boto.exception import FPSResponseError + +class FPSConnection(AWSQueryConnection): + + APIVersion = '2007-01-08' + SignatureVersion = '1' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + host='fps.sandbox.amazonaws.com', debug=0, + https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + host, debug, https_connection_factory) + + def install_payment_instruction(self, instruction, token_type="Unrestricted", transaction_id=None): + """ + InstallPaymentInstruction + instruction: The PaymentInstruction to send, for example: + + MyRole=='Caller' orSay 'Roles do not match'; + + token_type: Defaults to "Unrestricted" + transaction_id: Defaults to a new ID + """ + + if(transaction_id == None): + transaction_id = uuid.uuid4() + params = {} + params['PaymentInstruction'] = instruction + params['TokenType'] = token_type + params['CallerReference'] = transaction_id + response = self.make_request("InstallPaymentInstruction", params) + return response + + def install_caller_instruction(self, token_type="Unrestricted", transaction_id=None): + """ + Set us up as a caller + This will install a new caller_token into the FPS section. + This should really only be called to regenerate the caller token. + """ + response = self.install_payment_instruction("MyRole=='Caller';", token_type=token_type, transaction_id=transaction_id) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + caller_token = rs.TokenId + try: + boto.config.save_system_option("FPS", "caller_token", caller_token) + except(IOError): + boto.config.save_user_option("FPS", "caller_token", caller_token) + return caller_token + else: + raise FPSResponseError(response.status, respons.reason, body) + + def install_recipient_instruction(self, token_type="Unrestricted", transaction_id=None): + """ + Set us up as a Recipient + This will install a new caller_token into the FPS section. + This should really only be called to regenerate the recipient token. + """ + response = self.install_payment_instruction("MyRole=='Recipient';", token_type=token_type, transaction_id=transaction_id) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + recipient_token = rs.TokenId + try: + boto.config.save_system_option("FPS", "recipient_token", recipient_token) + except(IOError): + boto.config.save_user_option("FPS", "recipient_token", recipient_token) + + return recipient_token + else: + raise FPSResponseError(response.status, respons.reason, body) + + def make_url(self, returnURL, paymentReason, pipelineName, **params): + """ + Generate the URL with the signature required for a transaction + """ + params['callerKey'] = str(self.aws_access_key_id) + params['returnURL'] = str(returnURL) + params['paymentReason'] = str(paymentReason) + params['pipelineName'] = pipelineName + + if(not params.has_key('callerReference')): + params['callerReference'] = str(uuid.uuid4()) + + url = "" + keys = params.keys() + keys.sort() + for k in keys: + url += "&%s=%s" % (k, urllib.quote_plus(str(params[k]))) + + url = "/cobranded-ui/actions/start?%s" % ( url[1:]) + signature= boto.utils.encode(self.aws_secret_access_key, url, True) + return "https://authorize.payments-sandbox.amazon.com%s&awsSignature=%s" % (url, signature) + + def make_payment(self, amount, sender_token, charge_fee_to="Recipient", reference=None, senderReference=None, recipientReference=None, senderDescription=None, recipientDescription=None, callerDescription=None, metadata=None, transactionDate=None): + """ + Make a payment transaction + You must specify the amount and the sender token. + """ + params = {} + params['RecipientTokenId'] = boto.config.get("FPS", "recipient_token") + params['CallerTokenId'] = boto.config.get("FPS", "caller_token") + params['SenderTokenId'] = sender_token + params['TransactionAmount.Amount'] = str(amount) + params['TransactionAmount.CurrencyCode'] = "USD" + params['ChargeFeeTo'] = charge_fee_to + + if(transactionDate != None): + params['TransactionDate'] = transactionDate + if(senderReference != None): + params['SenderReference'] = senderReference + if(recipientReference != None): + params['RecipientReference'] = recipientReference + if(senderDescription != None): + params['SenderDescription'] = senderDescription + if(recipientDescription != None): + params['RecipientDescription'] = recipientDescription + if(callerDescription != None): + params['CallerDescription'] = callerDescription + if(metadata != None): + params['MetaData'] = metadata + if(transactionDate != None): + params['TransactionDate'] = transactionDate + if(reference == None): + reference = uuid.uuid4() + params['CallerReference'] = reference + + response = self.make_request("Pay", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) diff --git a/api/boto/handler.py b/api/boto/handler.py new file mode 100644 index 0000000..525f9c9 --- /dev/null +++ b/api/boto/handler.py @@ -0,0 +1,46 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax + +class XmlHandler(xml.sax.ContentHandler): + + def __init__(self, root_node, connection): + self.connection = connection + self.nodes = [('root', root_node)] + self.current_text = '' + + def startElement(self, name, attrs): + self.current_text = '' + new_node = self.nodes[-1][1].startElement(name, attrs, self.connection) + if new_node != None: + self.nodes.append((name, new_node)) + + def endElement(self, name): + self.nodes[-1][1].endElement(name, self.current_text, self.connection) + if self.nodes[-1][0] == name: + self.nodes.pop() + self.current_text = '' + + def characters(self, content): + self.current_text += content + + diff --git a/api/boto/manage/__init__.py b/api/boto/manage/__init__.py new file mode 100644 index 0000000..49d029b --- /dev/null +++ b/api/boto/manage/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/api/boto/manage/cmdshell.py b/api/boto/manage/cmdshell.py new file mode 100644 index 0000000..340b1e2 --- /dev/null +++ b/api/boto/manage/cmdshell.py @@ -0,0 +1,165 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.mashups.interactive import interactive_shell +import boto +import os, time, shutil +import StringIO +import paramiko +import socket + +class SSHClient(object): + + def __init__(self, server, host_key_file='~/.ssh/known_hosts', uname='root'): + self.server = server + self.host_key_file = host_key_file + self.uname = uname + self._pkey = paramiko.RSAKey.from_private_key_file(server.ssh_key_file) + self._ssh_client = paramiko.SSHClient() + self._ssh_client.load_system_host_keys() + self._ssh_client.load_host_keys(os.path.expanduser(host_key_file)) + self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.connect() + + def connect(self): + retry = 0 + while retry < 5: + try: + self._ssh_client.connect(self.server.hostname, username=self.uname, pkey=self._pkey) + return + except socket.error, (value,message): + if value == 61: + print 'SSH Connection refused, will retry in 5 seconds' + time.sleep(5) + retry += 1 + else: + raise + except paramiko.BadHostKeyException: + print "%s has an entry in ~/.ssh/known_hosts and it doesn't match" % self.server.hostname + print 'Edit that file to remove the entry and then hit return to try again' + rawinput('Hit Enter when ready') + retry += 1 + except EOFError: + print 'Unexpected Error from SSH Connection, retry in 5 seconds' + time.sleep(5) + retry += 1 + print 'Could not establish SSH connection' + + def get_file(self, src, dst): + sftp_client = self._ssh_client.open_sftp() + sftp_client.get(src, dst) + + def put_file(self, src, dst): + sftp_client = self._ssh_client.open_sftp() + sftp_client.put(src, dst) + + def listdir(self, path): + sftp_client = self._ssh_client.open_sftp() + return sftp_client.listdir(path) + + def open_sftp(self): + return self._ssh_client.open_sftp() + + def isdir(self, path): + status = self.run('[ -d %s ] || echo "FALSE"' % path) + if status[1].startswith('FALSE'): + return 0 + return 1 + + def exists(self, path): + status = self.run('[ -a %s ] || echo "FALSE"' % path) + if status[1].startswith('FALSE'): + return 0 + return 1 + + def shell(self): + channel = self._ssh_client.invoke_shell() + interactive_shell(channel) + + def run(self, command): + boto.log.info('running:%s on %s' % (command, self.server.instance_id)) + log_fp = StringIO.StringIO() + status = 0 + try: + t = self._ssh_client.exec_command(command) + except paramiko.SSHException: + status = 1 + log_fp.write(t[1].read()) + log_fp.write(t[2].read()) + t[0].close() + t[1].close() + t[2].close() + boto.log.info('output: %s' % log_fp.getvalue()) + return (status, log_fp.getvalue()) + + def close(self): + transport = self._ssh_client.get_transport() + transport.close() + self.server.reset_cmdshell() + +class LocalClient(object): + + def __init__(self, server, host_key_file=None, uname='root'): + self.server = server + self.host_key_file = host_key_file + self.uname = uname + + def get_file(self, src, dst): + shutil.copyfile(src, dst) + + def put_file(self, src, dst): + shutil.copyfile(src, dst) + + def listdir(self, path): + return os.listdir(path) + + def isdir(self, path): + return os.path.isdir(path) + + def exists(self, path): + return os.path.exists(path) + + def shell(self): + raise NotImplementedError, 'shell not supported with LocalClient' + + def run(self): + boto.log.info('running:%s' % self.command) + log_fp = StringIO.StringIO() + process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + while process.poll() == None: + time.sleep(1) + t = process.communicate() + log_fp.write(t[0]) + log_fp.write(t[1]) + boto.log.info(log_fp.getvalue()) + boto.log.info('output: %s' % log_fp.getvalue()) + return (process.returncode, log_fp.getvalue()) + + def close(self): + pass + +def start(server): + instance_id = boto.config.get('Instance', 'instance-id', None) + if instance_id == server.instance_id: + return LocalClient(server) + else: + return SSHClient(server) diff --git a/api/boto/manage/propget.py b/api/boto/manage/propget.py new file mode 100644 index 0000000..172e1aa --- /dev/null +++ b/api/boto/manage/propget.py @@ -0,0 +1,66 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import os +from boto.sdb.db.property import * + +def get(prop, choices=None): + prompt = prop.verbose_name + if not prompt: + prompt = prop.name + if choices: + if callable(choices): + choices = choices() + else: + choices = prop.get_choices() + valid = False + while not valid: + if choices: + min = 1 + max = len(choices) + for i in range(min, max+1): + value = choices[i-1] + if isinstance(value, tuple): + value = value[0] + print '[%d] %s' % (i, value) + value = raw_input('%s [%d-%d]: ' % (prompt, min, max)) + try: + int_value = int(value) + value = choices[int_value-1] + if isinstance(value, tuple): + value = value[1] + valid = True + except ValueError: + print '%s is not a valid choice' % value + except IndexError: + print '%s is not within the range[%d-%d]' % (min, max) + else: + value = raw_input('%s: ' % prompt) + try: + value = prop.validate(value) + if prop.empty(value) and prop.required: + print 'A value is required' + else: + valid = True + except: + print 'Invalid value: %s' % value + return value + diff --git a/api/boto/manage/server.py b/api/boto/manage/server.py new file mode 100644 index 0000000..cc623ef --- /dev/null +++ b/api/boto/manage/server.py @@ -0,0 +1,542 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +High-level abstraction of an EC2 server +""" +from __future__ import with_statement +import boto.ec2 +from boto.mashups.iobject import IObject +from boto.pyami.config import BotoConfigPath, Config +from boto.sdb.db.model import Model +from boto.sdb.db.property import * +from boto.manage import propget +from boto.ec2.zone import Zone +from boto.ec2.keypair import KeyPair +import os, time, StringIO +from contextlib import closing +from boto.exception import EC2ResponseError + +InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', + 'c1.medium', 'c1.xlarge', + 'm2.2xlarge', 'm2.4xlarge'] + +class Bundler(object): + + def __init__(self, server, uname='root'): + from boto.manage.cmdshell import SSHClient + self.server = server + self.uname = uname + self.ssh_client = SSHClient(server, uname=uname) + + def copy_x509(self, key_file, cert_file): + print '\tcopying cert and pk over to /mnt directory on server' + sftp_client = self.ssh_client.open_sftp() + path, name = os.path.split(key_file) + self.remote_key_file = '/mnt/%s' % name + self.ssh_client.put_file(key_file, self.remote_key_file) + path, name = os.path.split(cert_file) + self.remote_cert_file = '/mnt/%s' % name + self.ssh_client.put_file(cert_file, self.remote_cert_file) + print '...complete!' + + def bundle_image(self, prefix, size, ssh_key): + command = "" + if self.uname != 'root': + command = "sudo " + command += 'ec2-bundle-vol ' + command += '-c %s -k %s ' % (self.remote_cert_file, self.remote_key_file) + command += '-u %s ' % self.server._reservation.owner_id + command += '-p %s ' % prefix + command += '-s %d ' % size + command += '-d /mnt ' + if self.server.instance_type == 'm1.small' or self.server.instance_type == 'c1.medium': + command += '-r i386' + else: + command += '-r x86_64' + return command + + def upload_bundle(self, bucket, prefix, ssh_key): + command = "" + if self.uname != 'root': + command = "sudo " + command += 'ec2-upload-bundle ' + command += '-m /mnt/%s.manifest.xml ' % prefix + command += '-b %s ' % bucket + command += '-a %s ' % self.server.ec2.aws_access_key_id + command += '-s %s ' % self.server.ec2.aws_secret_access_key + return command + + def bundle(self, bucket=None, prefix=None, key_file=None, cert_file=None, + size=None, ssh_key=None, fp=None, clear_history=True): + iobject = IObject() + if not bucket: + bucket = iobject.get_string('Name of S3 bucket') + if not prefix: + prefix = iobject.get_string('Prefix for AMI file') + if not key_file: + key_file = iobject.get_filename('Path to RSA private key file') + if not cert_file: + cert_file = iobject.get_filename('Path to RSA public cert file') + if not size: + size = iobject.get_int('Size (in MB) of bundled image') + if not ssh_key: + ssh_key = self.server.get_ssh_key_file() + self.copy_x509(key_file, cert_file) + if not fp: + fp = StringIO.StringIO() + fp.write('mv %s /mnt/boto.cfg; ' % BotoConfigPath) + fp.write('mv /root/.ssh/authorized_keys /mnt/authorized_keys; ') + if clear_history: + fp.write('history -c; ') + fp.write(self.bundle_image(prefix, size, ssh_key)) + fp.write('; ') + fp.write(self.upload_bundle(bucket, prefix, ssh_key)) + fp.write('; ') + fp.write('mv /mnt/boto.cfg %s; ' % BotoConfigPath) + fp.write('mv /mnt/authorized_keys /root/.ssh/authorized_keys\n') + command = fp.getvalue() + print 'running the following command on the remote server:' + print command + t = self.ssh_client.run(command) + print '\t%s' % t[0] + print '\t%s' % t[1] + print '...complete!' + print 'registering image...' + self.image_id = self.server.ec2.register_image('%s/%s.manifest.xml' % (bucket, prefix)) + return self.image_id + +class CommandLineGetter(object): + + def get_ami_list(self): + my_amis = [] + for ami in self.ec2.get_all_images(): + # hack alert, need a better way to do this! + if ami.location.find('pyami') >= 0: + my_amis.append((ami.location, ami)) + return my_amis + + def get_region(self, params): + region = params.get('region', None) + if isinstance(region, str) or isinstance(region, unicode): + region = boto.ec2.get_region(region) + params['region'] = region + if not region: + prop = self.cls.find_property('region_name') + params['region'] = propget.get(prop, choices=boto.ec2.regions) + + def get_name(self, params): + if not params.get('name', None): + prop = self.cls.find_property('name') + params['name'] = propget.get(prop) + + def get_description(self, params): + if not params.get('description', None): + prop = self.cls.find_property('description') + params['description'] = propget.get(prop) + + def get_instance_type(self, params): + if not params.get('instance_type', None): + prop = StringProperty(name='instance_type', verbose_name='Instance Type', + choices=InstanceTypes) + params['instance_type'] = propget.get(prop) + + def get_quantity(self, params): + if not params.get('quantity', None): + prop = IntegerProperty(name='quantity', verbose_name='Number of Instances') + params['quantity'] = propget.get(prop) + + def get_zone(self, params): + if not params.get('zone', None): + prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', + choices=self.ec2.get_all_zones) + params['zone'] = propget.get(prop) + + def get_ami_id(self, params): + ami = params.get('ami', None) + if isinstance(ami, str) or isinstance(ami, unicode): + ami_list = self.get_ami_list() + for l,a in ami_list: + if a.id == ami: + ami = a + params['ami'] = a + if not params.get('ami', None): + prop = StringProperty(name='ami', verbose_name='AMI', + choices=self.get_ami_list) + params['ami'] = propget.get(prop) + + def get_group(self, params): + group = params.get('group', None) + if isinstance(group, str) or isinstance(group, unicode): + group_list = self.ec2.get_all_security_groups() + for g in group_list: + if g.name == group: + group = g + params['group'] = g + if not group: + prop = StringProperty(name='group', verbose_name='EC2 Security Group', + choices=self.ec2.get_all_security_groups) + params['group'] = propget.get(prop) + + def get_key(self, params): + keypair = params.get('keypair', None) + if isinstance(keypair, str) or isinstance(keypair, unicode): + key_list = self.ec2.get_all_key_pairs() + for k in key_list: + if k.name == keypair: + keypair = k.name + params['keypair'] = k.name + if not keypair: + prop = StringProperty(name='keypair', verbose_name='EC2 KeyPair', + choices=self.ec2.get_all_key_pairs) + params['keypair'] = propget.get(prop).name + + def get(self, cls, params): + self.cls = cls + self.get_region(params) + self.ec2 = params['region'].connect() + self.get_name(params) + self.get_description(params) + self.get_instance_type(params) + self.get_zone(params) + self.get_quantity(params) + self.get_ami_id(params) + self.get_group(params) + self.get_key(params) + +class Server(Model): + + # + # The properties of this object consists of real properties for data that + # is not already stored in EC2 somewhere (e.g. name, description) plus + # calculated properties for all of the properties that are already in + # EC2 (e.g. hostname, security groups, etc.) + # + name = StringProperty(unique=True, verbose_name="Name") + description = StringProperty(verbose_name="Description") + region_name = StringProperty(verbose_name="EC2 Region Name") + instance_id = StringProperty(verbose_name="EC2 Instance ID") + elastic_ip = StringProperty(verbose_name="EC2 Elastic IP Address") + production = BooleanProperty(verbose_name="Is This Server Production", default=False) + ami_id = CalculatedProperty(verbose_name="AMI ID", calculated_type=str, use_method=True) + zone = CalculatedProperty(verbose_name="Availability Zone Name", calculated_type=str, use_method=True) + hostname = CalculatedProperty(verbose_name="Public DNS Name", calculated_type=str, use_method=True) + private_hostname = CalculatedProperty(verbose_name="Private DNS Name", calculated_type=str, use_method=True) + groups = CalculatedProperty(verbose_name="Security Groups", calculated_type=list, use_method=True) + security_group = CalculatedProperty(verbose_name="Primary Security Group Name", calculated_type=str, use_method=True) + key_name = CalculatedProperty(verbose_name="Key Name", calculated_type=str, use_method=True) + instance_type = CalculatedProperty(verbose_name="Instance Type", calculated_type=str, use_method=True) + status = CalculatedProperty(verbose_name="Current Status", calculated_type=str, use_method=True) + launch_time = CalculatedProperty(verbose_name="Server Launch Time", calculated_type=str, use_method=True) + console_output = CalculatedProperty(verbose_name="Console Output", calculated_type=file, use_method=True) + + packages = [] + plugins = [] + + @classmethod + def add_credentials(cls, cfg, aws_access_key_id, aws_secret_access_key): + if not cfg.has_section('Credentials'): + cfg.add_section('Credentials') + cfg.set('Credentials', 'aws_access_key_id', aws_access_key_id) + cfg.set('Credentials', 'aws_secret_access_key', aws_secret_access_key) + if not cfg.has_section('DB_Server'): + cfg.add_section('DB_Server') + cfg.set('DB_Server', 'db_type', 'SimpleDB') + cfg.set('DB_Server', 'db_name', cls._manager.domain.name) + + ''' + Create a new instance based on the specified configuration file or the specified + configuration and the passed in parameters. + + If the config_file argument is not None, the configuration is read from there. + Otherwise, the cfg argument is used. + + The config file may include other config files with a #import reference. The included + config files must reside in the same directory as the specified file. + + The logical_volume argument, if supplied, will be used to get the current physical + volume ID and use that as an override of the value specified in the config file. This + may be useful for debugging purposes when you want to debug with a production config + file but a test Volume. + + The dictionary argument may be used to override any EC2 configuration values in the + config file. + ''' + @classmethod + def create(cls, config_file=None, logical_volume = None, cfg = None, **params): + if config_file: + cfg = Config(path=config_file) + if cfg.has_section('EC2'): + # include any EC2 configuration values that aren't specified in params: + for option in cfg.options('EC2'): + if option not in params: + params[option] = cfg.get('EC2', option) + getter = CommandLineGetter() + getter.get(cls, params) + region = params.get('region') + ec2 = region.connect() + cls.add_credentials(cfg, ec2.aws_access_key_id, ec2.aws_secret_access_key) + ami = params.get('ami') + kp = params.get('keypair') + group = params.get('group') + zone = params.get('zone') + # deal with possibly passed in logical volume: + if logical_volume != None: + cfg.set('EBS', 'logical_volume_name', logical_volume.name) + cfg_fp = StringIO.StringIO() + cfg.write(cfg_fp) + # deal with the possibility that zone and/or keypair are strings read from the config file: + if isinstance(zone, Zone): + zone = zone.name + if isinstance(kp, KeyPair): + kp = kp.name + reservation = ami.run(min_count=1, + max_count=params.get('quantity', 1), + key_name=kp, + security_groups=[group], + instance_type=params.get('instance_type'), + placement = zone, + user_data = cfg_fp.getvalue()) + l = [] + i = 0 + elastic_ip = params.get('elastic_ip') + instances = reservation.instances + if elastic_ip != None and instances.__len__() > 0: + instance = instances[0] + while instance.update() != 'running': + time.sleep(1) + instance.use_ip(elastic_ip) + print 'set the elastic IP of the first instance to %s' % elastic_ip + for instance in instances: + s = cls() + s.ec2 = ec2 + s.name = params.get('name') + '' if i==0 else str(i) + s.description = params.get('description') + s.region_name = region.name + s.instance_id = instance.id + if elastic_ip and i == 0: + s.elastic_ip = elastic_ip + s.put() + l.append(s) + i += 1 + return l + + @classmethod + def create_from_instance_id(cls, instance_id, name, description=''): + regions = boto.ec2.regions() + for region in regions: + ec2 = region.connect() + try: + rs = ec2.get_all_instances([instance_id]) + except: + rs = [] + if len(rs) == 1: + s = cls() + s.ec2 = ec2 + s.name = name + s.description = description + s.region_name = region.name + s.instance_id = instance_id + s._reservation = rs[0] + for instance in s._reservation.instances: + if instance.id == instance_id: + s._instance = instance + s.put() + return s + return None + + @classmethod + def create_from_current_instances(cls): + servers = [] + regions = boto.ec2.regions() + for region in regions: + ec2 = region.connect() + rs = ec2.get_all_instances() + for reservation in rs: + for instance in reservation.instances: + try: + Server.find(instance_id=instance.id).next() + boto.log.info('Server for %s already exists' % instance.id) + except StopIteration: + s = cls() + s.ec2 = ec2 + s.name = instance.id + s.region_name = region.name + s.instance_id = instance.id + s._reservation = reservation + s.put() + servers.append(s) + return servers + + def __init__(self, id=None, **kw): + Model.__init__(self, id, **kw) + self.ssh_key_file = None + self.ec2 = None + self._cmdshell = None + self._reservation = None + self._instance = None + self._setup_ec2() + + def _setup_ec2(self): + if self.ec2 and self._instance and self._reservation: + return + if self.id: + if self.region_name: + for region in boto.ec2.regions(): + if region.name == self.region_name: + self.ec2 = region.connect() + if self.instance_id and not self._instance: + try: + rs = self.ec2.get_all_instances([self.instance_id]) + if len(rs) >= 1: + for instance in rs[0].instances: + if instance.id == self.instance_id: + self._reservation = rs[0] + self._instance = instance + except EC2ResponseError: + pass + + def _status(self): + status = '' + if self._instance: + self._instance.update() + status = self._instance.state + return status + + def _hostname(self): + hostname = '' + if self._instance: + hostname = self._instance.public_dns_name + return hostname + + def _private_hostname(self): + hostname = '' + if self._instance: + hostname = self._instance.private_dns_name + return hostname + + def _instance_type(self): + it = '' + if self._instance: + it = self._instance.instance_type + return it + + def _launch_time(self): + lt = '' + if self._instance: + lt = self._instance.launch_time + return lt + + def _console_output(self): + co = '' + if self._instance: + co = self._instance.get_console_output() + return co + + def _groups(self): + gn = [] + if self._reservation: + gn = self._reservation.groups + return gn + + def _security_group(self): + groups = self._groups() + if len(groups) >= 1: + return groups[0].id + return "" + + def _zone(self): + zone = None + if self._instance: + zone = self._instance.placement + return zone + + def _key_name(self): + kn = None + if self._instance: + kn = self._instance.key_name + return kn + + def put(self): + Model.put(self) + self._setup_ec2() + + def delete(self): + if self.production: + raise ValueError, "Can't delete a production server" + #self.stop() + Model.delete(self) + + def stop(self): + if self.production: + raise ValueError, "Can't delete a production server" + if self._instance: + self._instance.stop() + + def reboot(self): + if self._instance: + self._instance.reboot() + + def wait(self): + while self.status != 'running': + time.sleep(5) + + def get_ssh_key_file(self): + if not self.ssh_key_file: + ssh_dir = os.path.expanduser('~/.ssh') + if os.path.isdir(ssh_dir): + ssh_file = os.path.join(ssh_dir, '%s.pem' % self.key_name) + if os.path.isfile(ssh_file): + self.ssh_key_file = ssh_file + if not self.ssh_key_file: + iobject = IObject() + self.ssh_key_file = iobject.get_filename('Path to OpenSSH Key file') + return self.ssh_key_file + + def get_cmdshell(self): + if not self._cmdshell: + import cmdshell + self.get_ssh_key_file() + self._cmdshell = cmdshell.start(self) + return self._cmdshell + + def reset_cmdshell(self): + self._cmdshell = None + + def run(self, command): + with closing(self.get_cmdshell()) as cmd: + status = cmd.run(command) + return status + + def get_bundler(self, uname='root'): + ssh_key_file = self.get_ssh_key_file() + return Bundler(self, uname) + + def get_ssh_client(self, uname='root'): + from boto.manage.cmdshell import SSHClient + ssh_key_file = self.get_ssh_key_file() + return SSHClient(self, uname=uname) + + def install(self, pkg): + return self.run('apt-get -y install %s' % pkg) + + + diff --git a/api/boto/manage/task.py b/api/boto/manage/task.py new file mode 100644 index 0000000..5fb234d --- /dev/null +++ b/api/boto/manage/task.py @@ -0,0 +1,173 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto +from boto.sdb.db.property import * +from boto.sdb.db.model import Model +import datetime, subprocess, StringIO, time + +def check_hour(val): + if val == '*': + return + if int(val) < 0 or int(val) > 23: + raise ValueError + +class Task(Model): + + """ + A scheduled, repeating task that can be executed by any participating servers. + The scheduling is similar to cron jobs. Each task has an hour attribute. + The allowable values for hour are [0-23|*]. + + To keep the operation reasonably efficient and not cause excessive polling, + the minimum granularity of a Task is hourly. Some examples: + + hour='*' - the task would be executed each hour + hour='3' - the task would be executed at 3AM GMT each day. + + """ + name = StringProperty() + hour = StringProperty(required=True, validator=check_hour, default='*') + command = StringProperty(required=True) + last_executed = DateTimeProperty() + last_status = IntegerProperty() + last_output = StringProperty() + message_id = StringProperty() + + @classmethod + def start_all(cls, queue_name): + for task in cls.all(): + task.start(queue_name) + + def __init__(self, id=None, **kw): + Model.__init__(self, id, **kw) + self.hourly = self.hour == '*' + self.daily = self.hour != '*' + self.now = datetime.datetime.utcnow() + + def check(self): + """ + Determine how long until the next scheduled time for a Task. + Returns the number of seconds until the next scheduled time or zero + if the task needs to be run immediately. + If it's an hourly task and it's never been run, run it now. + If it's a daily task and it's never been run and the hour is right, run it now. + """ + need_to_run = False + boto.log.info('checking Task[%s]-now=%s, last=%s' % (self.name, self.now, self.last_executed)) + + if self.hourly and not self.last_executed: + return 0 + + if self.daily and not self.last_executed: + if int(self.hour) == self.now.hour: + return 0 + else: + return max((int(self.hour) - self.now.hour),0)*60*60 + + delta = self.now - self.last_executed + if self.hourly: + if delta.seconds >= 60*60: + return 0 + else: + return 60*60 - delta.seconds + else: + if delta.days >= 1: + return 0 + else: + return min(60*60*24-delta.seconds, 43200) + + def _run(self, msg, vtimeout): + boto.log.info('Task[%s] - running:%s' % (self.name, self.command)) + log_fp = StringIO.StringIO() + process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + nsecs = 5 + current_timeout = vtimeout + while process.poll() == None: + boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout)) + if nsecs >= current_timeout: + current_timeout += vtimeout + boto.log.info('Task[%s] - setting timeout to %d seconds' % (self.name, current_timeout)) + if msg: + msg.change_visibility(current_timeout) + time.sleep(5) + nsecs += 5 + t = process.communicate() + log_fp.write(t[0]) + log_fp.write(t[1]) + boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue())) + self.last_executed = self.now + self.last_status = process.returncode + self.last_output = log_fp.getvalue()[0:1023] + + def run(self, msg, vtimeout=60): + delay = self.check() + boto.log.info('Task[%s] - delay=%s seconds' % (self.name, delay)) + if delay == 0: + self._run(msg, vtimeout) + queue = msg.queue + new_msg = queue.new_message(self.id) + new_msg = queue.write(msg) + self.message_id = new_msg.id + self.put() + boto.log.info('Task[%s] - new message id=%s' % (self.name, new_msg.id)) + msg.delete() + boto.log.info('Task[%s] - deleted message %s' % (self.name, msg.id)) + else: + boto.log.info('new_vtimeout: %d' % delay) + msg.change_visibility(delay) + + def start(self, queue_name): + boto.log.info('Task[%s] - starting with queue: %s' % (self.name, queue_name)) + queue = boto.lookup('sqs', queue_name) + msg = queue.new_message(self.id) + msg = queue.write(msg) + self.message_id = msg.id + self.put() + boto.log.info('Task[%s] - start successful' % self.name) + +class TaskPoller(object): + + def __init__(self, queue_name): + self.sqs = boto.connect_sqs() + self.queue = self.sqs.lookup(queue_name) + + def poll(self, wait=60, vtimeout=60): + while 1: + m = self.queue.read(vtimeout) + if m: + task = Task.get_by_id(m.get_body()) + if task: + if not task.message_id or m.id == task.message_id: + boto.log.info('Task[%s] - read message %s' % (task.name, m.id)) + task.run(m, vtimeout) + else: + boto.log.info('Task[%s] - found extraneous message, ignoring' % task.name) + else: + time.sleep(wait) + + + + + + diff --git a/api/boto/manage/test_manage.py b/api/boto/manage/test_manage.py new file mode 100644 index 0000000..e0b032a --- /dev/null +++ b/api/boto/manage/test_manage.py @@ -0,0 +1,34 @@ +from boto.manage.server import Server +from boto.manage.volume import Volume +import time + +print '--> Creating New Volume' +volume = Volume.create() +print volume + +print '--> Creating New Server' +server_list = Server.create() +server = server_list[0] +print server + +print '----> Waiting for Server to start up' +while server.status != 'running': + print '*' + time.sleep(10) +print '----> Server is running' + +print '--> Run "df -k" on Server' +status = server.run('df -k') +print status[1] + +print '--> Now run volume.make_ready to make the volume ready to use on server' +volume.make_ready(server) + +print '--> Run "df -k" on Server' +status = server.run('df -k') +print status[1] + +print '--> Do an "ls -al" on the new filesystem' +status = server.run('ls -al %s' % volume.mount_point) +print status[1] + diff --git a/api/boto/manage/volume.py b/api/boto/manage/volume.py new file mode 100644 index 0000000..bed5594 --- /dev/null +++ b/api/boto/manage/volume.py @@ -0,0 +1,417 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from __future__ import with_statement +from boto.sdb.db.model import Model +from boto.sdb.db.property import * +from boto.manage.server import Server +from boto.manage import propget +import boto.ec2 +import time, traceback +from contextlib import closing +import dateutil.parser + +class CommandLineGetter(object): + + def get_region(self, params): + if not params.get('region', None): + prop = self.cls.find_property('region_name') + params['region'] = propget.get(prop, choices=boto.ec2.regions) + + def get_zone(self, params): + if not params.get('zone', None): + prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', + choices=self.ec2.get_all_zones) + params['zone'] = propget.get(prop) + + def get_name(self, params): + if not params.get('name', None): + prop = self.cls.find_property('name') + params['name'] = propget.get(prop) + + def get_size(self, params): + if not params.get('size', None): + prop = IntegerProperty(name='size', verbose_name='Size (GB)') + params['size'] = propget.get(prop) + + def get_mount_point(self, params): + if not params.get('mount_point', None): + prop = self.cls.find_property('mount_point') + params['mount_point'] = propget.get(prop) + + def get_device(self, params): + if not params.get('device', None): + prop = self.cls.find_property('device') + params['device'] = propget.get(prop) + + def get(self, cls, params): + self.cls = cls + self.get_region(params) + self.ec2 = params['region'].connect() + self.get_zone(params) + self.get_name(params) + self.get_size(params) + self.get_mount_point(params) + self.get_device(params) + +class Volume(Model): + + name = StringProperty(required=True, unique=True, verbose_name='Name') + region_name = StringProperty(required=True, verbose_name='EC2 Region') + zone_name = StringProperty(required=True, verbose_name='EC2 Zone') + mount_point = StringProperty(verbose_name='Mount Point') + device = StringProperty(verbose_name="Device Name", default='/dev/sdp') + volume_id = StringProperty(required=True) + past_volume_ids = ListProperty(item_type=str) + server = ReferenceProperty(Server, collection_name='volumes', + verbose_name='Server Attached To') + volume_state = CalculatedProperty(verbose_name="Volume State", + calculated_type=str, use_method=True) + attachment_state = CalculatedProperty(verbose_name="Attachment State", + calculated_type=str, use_method=True) + size = CalculatedProperty(verbose_name="Size (GB)", + calculated_type=int, use_method=True) + + @classmethod + def create(cls, **params): + getter = CommandLineGetter() + getter.get(cls, params) + region = params.get('region') + ec2 = region.connect() + zone = params.get('zone') + size = params.get('size') + ebs_volume = ec2.create_volume(size, zone.name) + v = cls() + v.ec2 = ec2 + v.volume_id = ebs_volume.id + v.name = params.get('name') + v.mount_point = params.get('mount_point') + v.device = params.get('device') + v.region_name = region.name + v.zone_name = zone.name + v.put() + return v + + @classmethod + def create_from_volume_id(cls, region_name, volume_id, name): + vol = None + ec2 = boto.ec2.connect_to_region(region_name) + rs = ec2.get_all_volumes([volume_id]) + if len(rs) == 1: + v = rs[0] + vol = cls() + vol.volume_id = v.id + vol.name = name + vol.region_name = v.region.name + vol.zone_name = v.zone + vol.put() + return vol + + def create_from_latest_snapshot(self, name, size=None): + snapshot = self.get_snapshots()[-1] + return self.create_from_snapshot(name, snapshot, size) + + def create_from_snapshot(self, name, snapshot, size=None): + if size < self.size: + size = self.size + ec2 = self.get_ec2_connection() + if self.zone_name == None or self.zone_name == '': + # deal with the migration case where the zone is not set in the logical volume: + current_volume = ec2.get_all_volumes([self.volume_id])[0] + self.zone_name = current_volume.zone + ebs_volume = ec2.create_volume(size, self.zone_name, snapshot) + v = Volume() + v.ec2 = self.ec2 + v.volume_id = ebs_volume.id + v.name = name + v.mount_point = self.mount_point + v.device = self.device + v.region_name = self.region_name + v.zone_name = self.zone_name + v.put() + return v + + def get_ec2_connection(self): + if self.server: + return self.server.ec2 + if not hasattr(self, 'ec2') or self.ec2 == None: + self.ec2 = boto.ec2.connect_to_region(self.region_name) + return self.ec2 + + def _volume_state(self): + ec2 = self.get_ec2_connection() + rs = ec2.get_all_volumes([self.volume_id]) + return rs[0].volume_state() + + def _attachment_state(self): + ec2 = self.get_ec2_connection() + rs = ec2.get_all_volumes([self.volume_id]) + return rs[0].attachment_state() + + def _size(self): + if not hasattr(self, '__size'): + ec2 = self.get_ec2_connection() + rs = ec2.get_all_volumes([self.volume_id]) + self.__size = rs[0].size + return self.__size + + def install_xfs(self): + if self.server: + self.server.install('xfsprogs xfsdump') + + def get_snapshots(self): + """ + Returns a list of all completed snapshots for this volume ID. + """ + ec2 = self.get_ec2_connection() + rs = ec2.get_all_snapshots() + all_vols = [self.volume_id] + self.past_volume_ids + snaps = [] + for snapshot in rs: + if snapshot.volume_id in all_vols: + if snapshot.progress == '100%': + snapshot.date = dateutil.parser.parse(snapshot.start_time) + snapshot.keep = True + snaps.append(snapshot) + snaps.sort(cmp=lambda x,y: cmp(x.date, y.date)) + return snaps + + def attach(self, server=None): + if self.attachment_state == 'attached': + print 'already attached' + return None + if server: + self.server = server + self.put() + ec2 = self.get_ec2_connection() + ec2.attach_volume(self.volume_id, self.server.instance_id, self.device) + + def detach(self, force=False): + state = self.attachment_state + if state == 'available' or state == None or state == 'detaching': + print 'already detached' + return None + ec2 = self.get_ec2_connection() + ec2.detach_volume(self.volume_id, self.server.instance_id, self.device, force) + self.server = None + self.put() + + def checkfs(self, use_cmd=None): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + # detemine state of file system on volume, only works if attached + if use_cmd: + cmd = use_cmd + else: + cmd = self.server.get_cmdshell() + status = cmd.run('xfs_check %s' % self.device) + if not use_cmd: + cmd.close() + if status[1].startswith('bad superblock magic number 0'): + return False + return True + + def wait(self): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + with closing(self.server.get_cmdshell()) as cmd: + # wait for the volume device to appear + cmd = self.server.get_cmdshell() + while not cmd.exists(self.device): + boto.log.info('%s still does not exist, waiting 10 seconds' % self.device) + time.sleep(10) + + def format(self): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + status = None + with closing(self.server.get_cmdshell()) as cmd: + if not self.checkfs(cmd): + boto.log.info('make_fs...') + status = cmd.run('mkfs -t xfs %s' % self.device) + return status + + def mount(self): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + boto.log.info('handle_mount_point') + with closing(self.server.get_cmdshell()) as cmd: + cmd = self.server.get_cmdshell() + if not cmd.isdir(self.mount_point): + boto.log.info('making directory') + # mount directory doesn't exist so create it + cmd.run("mkdir %s" % self.mount_point) + else: + boto.log.info('directory exists already') + status = cmd.run('mount -l') + lines = status[1].split('\n') + for line in lines: + t = line.split() + if t and t[2] == self.mount_point: + # something is already mounted at the mount point + # unmount that and mount it as /tmp + if t[0] != self.device: + cmd.run('umount %s' % self.mount_point) + cmd.run('mount %s /tmp' % t[0]) + cmd.run('chmod 777 /tmp') + break + # Mount up our new EBS volume onto mount_point + cmd.run("mount %s %s" % (self.device, self.mount_point)) + cmd.run('xfs_growfs %s' % self.mount_point) + + def make_ready(self, server): + self.server = server + self.put() + self.install_xfs() + self.attach() + self.wait() + self.format() + self.mount() + + def freeze(self): + if self.server: + return self.server.run("/usr/sbin/xfs_freeze -f %s" % self.mount_point) + + def unfreeze(self): + if self.server: + return self.server.run("/usr/sbin/xfs_freeze -u %s" % self.mount_point) + + def snapshot(self): + # if this volume is attached to a server + # we need to freeze the XFS file system + try: + self.freeze() + if self.server == None: + snapshot = self.get_ec2_connection().create_snapshot(self.volume_id) + else: + snapshot = self.server.ec2.create_snapshot(self.volume_id) + boto.log.info('Snapshot of Volume %s created: %s' % (self.name, snapshot)) + except Exception, e: + boto.log.info('Snapshot error') + boto.log.info(traceback.format_exc()) + finally: + status = self.unfreeze() + return status + + def get_snapshot_range(self, snaps, start_date=None, end_date=None): + l = [] + for snap in snaps: + if start_date and end_date: + if snap.date >= start_date and snap.date <= end_date: + l.append(snap) + elif start_date: + if snap.date >= start_date: + l.append(snap) + elif end_date: + if snap.date <= end_date: + l.append(snap) + else: + l.append(snap) + return l + + def trim_snapshots(self, delete=False): + """ + Trim the number of snapshots for this volume. This method always + keeps the oldest snapshot. It then uses the parameters passed in + to determine how many others should be kept. + + The algorithm is to keep all snapshots from the current day. Then + it will keep the first snapshot of the day for the previous seven days. + Then, it will keep the first snapshot of the week for the previous + four weeks. After than, it will keep the first snapshot of the month + for as many months as there are. + + """ + snaps = self.get_snapshots() + # Always keep the oldest and the newest + if len(snaps) <= 2: + return snaps + snaps = snaps[1:-1] + now = datetime.datetime.now(snaps[0].date.tzinfo) + midnight = datetime.datetime(year=now.year, month=now.month, + day=now.day, tzinfo=now.tzinfo) + # Keep the first snapshot from each day of the previous week + one_week = datetime.timedelta(days=7, seconds=60*60) + print midnight-one_week, midnight + previous_week = self.get_snapshot_range(snaps, midnight-one_week, midnight) + print previous_week + if not previous_week: + return snaps + current_day = None + for snap in previous_week: + if current_day and current_day == snap.date.day: + snap.keep = False + else: + current_day = snap.date.day + # Get ourselves onto the next full week boundary + if previous_week: + week_boundary = previous_week[0].date + if week_boundary.weekday() != 0: + delta = datetime.timedelta(days=week_boundary.weekday()) + week_boundary = week_boundary - delta + # Keep one within this partial week + partial_week = self.get_snapshot_range(snaps, week_boundary, previous_week[0].date) + if len(partial_week) > 1: + for snap in partial_week[1:]: + snap.keep = False + # Keep the first snapshot of each week for the previous 4 weeks + for i in range(0,4): + weeks_worth = self.get_snapshot_range(snaps, week_boundary-one_week, week_boundary) + if len(weeks_worth) > 1: + for snap in weeks_worth[1:]: + snap.keep = False + week_boundary = week_boundary - one_week + # Now look through all remaining snaps and keep one per month + remainder = self.get_snapshot_range(snaps, end_date=week_boundary) + current_month = None + for snap in remainder: + if current_month and current_month == snap.date.month: + snap.keep = False + else: + current_month = snap.date.month + if delete: + for snap in snaps: + if not snap.keep: + boto.log.info('Deleting %s(%s) for %s' % (snap, snap.date, self.name)) + snap.delete() + return snaps + + def grow(self, size): + pass + + def copy(self, snapshot): + pass + + def get_snapshot_from_date(self, date): + pass + + def delete(self, delete_ebs_volume=False): + if delete_ebs_volume: + self.detach() + ec2 = self.get_ec2_connection() + ec2.delete_volume(self.volume_id) + Model.delete(self) + + def archive(self): + # snapshot volume, trim snaps, delete volume-id + pass + + diff --git a/api/boto/mapreduce/__init__.py b/api/boto/mapreduce/__init__.py new file mode 100644 index 0000000..ac3ddc4 --- /dev/null +++ b/api/boto/mapreduce/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/api/boto/mapreduce/lqs.py b/api/boto/mapreduce/lqs.py new file mode 100644 index 0000000..fc76e50 --- /dev/null +++ b/api/boto/mapreduce/lqs.py @@ -0,0 +1,152 @@ +import SocketServer, os, datetime, sys, random, time +import simplejson + +class LQSCommand: + + def __init__(self, line): + self.raw_line = line + self.line = self.raw_line.strip() + l = self.line.split(' ') + self.name = l[0] + if len(l) > 1: + self.args = [arg for arg in l[1:] if arg] + else: + self.args = [] + +class LQSMessage(dict): + + def __init__(self, item=None, args=None, jsonvalue=None): + dict.__init__(self) + if jsonvalue: + self.decode(jsonvalue) + else: + self['id'] = '%d_%d' % (int(time.time()), int(random.random()*1000000)) + self['item'] = item + self['args'] = args + + def encode(self): + return simplejson.dumps(self) + + def decode(self, value): + self.update(simplejson.loads(value)) + + def is_empty(self): + if self['item'] == None: + return True + return False + +class LQSServer(SocketServer.UDPServer): + + PORT = 5151 + TIMEOUT = 30 + MAXSIZE = 8192 + + def __init__(self, server_address, RequestHandlerClass, iterator, args=None): + server_address = (server_address, self.PORT) + SocketServer.UDPServer.__init__(self, server_address, RequestHandlerClass) + self.count = 0 + self.iterator = iterator + self.args = args + self.start = datetime.datetime.now() + self.end = None + self.extant = [] + +class LQSHandler(SocketServer.DatagramRequestHandler): + + def get_cmd(self): + return LQSCommand(self.rfile.readline()) + + def build_msg(self): + if not self.server.iterator: + return LQSMessage(None) + try: + item = self.server.iterator.next() + msg = LQSMessage(item, self.server.args) + return msg + except StopIteration: + self.server.iterator = None + return LQSMessage(None) + + def respond(self, msg): + self.wfile.write(msg.encode()) + + def check_extant(self): + if len(self.server.extant) == 0 and not self.server.iterator: + self.server.end = datetime.datetime.now() + delta = self.server.end - self.server.start + print 'Total Processing Time: %s' % delta + print 'Total Messages Processed: %d' % self.server.count + + def do_debug(self, cmd): + args = {'extant' : self.server.extant, + 'count' : self.server.count} + msg = LQSMessage('debug', args) + self.respond(msg) + + def do_next(self, cmd): + out_msg = self.build_msg() + if not out_msg.is_empty(): + self.server.count += 1 + self.server.extant.append(out_msg['id']) + self.respond(out_msg) + + def do_delete(self, cmd): + if len(cmd.args) != 1: + self.error(cmd, 'delete command requires message id') + else: + mid = cmd.args[0] + try: + self.server.extant.remove(mid) + except ValueError: + self.error(cmd, 'message id not found') + args = {'deleted' : True} + msg = LQSMessage(mid, args) + self.respond(msg) + self.check_extant() + + def error(self, cmd, error_msg=None): + args = {'error_msg' : error_msg, + 'cmd_name' : cmd.name, + 'cmd_args' : cmd.args} + msg = LQSMessage('error', args) + self.respond(msg) + + def do_stop(self, cmd): + sys.exit(0) + + def handle(self): + cmd = self.get_cmd() + if hasattr(self, 'do_%s' % cmd.name): + method = getattr(self, 'do_%s' % cmd.name) + method(cmd) + else: + self.error(cmd, 'unrecognized command') + +class PersistHandler(LQSHandler): + + def build_msg(self): + if not self.server.iterator: + return LQSMessage(None) + try: + obj = self.server.iterator.next() + msg = LQSMessage(obj.id, self.server.args) + return msg + except StopIteration: + self.server.iterator = None + return LQSMessage(None) + +def test_file(path, args=None): + l = os.listdir(path) + if not args: + args = {} + args['path'] = path + s = LQSServer('', LQSHandler, iter(l), args) + print "Awaiting UDP messages on port %d" % s.PORT + s.serve_forever() + +def test_simple(n): + l = range(0, n) + s = LQSServer('', LQSHandler, iter(l), None) + print "Awaiting UDP messages on port %d" % s.PORT + s.serve_forever() + diff --git a/api/boto/mapreduce/partitiondb.py b/api/boto/mapreduce/partitiondb.py new file mode 100644 index 0000000..c5b0475 --- /dev/null +++ b/api/boto/mapreduce/partitiondb.py @@ -0,0 +1,172 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import random, time, os, datetime +import boto +from boto.sdb.persist.object import SDBObject +from boto.sdb.persist.property import * + +class Identifier(object): + + _hex_digits = '0123456789abcdef' + + @classmethod + def gen(cls, prefix): + suffix = '' + for i in range(0,8): + suffix += random.choice(cls._hex_digits) + return ts + '-' + suffix + +class Version(SDBObject): + + name = StringProperty() + pdb = ObjectProperty(ref_class=SDBObject) + date = DateTimeProperty() + + def __init__(self, id=None, manager=None): + SDBObject.__init__(self, id, manager) + if id == None: + self.name = Identifier.gen('v') + self.date = datetime.datetime.now() + print 'created Version %s' % self.name + + def partitions(self): + """ + Return an iterator containing all Partition objects related to this Version. + + :rtype: iterator of :class:`boto.mapreduce.partitiondb.Partition` + :return: The Partitions in this Version + """ + return self.get_related_objects('version', Partition) + + def add_partition(self, name=None): + """ + Add a new Partition to this Version. + + :type name: string + :param name: The name of the new Partition (optional) + + :rtype: :class:`boto.mapreduce.partitiondb.Partition` + :return: The new Partition object + """ + p = Partition(manager=self.manager, name=name) + p.version = self + p.pdb = self.pdb + p.save() + return p + + def get_s3_prefix(self): + if not self.pdb: + raise ValueError, 'pdb attribute must be set to compute S3 prefix' + return self.pdb.get_s3_prefix() + self.name + '/' + +class PartitionDB(SDBObject): + + name = StringProperty() + bucket_name = StringProperty() + versions = ObjectListProperty(ref_class=Version) + + def __init__(self, id=None, manager=None, name='', bucket_name=''): + SDBObject.__init__(self, id, manager) + if id == None: + self.name = name + self.bucket_name = bucket_name + + def get_s3_prefix(self): + return self.name + '/' + + def add_version(self): + """ + Add a new Version to this PartitionDB. The newly added version becomes the + current version. + + :rtype: :class:`boto.mapreduce.partitiondb.Version` + :return: The newly created Version object. + """ + v = Version() + v.pdb = self + v.save() + self.versions.append(v) + return v + + def revert(self): + """ + Revert to the previous version of this PartitionDB. The current version is removed from the + list of Versions and the Version immediately preceeding it becomes the current version. + Note that this method does not delete the Version object or any Partitions related to the + Version object. + + :rtype: :class:`boto.mapreduce.partitiondb.Version` + :return: The previous current Version object. + """ + v = self.current_version() + if v: + self.versions.remove(v) + return v + + def current_version(self): + """ + Get the currently active Version of this PartitionDB object. + + :rtype: :class:`boto.mapreduce.partitiondb.Version` + :return: The current Version object or None if there are no Versions associated + with this PartitionDB object. + """ + if self.versions: + if len(self.versions) > 0: + return self.versions[-1] + return None + +class Partition(SDBObject): + + def __init__(self, id=None, manager=None, name=None): + SDBObject.__init__(self, id, manager) + if id == None: + self.name = name + + name = StringProperty() + version = ObjectProperty(ref_class=Version) + pdb = ObjectProperty(ref_class=PartitionDB) + data = S3KeyProperty() + + def get_key_name(self): + return self.version.get_s3_prefix() + self.name + + def upload(self, path, bucket_name=None): + if not bucket_name: + bucket_name = self.version.pdb.bucket_name + s3 = self.manager.get_s3_connection() + bucket = s3.lookup(bucket_name) + directory, filename = os.path.split(path) + self.name = filename + key = bucket.new_key(self.get_key_name()) + key.set_contents_from_filename(path) + self.data = key + self.save() + + def delete(self): + if self.data: + self.data.delete() + SDBObject.delete(self) + + + diff --git a/api/boto/mapreduce/pdb_delete b/api/boto/mapreduce/pdb_delete new file mode 100644 index 0000000..b7af9cc --- /dev/null +++ b/api/boto/mapreduce/pdb_delete @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import queuetools, os, signal, sys +import subprocess +import time +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from lqs import LQSServer, PersistHandler +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager, revive_object_from_id + +USAGE = """ + SYNOPSIS + %prog [options] [command] + DESCRIPTION + Delete a PartitionDB and all related data in SimpleDB and S3. +""" +class Client: + + def __init__(self, queue_name): + self.q = queuetools.get_queue(queue_name) + self.q.connect() + self.manager = get_manager() + self.process() + + def process(self): + m = self.q.get() + while m['item']: + print 'Deleting: %s' % m['item'] + obj = revive_object_from_id(m['item'], manager=self.manager) + obj.delete() + self.q.delete(m) + m = self.q.get() + print 'client processing complete' + +class Server: + + def __init__(self, pdb_name, domain_name=None): + self.pdb_name = pdb_name + self.manager = get_manager(domain_name) + self.pdb = PartitionDB.get(name=self.pdb_name) + self.serve() + + def serve(self): + args = {'pdb_id' : self.pdb.id} + rs = self.pdb.get_related_objects('pdb') + self.pdb.delete() + s = LQSServer('', PersistHandler, rs, args) + s.serve_forever() + +class Delete: + + Commands = {'client' : 'Start a Delete client', + 'server' : 'Start a Delete server'} + + def __init__(self): + self.parser = OptionParser(usage=USAGE) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-processes', action='store', type='int', dest='num_processes', + help='the number of client processes launched') + self.parser.set_defaults(num_processes=5) + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB in which to store files (will create if necessary)') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_server(self): + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + s = Server(self.options.pdb_name, self.options.domain_name) + + def do_client(self): + c = Client('localhost') + + def main(self): + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) == 0: + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + server_command = '%s -p %s ' % (self.prog_name, self.options.pdb_name) + server_command += ' server' + client_command = '%s client' % self.prog_name + server = subprocess.Popen(server_command, shell=True) + print 'server pid: %s' % server.pid + time.sleep(5) + clients = [] + for i in range(0, self.options.num_processes): + client = subprocess.Popen(client_command, shell=True) + clients.append(client) + print 'waiting for clients to finish' + for client in clients: + client.wait() + os.kill(server.pid, signal.SIGTERM) + elif len(self.args) == 1: + self.command = self.args[0] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + else: + self.parser.error('unrecognized commands') + +if __name__ == "__main__": + delete = Delete() + delete.main() diff --git a/api/boto/mapreduce/pdb_describe b/api/boto/mapreduce/pdb_describe new file mode 100755 index 0000000..d0fa86c --- /dev/null +++ b/api/boto/mapreduce/pdb_describe @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import sys +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager, get_domain + +USAGE = """ + SYNOPSIS + %prog [options] + DESCRIPTION + List and describe your PartitionDBs. + Called with no options, all PartitionDB objects defined in your default + domain (as specified in the "default_domain" option in the "[Persist]" + section of your boto config file) will be listed. + When called with a particular PartitionDB name (using -p option) all + Version objects of that PartitionDB object will be listed. + When called with the -p option and a particular Version name specified + (using the -v option) all Partitions in that Version object will be listed. +""" +class Describe: + + def __init__(self): + self.parser = OptionParser(usage=USAGE) + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-entries', action='store', type='int', + help='maximum number of entries to print (default 100)') + self.parser.set_defaults(num_entries=100) + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB to describe') + self.parser.add_option('-v', '--version-name', action='store', type='string', + help='name of the PDB Version to describe') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def describe_all(self): + print 'Using SimpleDB Domain: %s' % get_domain() + print 'PDBs:' + rs = PartitionDB.list() + i = 0 + for pdb in rs: + print '%s\t%s\t%s' % (pdb.id, pdb.name, pdb.bucket_name) + i += 1 + if i == self.options.num_entries: + break + + def describe_pdb(self, pdb_name): + print 'Using SimpleDB Domain: %s' % get_domain() + print 'PDB: %s' % pdb_name + print 'Versions:' + try: + pdb = PartitionDB.get(name=pdb_name) + i = 0 + for v in pdb.versions: + if v.date: + ds = v.date.isoformat() + else: + ds = 'unknown' + print '%s\t%s\t%s' % (v.id, v.name, ds) + i += 1 + if i == self.options.num_entries: + break + cv = pdb.current_version() + if cv: + print 'Current Version: %s' % cv.name + else: + print 'Current Version: None' + except SDBPersistenceError: + self.parser.error('pdb_name (%s) unknown' % pdb_name) + + def describe_version(self, pdb_name, version_name): + print 'Using SimpleDB Domain: %s' % get_domain() + print 'PDB: %s' % pdb_name + print 'Version: %s' % version_name + print 'Partitions:' + try: + pdb = PartitionDB.get(name=pdb_name) + for v in pdb.versions: + if v.name == version_name: + i = 0 + for p in v.partitions(): + print '%s\t%s' % (p.id, p.name) + i += 1 + if i == self.options.num_entries: + break + except SDBPersistenceError: + self.parser.error('pdb_name (%s) unknown' % pdb_name) + + def main(self): + self.options, self.args = self.parser.parse_args() + self.manager = get_manager(self.options.domain_name) + + if self.options.pdb_name: + if self.options.version_name: + self.describe_version(self.options.pdb_name, self.options.version_name) + else: + self.describe_pdb(self.options.pdb_name) + else: + self.describe_all() + +if __name__ == "__main__": + describe = Describe() + describe.main() diff --git a/api/boto/mapreduce/pdb_revert b/api/boto/mapreduce/pdb_revert new file mode 100755 index 0000000..daffeef --- /dev/null +++ b/api/boto/mapreduce/pdb_revert @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import queuetools, os, signal, sys +import subprocess +import time +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from lqs import LQSServer, PersistHandler +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager + +USAGE = """ + SYNOPSIS + %prog [options] [command] + DESCRIPTION + Revert to the previous Version in a PartitionDB. +""" +class Client: + + def __init__(self, queue_name): + self.q = queuetools.get_queue(queue_name) + self.q.connect() + self.manager = get_manager() + self.process() + + def process(self): + m = self.q.get() + while m['item']: + print 'Deleting: %s' % m['item'] + p = Partition(id=m['item'], manager=self.manager) + p.delete() + self.q.delete(m) + m = self.q.get() + print 'client processing complete' + +class Server: + + def __init__(self, pdb_name, domain_name=None): + self.pdb_name = pdb_name + self.manager = get_manager(domain_name) + self.pdb = PartitionDB.get(name=self.pdb_name) + self.serve() + + def serve(self): + v = self.pdb.revert() + args = {'v_id' : v.id} + rs = v.partitions() + s = LQSServer('', PersistHandler, rs, args) + s.serve_forever() + +class Revert: + + Commands = {'client' : 'Start a Revert client', + 'server' : 'Start a Revert server'} + + def __init__(self): + self.parser = OptionParser(usage=USAGE) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-processes', action='store', type='int', dest='num_processes', + help='the number of client processes launched') + self.parser.set_defaults(num_processes=5) + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB in which to store files (will create if necessary)') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_server(self): + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + s = Server(self.options.pdb_name, self.options.domain_name) + + def do_client(self): + c = Client('localhost') + + def main(self): + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) == 0: + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + server_command = '%s -p %s ' % (self.prog_name, self.options.pdb_name) + server_command += ' server' + client_command = '%s client' % self.prog_name + server = subprocess.Popen(server_command, shell=True) + print 'server pid: %s' % server.pid + time.sleep(5) + clients = [] + for i in range(0, self.options.num_processes): + client = subprocess.Popen(client_command, shell=True) + clients.append(client) + print 'waiting for clients to finish' + for client in clients: + client.wait() + os.kill(server.pid, signal.SIGTERM) + elif len(self.args) == 1: + self.command = self.args[0] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + else: + self.parser.error('unrecognized commands') + +if __name__ == "__main__": + revert = Revert() + revert.main() diff --git a/api/boto/mapreduce/pdb_upload b/api/boto/mapreduce/pdb_upload new file mode 100755 index 0000000..1ca2b6d --- /dev/null +++ b/api/boto/mapreduce/pdb_upload @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import queuetools, os, signal, sys +import subprocess +import time +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from lqs import LQSServer, LQSHandler +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager + +USAGE = """ + SYNOPSIS + %prog [options] + DESCRIPTION + Upload partition files to a PartitionDB. + Called with no options, all PartitionDB objects defined in your default + domain (as specified in the "default_domain" option in the "[Persist]" + section of your boto config file) will be listed. + When called with a particular PartitionDB name (using -p option) all + Version objects of that PartitionDB object will be listed. + When called with the -p option and a particular Version name specified + (using the -v option) all Partitions in that Version object will be listed. +""" +class Client: + + def __init__(self, queue_name): + self.q = queuetools.get_queue(queue_name) + self.q.connect() + self.manager = get_manager() + self.process() + + def process(self): + m = self.q.get() + if m['item']: + v = Version(m['args']['v_id'], self.manager) + bucket_name = v.pdb.bucket_name + while m['item']: + print 'Uploading: %s' % m['item'] + p = v.add_partition(name=m['item']) + p.upload(os.path.join(m['args']['path'], m['item']), bucket_name) + self.q.delete(m) + m = self.q.get() + print 'client processing complete' + +class Server: + + def __init__(self, path, pdb_name, bucket_name=None, domain_name=None): + self.path = path + self.pdb_name = pdb_name + self.bucket_name = bucket_name + self.manager = get_manager(domain_name) + self.get_pdb() + self.serve() + + def get_pdb(self): + try: + self.pdb = PartitionDB.get(name=self.pdb_name) + except SDBPersistenceError: + self.pdb = PartitionDB(manager=self.manager, name=self.pdb_name, bucket_name=self.bucket_name) + self.pdb.save() + + def serve(self): + v = self.pdb.add_version() + args = {'path' : self.path, + 'v_id' : v.id} + l = os.listdir(self.path) + s = LQSServer('', LQSHandler, iter(l), args) + s.serve_forever() + +class Upload: + + Usage = "usage: %prog [options] command" + + Commands = {'client' : 'Start an Upload client', + 'server' : 'Start an Upload server'} + + def __init__(self): + self.parser = OptionParser(usage=self.Usage) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-processes', action='store', type='int', dest='num_processes', + help='the number of client processes launched') + self.parser.set_defaults(num_processes=2) + self.parser.add_option('-i', '--input-path', action='store', type='string', + help='the path to directory to upload') + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB in which to store files (will create if necessary)') + self.parser.add_option('-b', '--bucket-name', action='store', type='string', + help='name of S3 bucket (only needed if creating new PDB)') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_server(self): + if not self.options.input_path: + self.parser.error('No path provided') + if not os.path.isdir(self.options.input_path): + self.parser.error('Invalid path (%s)' % self.options.input_path) + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + s = Server(self.options.input_path, self.options.pdb_name, + self.options.bucket_name, self.options.domain_name) + + def do_client(self): + c = Client('localhost') + + def main(self): + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) == 0: + if not self.options.input_path: + self.parser.error('No path provided') + if not os.path.isdir(self.options.input_path): + self.parser.error('Invalid path (%s)' % self.options.input_path) + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + server_command = '%s -p %s -i %s' % (self.prog_name, self.options.pdb_name, self.options.input_path) + if self.options.bucket_name: + server_command += ' -b %s' % self.options.bucket_name + server_command += ' server' + client_command = '%s client' % self.prog_name + server = subprocess.Popen(server_command, shell=True) + print 'server pid: %s' % server.pid + time.sleep(5) + clients = [] + for i in range(0, self.options.num_processes): + client = subprocess.Popen(client_command, shell=True) + clients.append(client) + print 'waiting for clients to finish' + for client in clients: + client.wait() + os.kill(server.pid, signal.SIGTERM) + elif len(self.args) == 1: + self.command = self.args[0] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + else: + self.parser.error('unrecognized commands') + +if __name__ == "__main__": + upload = Upload() + upload.main() diff --git a/api/boto/mapreduce/queuetools.py b/api/boto/mapreduce/queuetools.py new file mode 100644 index 0000000..3e08a10 --- /dev/null +++ b/api/boto/mapreduce/queuetools.py @@ -0,0 +1,66 @@ +#!/usr/bin/python +import socket, sys +from lqs import LQSServer, LQSMessage +import boto +from boto.sqs.jsonmessage import JSONMessage + +class LQSClient: + + def __init__(self, host): + self.host = host + self.port = LQSServer.PORT + self.timeout = LQSServer.TIMEOUT + self.max_len = LQSServer.MAXSIZE + self.sock = None + + def connect(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.sock.settimeout(self.timeout) + self.sock.connect((self.host, self.port)) + + def decode(self, jsonstr): + return LQSMessage(jsonvalue=jsonstr) + + def get(self): + self.sock.send('next') + try: + jsonstr = self.sock.recv(self.max_len) + msg = LQSMessage(jsonvalue=jsonstr) + return msg + except: + print "recv from %s failed" % self.host + + def delete(self, msg): + self.sock.send('delete %s' % msg['id']) + try: + jsonstr = self.sock.recv(self.max_len) + msg = LQSMessage(jsonvalue=jsonstr) + return msg + except: + print "recv from %s failed" % self.host + + def close(self): + self.sock.close() + +class SQSClient: + + def __init__(self, queue_name): + self.queue_name = queue_name + + def connect(self): + self.queue = boto.lookup('sqs', self.queue_name) + self.queue.set_mesasge_class(JSONMessage) + + def get(self): + m = self.queue.read() + return m.get_body() + + def close(self): + pass + +def get_queue(name): + if name == 'localhost': + return LQSClient(name) + else: + return SQSClient(name) + diff --git a/api/boto/mashups/__init__.py b/api/boto/mashups/__init__.py new file mode 100644 index 0000000..449bd16 --- /dev/null +++ b/api/boto/mashups/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/api/boto/mashups/iobject.py b/api/boto/mashups/iobject.py new file mode 100644 index 0000000..a226b5c --- /dev/null +++ b/api/boto/mashups/iobject.py @@ -0,0 +1,115 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import os + +def int_val_fn(v): + try: + int(v) + return True + except: + return False + +class IObject(object): + + def choose_from_list(self, item_list, search_str='', + prompt='Enter Selection'): + if not item_list: + print 'No Choices Available' + return + choice = None + while not choice: + n = 1 + choices = [] + for item in item_list: + if isinstance(item, str): + print '[%d] %s' % (n, item) + choices.append(item) + n += 1 + else: + obj, id, desc = item + if desc: + if desc.find(search_str) >= 0: + print '[%d] %s - %s' % (n, id, desc) + choices.append(obj) + n += 1 + else: + if id.find(search_str) >= 0: + print '[%d] %s' % (n, id) + choices.append(obj) + n += 1 + if choices: + val = raw_input('%s[1-%d]: ' % (prompt, len(choices))) + if val.startswith('/'): + search_str = val[1:] + else: + try: + int_val = int(val) + if int_val == 0: + return None + choice = choices[int_val-1] + except ValueError: + print '%s is not a valid choice' % val + except IndexError: + print '%s is not within the range[1-%d]' % (val, + len(choices)) + else: + print "No objects matched your pattern" + search_str = '' + return choice + + def get_string(self, prompt, validation_fn=None): + okay = False + while not okay: + val = raw_input('%s: ' % prompt) + if validation_fn: + okay = validation_fn(val) + if not okay: + print 'Invalid value: %s' % val + else: + okay = True + return val + + def get_filename(self, prompt): + okay = False + val = '' + while not okay: + val = raw_input('%s: %s' % (prompt, val)) + val = os.path.expanduser(val) + if os.path.isfile(val): + okay = True + elif os.path.isdir(val): + path = val + val = self.choose_from_list(os.listdir(path)) + if val: + val = os.path.join(path, val) + okay = True + else: + val = '' + else: + print 'Invalid value: %s' % val + val = '' + return val + + def get_int(self, prompt): + s = self.get_string(prompt, int_val_fn) + return int(s) + diff --git a/api/boto/mashups/order.py b/api/boto/mashups/order.py new file mode 100644 index 0000000..6efdc3e --- /dev/null +++ b/api/boto/mashups/order.py @@ -0,0 +1,211 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +High-level abstraction of an EC2 order for servers +""" + +import boto +import boto.ec2 +from boto.mashups.server import Server, ServerSet +from boto.mashups.iobject import IObject +from boto.pyami.config import Config +from boto.sdb.persist import get_domain, set_domain +import time, StringIO + +InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', 'c1.medium', 'c1.xlarge'] + +class Item(IObject): + + def __init__(self): + self.region = None + self.name = None + self.instance_type = None + self.quantity = 0 + self.zone = None + self.ami = None + self.groups = [] + self.key = None + self.ec2 = None + self.config = None + + def set_userdata(self, key, value): + self.userdata[key] = value + + def get_userdata(self, key): + return self.userdata[key] + + def set_region(self, region=None): + if region: + self.region = region + else: + l = [(r, r.name, r.endpoint) for r in boto.ec2.regions()] + self.region = self.choose_from_list(l, prompt='Choose Region') + + def set_name(self, name=None): + if name: + self.name = name + else: + self.name = self.get_string('Name') + + def set_instance_type(self, instance_type=None): + if instance_type: + self.instance_type = instance_type + else: + self.instance_type = self.choose_from_list(InstanceTypes, 'Instance Type') + + def set_quantity(self, n=0): + if n > 0: + self.quantity = n + else: + self.quantity = self.get_int('Quantity') + + def set_zone(self, zone=None): + if zone: + self.zone = zone + else: + l = [(z, z.name, z.state) for z in self.ec2.get_all_zones()] + self.zone = self.choose_from_list(l, prompt='Choose Availability Zone') + + def set_ami(self, ami=None): + if ami: + self.ami = ami + else: + l = [(a, a.id, a.location) for a in self.ec2.get_all_images()] + self.ami = self.choose_from_list(l, prompt='Choose AMI') + + def add_group(self, group=None): + if group: + self.groups.append(group) + else: + l = [(s, s.name, s.description) for s in self.ec2.get_all_security_groups()] + self.groups.append(self.choose_from_list(l, prompt='Choose Security Group')) + + def set_key(self, key=None): + if key: + self.key = key + else: + l = [(k, k.name, '') for k in self.ec2.get_all_key_pairs()] + self.key = self.choose_from_list(l, prompt='Choose Keypair') + + def update_config(self): + if not self.config.has_section('Credentials'): + self.config.add_section('Credentials') + self.config.set('Credentials', 'aws_access_key_id', self.ec2.aws_access_key_id) + self.config.set('Credentials', 'aws_secret_access_key', self.ec2.aws_secret_access_key) + if not self.config.has_section('Pyami'): + self.config.add_section('Pyami') + sdb_domain = get_domain() + if sdb_domain: + self.config.set('Pyami', 'server_sdb_domain', sdb_domain) + self.config.set('Pyami', 'server_sdb_name', self.name) + + def set_config(self, config_path=None): + if not config_path: + config_path = self.get_filename('Specify Config file') + self.config = Config(path=config_path) + + def get_userdata_string(self): + s = StringIO.StringIO() + self.config.write(s) + return s.getvalue() + + def enter(self, **params): + self.region = params.get('region', self.region) + if not self.region: + self.set_region() + self.ec2 = self.region.connect() + self.name = params.get('name', self.name) + if not self.name: + self.set_name() + self.instance_type = params.get('instance_type', self.instance_type) + if not self.instance_type: + self.set_instance_type() + self.zone = params.get('zone', self.zone) + if not self.zone: + self.set_zone() + self.quantity = params.get('quantity', self.quantity) + if not self.quantity: + self.set_quantity() + self.ami = params.get('ami', self.ami) + if not self.ami: + self.set_ami() + self.groups = params.get('groups', self.groups) + if not self.groups: + self.add_group() + self.key = params.get('key', self.key) + if not self.key: + self.set_key() + self.config = params.get('config', self.config) + if not self.config: + self.set_config() + self.update_config() + +class Order(IObject): + + def __init__(self): + self.items = [] + self.reservation = None + + def add_item(self, **params): + item = Item() + item.enter(**params) + self.items.append(item) + + def display(self): + print 'This Order consists of the following items' + print + print 'QTY\tNAME\tTYPE\nAMI\t\tGroups\t\t\tKeyPair' + for item in self.items: + print '%s\t%s\t%s\t%s\t%s\t%s' % (item.quantity, item.name, item.instance_type, + item.ami.id, item.groups, item.key.name) + + def place(self, block=True): + if get_domain() == None: + print 'SDB Persistence Domain not set' + domain_name = self.get_string('Specify SDB Domain') + set_domain(domain_name) + s = ServerSet() + for item in self.items: + r = item.ami.run(min_count=1, max_count=item.quantity, + key_name=item.key.name, user_data=item.get_userdata_string(), + security_groups=item.groups, instance_type=item.instance_type, + placement=item.zone.name) + if block: + states = [i.state for i in r.instances] + if states.count('running') != len(states): + print states + time.sleep(15) + states = [i.update() for i in r.instances] + for i in r.instances: + server = Server() + server.name = item.name + server.instance_id = i.id + server.reservation = r + server.save() + s.append(server) + if len(s) == 1: + return s[0] + else: + return s + + + diff --git a/api/boto/mashups/server.py b/api/boto/mashups/server.py new file mode 100644 index 0000000..48f637b --- /dev/null +++ b/api/boto/mashups/server.py @@ -0,0 +1,388 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +High-level abstraction of an EC2 server +""" +import boto, boto.utils +from boto.mashups.iobject import IObject +from boto.pyami.config import Config, BotoConfigPath +from boto.mashups.interactive import interactive_shell +from boto.sdb.db.model import Model +from boto.sdb.db.property import * +import os +import StringIO + +class ServerSet(list): + + def __getattr__(self, name): + results = [] + is_callable = False + for server in self: + try: + val = getattr(server, name) + if callable(val): + is_callable = True + results.append(val) + except: + results.append(None) + if is_callable: + self.map_list = results + return self.map + return results + + def map(self, *args): + results = [] + for fn in self.map_list: + results.append(fn(*args)) + return results + +class Server(Model): + + ec2 = boto.connect_ec2() + + @classmethod + def Inventory(cls): + """ + Returns a list of Server instances, one for each Server object + persisted in the db + """ + l = ServerSet() + rs = cls.find() + for server in rs: + l.append(server) + return l + + @classmethod + def Register(cls, name, instance_id, description=''): + s = cls() + s.name = name + s.instance_id = instance_id + s.description = description + s.save() + return s + + def __init__(self, id=None, **kw): + Model.__init__(self, id, **kw) + self._reservation = None + self._instance = None + self._ssh_client = None + self._pkey = None + self._config = None + + name = StringProperty(unique=True, verbose_name="Name") + instance_id = StringProperty(verbose_name="Instance ID") + config_uri = StringProperty() + ami_id = StringProperty(verbose_name="AMI ID") + zone = StringProperty(verbose_name="Availability Zone") + security_group = StringProperty(verbose_name="Security Group", default="default") + key_name = StringProperty(verbose_name="Key Name") + elastic_ip = StringProperty(verbose_name="Elastic IP") + instance_type = StringProperty(verbose_name="Instance Type") + description = StringProperty(verbose_name="Description") + log = StringProperty() + + def setReadOnly(self, value): + raise AttributeError + + def getInstance(self): + if not self._instance: + if self.instance_id: + try: + rs = self.ec2.get_all_instances([self.instance_id]) + except: + return None + if len(rs) > 0: + self._reservation = rs[0] + self._instance = self._reservation.instances[0] + return self._instance + + instance = property(getInstance, setReadOnly, None, 'The Instance for the server') + + def getAMI(self): + if self.instance: + return self.instance.image_id + + ami = property(getAMI, setReadOnly, None, 'The AMI for the server') + + def getStatus(self): + if self.instance: + self.instance.update() + return self.instance.state + + status = property(getStatus, setReadOnly, None, + 'The status of the server') + + def getHostname(self): + if self.instance: + return self.instance.public_dns_name + + hostname = property(getHostname, setReadOnly, None, + 'The public DNS name of the server') + + def getPrivateHostname(self): + if self.instance: + return self.instance.private_dns_name + + private_hostname = property(getPrivateHostname, setReadOnly, None, + 'The private DNS name of the server') + + def getLaunchTime(self): + if self.instance: + return self.instance.launch_time + + launch_time = property(getLaunchTime, setReadOnly, None, + 'The time the Server was started') + + def getConsoleOutput(self): + if self.instance: + return self.instance.get_console_output() + + console_output = property(getConsoleOutput, setReadOnly, None, + 'Retrieve the console output for server') + + def getGroups(self): + if self._reservation: + return self._reservation.groups + else: + return None + + groups = property(getGroups, setReadOnly, None, + 'The Security Groups controlling access to this server') + + def getConfig(self): + if not self._config: + remote_file = BotoConfigPath + local_file = '%s.ini' % self.instance.id + self.get_file(remote_file, local_file) + self._config = Config(local_file) + return self._config + + def setConfig(self, config): + local_file = '%s.ini' % self.instance.id + fp = open(local_file) + config.write(fp) + fp.close() + self.put_file(local_file, BotoConfigPath) + self._config = config + + config = property(getConfig, setConfig, None, + 'The instance data for this server') + + def set_config(self, config): + """ + Set SDB based config + """ + self._config = config + self._config.dump_to_sdb("botoConfigs", self.id) + + def load_config(self): + self._config = Config(do_load=False) + self._config.load_from_sdb("botoConfigs", self.id) + + def stop(self): + if self.instance: + self.instance.stop() + + def start(self): + self.stop() + ec2 = boto.connect_ec2() + ami = ec2.get_all_images(image_ids = [str(self.ami_id)])[0] + groups = ec2.get_all_security_groups(groupnames=[str(self.security_group)]) + if not self._config: + self.load_config() + if not self._config.has_section("Credentials"): + self._config.add_section("Credentials") + self._config.set("Credentials", "aws_access_key_id", ec2.aws_access_key_id) + self._config.set("Credentials", "aws_secret_access_key", ec2.aws_secret_access_key) + + if not self._config.has_section("Pyami"): + self._config.add_section("Pyami") + + if self._manager.domain: + self._config.set('Pyami', 'server_sdb_domain', self._manager.domain.name) + self._config.set("Pyami", 'server_sdb_name', self.name) + + cfg = StringIO.StringIO() + self._config.write(cfg) + cfg = cfg.getvalue() + r = ami.run(min_count=1, + max_count=1, + key_name=self.key_name, + security_groups = groups, + instance_type = self.instance_type, + placement = self.zone, + user_data = cfg) + i = r.instances[0] + self.instance_id = i.id + self.put() + if self.elastic_ip: + ec2.associate_address(self.instance_id, self.elastic_ip) + + def reboot(self): + if self.instance: + self.instance.reboot() + + def get_ssh_client(self, key_file=None, host_key_file='~/.ssh/known_hosts', + uname='root'): + import paramiko + if not self.instance: + print 'No instance yet!' + return + if not self._ssh_client: + if not key_file: + iobject = IObject() + key_file = iobject.get_filename('Path to OpenSSH Key file') + self._pkey = paramiko.RSAKey.from_private_key_file(key_file) + self._ssh_client = paramiko.SSHClient() + self._ssh_client.load_system_host_keys() + self._ssh_client.load_host_keys(os.path.expanduser(host_key_file)) + self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self._ssh_client.connect(self.instance.public_dns_name, + username=uname, pkey=self._pkey) + return self._ssh_client + + def get_file(self, remotepath, localpath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + sftp_client.get(remotepath, localpath) + + def put_file(self, localpath, remotepath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + sftp_client.put(localpath, remotepath) + + def listdir(self, remotepath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + return sftp_client.listdir(remotepath) + + def shell(self, key_file=None): + ssh_client = self.get_ssh_client(key_file) + channel = ssh_client.invoke_shell() + interactive_shell(channel) + + def bundle_image(self, prefix, key_file, cert_file, size): + print 'bundling image...' + print '\tcopying cert and pk over to /mnt directory on server' + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + path, name = os.path.split(key_file) + remote_key_file = '/mnt/%s' % name + self.put_file(key_file, remote_key_file) + path, name = os.path.split(cert_file) + remote_cert_file = '/mnt/%s' % name + self.put_file(cert_file, remote_cert_file) + print '\tdeleting %s' % BotoConfigPath + # delete the metadata.ini file if it exists + try: + sftp_client.remove(BotoConfigPath) + except: + pass + command = 'ec2-bundle-vol ' + command += '-c %s -k %s ' % (remote_cert_file, remote_key_file) + command += '-u %s ' % self._reservation.owner_id + command += '-p %s ' % prefix + command += '-s %d ' % size + command += '-d /mnt ' + if self.instance.instance_type == 'm1.small' or self.instance_type == 'c1.medium': + command += '-r i386' + else: + command += '-r x86_64' + print '\t%s' % command + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' + + def upload_bundle(self, bucket, prefix): + print 'uploading bundle...' + command = 'ec2-upload-bundle ' + command += '-m /mnt/%s.manifest.xml ' % prefix + command += '-b %s ' % bucket + command += '-a %s ' % self.ec2.aws_access_key_id + command += '-s %s ' % self.ec2.aws_secret_access_key + print '\t%s' % command + ssh_client = self.get_ssh_client() + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' + + def create_image(self, bucket=None, prefix=None, key_file=None, cert_file=None, size=None): + iobject = IObject() + if not bucket: + bucket = iobject.get_string('Name of S3 bucket') + if not prefix: + prefix = iobject.get_string('Prefix for AMI file') + if not key_file: + key_file = iobject.get_filename('Path to RSA private key file') + if not cert_file: + cert_file = iobject.get_filename('Path to RSA public cert file') + if not size: + size = iobject.get_int('Size (in MB) of bundled image') + self.bundle_image(prefix, key_file, cert_file, size) + self.upload_bundle(bucket, prefix) + print 'registering image...' + self.image_id = self.ec2.register_image('%s/%s.manifest.xml' % (bucket, prefix)) + return self.image_id + + def attach_volume(self, volume, device="/dev/sdp"): + """ + Attach an EBS volume to this server + + :param volume: EBS Volume to attach + :type volume: boto.ec2.volume.Volume + + :param device: Device to attach to (default to /dev/sdp) + :type device: string + """ + if hasattr(volume, "id"): + volume_id = volume.id + else: + volume_id = volume + return self.ec2.attach_volume(volume_id=volume_id, instance_id=self.instance_id, device=device) + + def detach_volume(self, volume): + """ + Detach an EBS volume from this server + + :param volume: EBS Volume to detach + :type volume: boto.ec2.volume.Volume + """ + if hasattr(volume, "id"): + volume_id = volume.id + else: + volume_id = volume + return self.ec2.detach_volume(volume_id=volume_id, instance_id=self.instance_id) + + def install_package(self, package_name): + print 'installing %s...' % package_name + command = 'yum -y install %s' % package_name + print '\t%s' % command + ssh_client = self.get_ssh_client() + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' diff --git a/api/boto/mturk/__init__.py b/api/boto/mturk/__init__.py new file mode 100644 index 0000000..449bd16 --- /dev/null +++ b/api/boto/mturk/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/api/boto/mturk/connection.py b/api/boto/mturk/connection.py new file mode 100644 index 0000000..261e2a7 --- /dev/null +++ b/api/boto/mturk/connection.py @@ -0,0 +1,504 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax +import datetime + +from boto import handler +from boto.mturk.price import Price +import boto.mturk.notification +from boto.connection import AWSQueryConnection +from boto.exception import EC2ResponseError +from boto.resultset import ResultSet + +class MTurkConnection(AWSQueryConnection): + + APIVersion = '2006-10-31' + SignatureVersion = '1' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host='mechanicalturk.amazonaws.com', debug=0, + https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory) + + def get_account_balance(self): + """ + """ + params = {} + return self._process_request('GetAccountBalance', params, [('AvailableBalance', Price), + ('OnHoldBalance', Price)]) + + def register_hit_type(self, title, description, reward, duration, + keywords=None, approval_delay=None, qual_req=None): + """ + Register a new HIT Type + \ttitle, description are strings + \treward is a Price object + \tduration can be an integer or string + """ + params = {'Title' : title, + 'Description' : description, + 'AssignmentDurationInSeconds' : duration} + params.update(MTurkConnection.get_price_as_price(reward).get_as_params('Reward')) + + if keywords: + params['Keywords'] = keywords + + if approval_delay is not None: + params['AutoApprovalDelayInSeconds']= approval_delay + + return self._process_request('RegisterHITType', params) + + def set_email_notification(self, hit_type, email, event_types=None): + """ + Performs a SetHITTypeNotification operation to set email notification for a specified HIT type + """ + return self._set_notification(hit_type, 'Email', email, event_types) + + def set_rest_notification(self, hit_type, url, event_types=None): + """ + Performs a SetHITTypeNotification operation to set REST notification for a specified HIT type + """ + return self._set_notification(hit_type, 'REST', url, event_types) + + def _set_notification(self, hit_type, transport, destination, event_types=None): + """ + Common SetHITTypeNotification operation to set notification for a specified HIT type + """ + assert type(hit_type) is str, "hit_type argument should be a string." + + params = {'HITTypeId': hit_type} + + # from the Developer Guide: + # The 'Active' parameter is optional. If omitted, the active status of the HIT type's + # notification specification is unchanged. All HIT types begin with their + # notification specifications in the "inactive" status. + notification_params = {'Destination': destination, + 'Transport': transport, + 'Version': boto.mturk.notification.NotificationMessage.NOTIFICATION_VERSION, + 'Active': True, + } + + # add specific event types if required + if event_types: + self.build_list_params(notification_params, event_types, 'EventType') + + # Set up dict of 'Notification.1.Transport' etc. values + notification_rest_params = {} + num = 1 + for key in notification_params: + notification_rest_params['Notification.%d.%s' % (num, key)] = notification_params[key] + + # Update main params dict + params.update(notification_rest_params) + + # Execute operation + return self._process_request('SetHITTypeNotification', params) + + def create_hit(self, hit_type=None, question=None, lifetime=60*60*24*7, max_assignments=1, + title=None, description=None, keywords=None, reward=None, + duration=60*60*24*7, approval_delay=None, annotation=None, qual_req=None, + questions=None, qualifications=None, response_groups=None): + """ + Creates a new HIT. + Returns a ResultSet + See: http://docs.amazonwebservices.com/AWSMechanicalTurkRequester/2006-10-31/ApiReference_CreateHITOperation.html + """ + + # handle single or multiple questions + if question is not None and questions is not None: + raise ValueError("Must specify either question (single Question instance) or questions (list), but not both") + if question is not None and questions is None: + questions = [question] + + + # Handle basic required arguments and set up params dict + params = {'Question': question.get_as_xml(), + 'LifetimeInSeconds' : lifetime, + 'MaxAssignments' : max_assignments, + } + + # if hit type specified then add it + # else add the additional required parameters + if hit_type: + params['HITTypeId'] = hit_type + else: + # Handle keywords + final_keywords = MTurkConnection.get_keywords_as_string(keywords) + + # Handle price argument + final_price = MTurkConnection.get_price_as_price(reward) + + additional_params = {'Title': title, + 'Description' : description, + 'Keywords': final_keywords, + 'AssignmentDurationInSeconds' : duration, + } + additional_params.update(final_price.get_as_params('Reward')) + + if approval_delay is not None: + additional_params['AutoApprovalDelayInSeconds'] = approval_delay + + # add these params to the others + params.update(additional_params) + + # add the annotation if specified + if annotation is not None: + params['RequesterAnnotation'] = annotation + + # Add the Qualifications if specified + if qualifications is not None: + params.update(qualifications.get_as_params()) + + # Handle optional response groups argument + if response_groups: + self.build_list_params(params, response_groups, 'ResponseGroup') + + # Submit + return self._process_request('CreateHIT', params, [('HIT', HIT),]) + + def get_reviewable_hits(self, hit_type=None, status='Reviewable', + sort_by='Expiration', sort_direction='Ascending', + page_size=10, page_number=1): + """ + Retrieve the HITs that have a status of Reviewable, or HITs that + have a status of Reviewing, and that belong to the Requester calling the operation. + """ + params = {'Status' : status, + 'SortProperty' : sort_by, + 'SortDirection' : sort_direction, + 'PageSize' : page_size, + 'PageNumber' : page_number} + + # Handle optional hit_type argument + if hit_type is not None: + params.update({'HITTypeId': hit_type}) + + return self._process_request('GetReviewableHITs', params, [('HIT', HIT),]) + + def search_hits(self, sort_by='CreationTime', sort_direction='Ascending', + page_size=10, page_number=1): + """ + Return all of a Requester's HITs, on behalf of the Requester. + The operation returns HITs of any status, except for HITs that have been disposed + with the DisposeHIT operation. + Note: + The SearchHITs operation does not accept any search parameters that filter the results. + """ + params = {'SortProperty' : sort_by, + 'SortDirection' : sort_direction, + 'PageSize' : page_size, + 'PageNumber' : page_number} + + return self._process_request('SearchHITs', params, [('HIT', HIT),]) + + def get_assignments(self, hit_id, status=None, + sort_by='SubmitTime', sort_direction='Ascending', + page_size=10, page_number=1): + """ + Retrieves completed assignments for a HIT. + Use this operation to retrieve the results for a HIT. + + The returned ResultSet will have the following attributes: + + NumResults + The number of assignments on the page in the filtered results list, + equivalent to the number of assignments being returned by this call. + A non-negative integer + PageNumber + The number of the page in the filtered results list being returned. + A positive integer + TotalNumResults + The total number of HITs in the filtered results list based on this call. + A non-negative integer + + The ResultSet will contain zero or more Assignment objects + + """ + params = {'HITId' : hit_id, + 'SortProperty' : sort_by, + 'SortDirection' : sort_direction, + 'PageSize' : page_size, + 'PageNumber' : page_number} + + if status is not None: + params['AssignmentStatus'] = status + + return self._process_request('GetAssignmentsForHIT', params, [('Assignment', Assignment),]) + + def approve_assignment(self, assignment_id, feedback=None): + """ + """ + params = {'AssignmentId' : assignment_id,} + if feedback: + params['RequesterFeedback'] = feedback + return self._process_request('ApproveAssignment', params) + + def reject_assignment(self, assignment_id, feedback=None): + """ + """ + params = {'AssignmentId' : assignment_id,} + if feedback: + params['RequesterFeedback'] = feedback + return self._process_request('RejectAssignment', params) + + def get_hit(self, hit_id): + """ + """ + params = {'HITId' : hit_id,} + return self._process_request('GetHIT', params, [('HIT', HIT),]) + + def set_reviewing(self, hit_id, revert=None): + """ + Update a HIT with a status of Reviewable to have a status of Reviewing, + or reverts a Reviewing HIT back to the Reviewable status. + + Only HITs with a status of Reviewable can be updated with a status of Reviewing. + Similarly, only Reviewing HITs can be reverted back to a status of Reviewable. + """ + params = {'HITId' : hit_id,} + if revert: + params['Revert'] = revert + return self._process_request('SetHITAsReviewing', params) + + def disable_hit(self, hit_id): + """ + Remove a HIT from the Mechanical Turk marketplace, approves all submitted assignments + that have not already been approved or rejected, and disposes of the HIT and all + assignment data. + + Assignments for the HIT that have already been submitted, but not yet approved or rejected, will be + automatically approved. Assignments in progress at the time of the call to DisableHIT will be + approved once the assignments are submitted. You will be charged for approval of these assignments. + DisableHIT completely disposes of the HIT and all submitted assignment data. Assignment results + data cannot be retrieved for a HIT that has been disposed. + + It is not possible to re-enable a HIT once it has been disabled. To make the work from a disabled HIT + available again, create a new HIT. + """ + params = {'HITId' : hit_id,} + return self._process_request('DisableHIT', params) + + def dispose_hit(self, hit_id): + """ + Dispose of a HIT that is no longer needed. + + Only HITs in the "reviewable" state, with all submitted assignments approved or rejected, + can be disposed. A Requester can call GetReviewableHITs to determine which HITs are + reviewable, then call GetAssignmentsForHIT to retrieve the assignments. + Disposing of a HIT removes the HIT from the results of a call to GetReviewableHITs. + """ + params = {'HITId' : hit_id,} + return self._process_request('DisposeHIT', params) + + def expire_hit(self, hit_id): + + """ + Expire a HIT that is no longer needed. + + The effect is identical to the HIT expiring on its own. The HIT no longer appears on the + Mechanical Turk web site, and no new Workers are allowed to accept the HIT. Workers who + have accepted the HIT prior to expiration are allowed to complete it or return it, or + allow the assignment duration to elapse (abandon the HIT). Once all remaining assignments + have been submitted, the expired HIT becomes "reviewable", and will be returned by a call + to GetReviewableHITs. + """ + params = {'HITId' : hit_id,} + return self._process_request('ForceExpireHIT', params) + + def extend_hit(self, hit_id, assignments_increment=None, expiration_increment=None): + """ + Increase the maximum number of assignments, or extend the expiration date, of an existing HIT. + + NOTE: If a HIT has a status of Reviewable and the HIT is extended to make it Available, the + HIT will not be returned by GetReviewableHITs, and its submitted assignments will not + be returned by GetAssignmentsForHIT, until the HIT is Reviewable again. + Assignment auto-approval will still happen on its original schedule, even if the HIT has + been extended. Be sure to retrieve and approve (or reject) submitted assignments before + extending the HIT, if so desired. + """ + # must provide assignment *or* expiration increment + if (assignments_increment is None and expiration_increment is None) or \ + (assignments_increment is not None and expiration_increment is not None): + raise ValueError("Must specify either assignments_increment or expiration_increment, but not both") + + params = {'HITId' : hit_id,} + if assignments_increment: + params['MaxAssignmentsIncrement'] = assignments_increment + if expiration_increment: + params['ExpirationIncrementInSeconds'] = expiration_increment + + return self._process_request('ExtendHIT', params) + + def get_help(self, about, help_type='Operation'): + """ + Return information about the Mechanical Turk Service operations and response group + NOTE - this is basically useless as it just returns the URL of the documentation + + help_type: either 'Operation' or 'ResponseGroup' + """ + params = {'About': about, 'HelpType': help_type,} + return self._process_request('Help', params) + + def grant_bonus(self, worker_id, assignment_id, bonus_price, reason): + """ + Issues a payment of money from your account to a Worker. + To be eligible for a bonus, the Worker must have submitted results for one of your + HITs, and have had those results approved or rejected. This payment happens separately + from the reward you pay to the Worker when you approve the Worker's assignment. + The Bonus must be passed in as an instance of the Price object. + """ + params = bonus_price.get_as_params('BonusAmount', 1) + params['WorkerId'] = worker_id + params['AssignmentId'] = assignment_id + params['Reason'] = reason + + return self._process_request('GrantBonus', params) + + def _process_request(self, request_type, params, marker_elems=None): + """ + Helper to process the xml response from AWS + """ + response = self.make_request(request_type, params) + return self._process_response(response, marker_elems) + + def _process_response(self, response, marker_elems=None): + """ + Helper to process the xml response from AWS + """ + body = response.read() + #print body + if '' not in body: + rs = ResultSet(marker_elems) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise EC2ResponseError(response.status, response.reason, body) + + @staticmethod + def get_keywords_as_string(keywords): + """ + Returns a comma+space-separated string of keywords from either a list or a string + """ + if type(keywords) is list: + final_keywords = ', '.join(keywords) + elif type(keywords) is str: + final_keywords = keywords + elif type(keywords) is unicode: + final_keywords = keywords.encode('utf-8') + elif keywords is None: + final_keywords = "" + else: + raise TypeError("keywords argument must be a string or a list of strings; got a %s" % type(keywords)) + return final_keywords + + @staticmethod + def get_price_as_price(reward): + """ + Returns a Price data structure from either a float or a Price + """ + if isinstance(reward, Price): + final_price = reward + else: + final_price = Price(reward) + return final_price + +class BaseAutoResultElement: + """ + Base class to automatically add attributes when parsing XML + """ + def __init__(self, connection): + self.connection = connection + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + setattr(self, name, value) + +class HIT(BaseAutoResultElement): + """ + Class to extract a HIT structure from a response (used in ResultSet) + + Will have attributes named as per the Developer Guide, + e.g. HITId, HITTypeId, CreationTime + """ + + # property helper to determine if HIT has expired + def _has_expired(self): + """ Has this HIT expired yet? """ + expired = False + if hasattr(self, 'Expiration'): + now = datetime.datetime.utcnow() + expiration = datetime.datetime.strptime(self.Expiration, '%Y-%m-%dT%H:%M:%SZ') + expired = (now >= expiration) + else: + raise ValueError("ERROR: Request for expired property, but no Expiration in HIT!") + return expired + + # are we there yet? + expired = property(_has_expired) + +class Assignment(BaseAutoResultElement): + """ + Class to extract an Assignment structure from a response (used in ResultSet) + + Will have attributes named as per the Developer Guide, + e.g. AssignmentId, WorkerId, HITId, Answer, etc + """ + + def __init__(self, connection): + BaseAutoResultElement.__init__(self, connection) + self.answers = [] + + def endElement(self, name, value, connection): + # the answer consists of embedded XML, so it needs to be parsed independantly + if name == 'Answer': + answer_rs = ResultSet([('Answer', QuestionFormAnswer),]) + h = handler.XmlHandler(answer_rs, connection) + value = self.connection.get_utf8_value(value) + xml.sax.parseString(value, h) + self.answers.append(answer_rs) + else: + BaseAutoResultElement.endElement(self, name, value, connection) + +class QuestionFormAnswer(BaseAutoResultElement): + """ + Class to extract Answers from inside the embedded XML QuestionFormAnswers element inside the + Answer element which is part of the Assignment structure + + A QuestionFormAnswers element contains an Answer element for each question in the HIT or + Qualification test for which the Worker provided an answer. Each Answer contains a + QuestionIdentifier element whose value corresponds to the QuestionIdentifier of a + Question in the QuestionForm. See the QuestionForm data structure for more information about + questions and answer specifications. + + If the question expects a free-text answer, the Answer element contains a FreeText element. This + element contains the Worker's answer + + *NOTE* - currently really only supports free-text answers + """ + + pass diff --git a/api/boto/mturk/notification.py b/api/boto/mturk/notification.py new file mode 100644 index 0000000..4904a99 --- /dev/null +++ b/api/boto/mturk/notification.py @@ -0,0 +1,95 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Provides NotificationMessage and Event classes, with utility methods, for +implementations of the Mechanical Turk Notification API. +""" + +import hmac +try: + from hashlib import sha1 as sha +except ImportError: + import sha +import base64 +import re + +class NotificationMessage: + + NOTIFICATION_WSDL = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurk/2006-05-05/AWSMechanicalTurkRequesterNotification.wsdl" + NOTIFICATION_VERSION = '2006-05-05' + + SERVICE_NAME = "AWSMechanicalTurkRequesterNotification" + OPERATION_NAME = "Notify" + + EVENT_PATTERN = r"Event\.(?P\d+)\.(?P\w+)" + EVENT_RE = re.compile(EVENT_PATTERN) + + def __init__(self, d): + """ + Constructor; expects parameter d to be a dict of string parameters from a REST transport notification message + """ + self.signature = d['Signature'] # vH6ZbE0NhkF/hfNyxz2OgmzXYKs= + self.timestamp = d['Timestamp'] # 2006-05-23T23:22:30Z + self.version = d['Version'] # 2006-05-05 + assert d['method'] == NotificationMessage.OPERATION_NAME, "Method should be '%s'" % NotificationMessage.OPERATION_NAME + + # Build Events + self.events = [] + events_dict = {} + if 'Event' in d: + # TurboGears surprised me by 'doing the right thing' and making { 'Event': { '1': { 'EventType': ... } } } etc. + events_dict = d['Event'] + else: + for k in d: + v = d[k] + if k.startswith('Event.'): + ed = NotificationMessage.EVENT_RE.search(k).groupdict() + n = int(ed['n']) + param = str(ed['param']) + if n not in events_dict: + events_dict[n] = {} + events_dict[n][param] = v + for n in events_dict: + self.events.append(Event(events_dict[n])) + + def verify(self, secret_key): + """ + Verifies the authenticity of a notification message. + """ + verification_input = NotificationMessage.SERVICE_NAME + NotificationMessage.OPERATION_NAME + self.timestamp + h = hmac.new(key=secret_key, digestmod=sha) + h.update(verification_input) + signature_calc = base64.b64encode(h.digest()) + return self.signature == signature_calc + +class Event: + def __init__(self, d): + self.event_type = d['EventType'] + self.event_time_str = d['EventTime'] + self.hit_type = d['HITTypeId'] + self.hit_id = d['HITId'] + self.assignment_id = d['AssignmentId'] + + #TODO: build self.event_time datetime from string self.event_time_str + + def __repr__(self): + return "" % (self.event_type, self.hit_id) diff --git a/api/boto/mturk/price.py b/api/boto/mturk/price.py new file mode 100644 index 0000000..3c88a96 --- /dev/null +++ b/api/boto/mturk/price.py @@ -0,0 +1,48 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Price: + + def __init__(self, amount=0.0, currency_code='USD'): + self.amount = amount + self.currency_code = currency_code + self.formatted_price = '' + + def __repr__(self): + if self.formatted_price: + return self.formatted_price + else: + return str(self.amount) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Amount': + self.amount = float(value) + elif name == 'CurrencyCode': + self.currency_code = value + elif name == 'FormattedPrice': + self.formatted_price = value + + def get_as_params(self, label, ord=1): + return {'%s.%d.Amount'%(label, ord) : str(self.amount), + '%s.%d.CurrencyCode'%(label, ord) : self.currency_code} diff --git a/api/boto/mturk/qualification.py b/api/boto/mturk/qualification.py new file mode 100644 index 0000000..ed02087 --- /dev/null +++ b/api/boto/mturk/qualification.py @@ -0,0 +1,118 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Qualifications: + + def __init__(self, requirements = []): + self.requirements = requirements + + def add(self, req): + self.requirements.append(req) + + def get_as_params(self): + params = {} + assert(len(self.requirements) <= 10) + for n, req in enumerate(self.requirements): + reqparams = req.get_as_params() + for rp in reqparams: + params['QualificationRequirement.%s.%s' % ((n+1),rp) ] = reqparams[rp] + return params + + +class Requirement(object): + """ + Representation of a single requirement + """ + + def __init__(self, qualification_type_id, comparator, integer_value, required_to_preview=False): + self.qualification_type_id = qualification_type_id + self.comparator = comparator + self.integer_value = integer_value + self.required_to_preview = required_to_preview + + def get_as_params(self): + params = { + "QualificationTypeId": self.qualification_type_id, + "Comparator": self.comparator, + "IntegerValue": self.integer_value, + } + if self.required_to_preview: + params['RequiredToPreview'] = "true" + return params + +class PercentAssignmentsSubmittedRequirement(Requirement): + """ + The percentage of assignments the Worker has submitted, over all assignments the Worker has accepted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="00000000000000000000", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsAbandonedRequirement(Requirement): + """ + The percentage of assignments the Worker has abandoned (allowed the deadline to elapse), over all assignments the Worker has accepted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="00000000000000000070", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsReturnedRequirement(Requirement): + """ + The percentage of assignments the Worker has returned, over all assignments the Worker has accepted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="000000000000000000E0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsApprovedRequirement(Requirement): + """ + The percentage of assignments the Worker has submitted that were subsequently approved by the Requester, over all assignments the Worker has submitted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="000000000000000000L0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsRejectedRequirement(Requirement): + """ + The percentage of assignments the Worker has submitted that were subsequently rejected by the Requester, over all assignments the Worker has submitted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="000000000000000000S0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class LocaleRequirement(Requirement): + """ + A Qualification requirement based on the Worker's location. The Worker's location is specified by the Worker to Mechanical Turk when the Worker creates his account. + """ + + def __init__(self, comparator, locale, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="00000000000000000071", comparator=comparator, integer_value=None, required_to_preview=required_to_preview) + self.locale = locale + + def get_as_params(self): + params = { + "QualificationTypeId": self.qualification_type_id, + "Comparator": self.comparator, + 'LocaleValue.Country': self.locale, + } + if self.required_to_preview: + params['RequiredToPreview'] = "true" + return params diff --git a/api/boto/mturk/question.py b/api/boto/mturk/question.py new file mode 100644 index 0000000..89f1a45 --- /dev/null +++ b/api/boto/mturk/question.py @@ -0,0 +1,353 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Question(object): + + QUESTION_XML_TEMPLATE = """%s%s%s%s%s""" + DISPLAY_NAME_XML_TEMPLATE = """%s""" + + def __init__(self, identifier, content, answer_spec, is_required=False, display_name=None): #amount=0.0, currency_code='USD'): + self.identifier = identifier + self.content = content + self.answer_spec = answer_spec + self.is_required = is_required + self.display_name = display_name + + def get_as_params(self, label='Question', identifier=None): + + if identifier is None: + raise ValueError("identifier (QuestionIdentifier) is required per MTurk spec.") + + return { label : self.get_as_xml() } + + def get_as_xml(self): + # add the display name if required + display_name_xml = '' + if self.display_name: + display_name_xml = self.DISPLAY_NAME_XML_TEMPLATE %(self.display_name) + + ret = Question.QUESTION_XML_TEMPLATE % (self.identifier, + display_name_xml, + str(self.is_required).lower(), + self.content.get_as_xml(), + self.answer_spec.get_as_xml()) + + return ret + +class ExternalQuestion(object): + + EXTERNAL_QUESTIONFORM_SCHEMA_LOCATION = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd" + EXTERNAL_QUESTION_XML_TEMPLATE = """%s%s""" + + def __init__(self, external_url, frame_height): + self.external_url = external_url + self.frame_height = frame_height + + def get_as_params(self, label='ExternalQuestion'): + return { label : self.get_as_xml() } + + def get_as_xml(self): + ret = ExternalQuestion.EXTERNAL_QUESTION_XML_TEMPLATE % (ExternalQuestion.EXTERNAL_QUESTIONFORM_SCHEMA_LOCATION, + self.external_url, + self.frame_height) + return ret + +class OrderedContent(object): + def __init__(self): + self.items = [] + + def append(self, field, value): + "Expects field type and value" + self.items.append((field, value)) + + def get_binary_xml(self, field, value): + return """ + + + %s + %s + + %s + %s +""" % (value['binary_type'], + value['binary_subtype'], + value['binary'], + value['binary_alttext']) + + def get_application_xml(self, field, value): + raise NotImplementedError("Application question content is not yet supported.") + + def get_as_xml(self): + default_handler = lambda f,v: '<%s>%s' % (f,v,f) + bulleted_list_handler = lambda _,list: '%s' % ''.join([('%s' % item) for item in list]) + formatted_content_handler = lambda _,content: "" % content + application_handler = self.get_application_xml + binary_handler = self.get_binary_xml + + children = '' + for (field,value) in self.items: + handler = default_handler + if field == 'List': + handler = bulleted_list_handler + elif field == 'Application': + handler = application_handler + elif field == 'Binary': + handler = binary_handler + elif field == 'FormattedContent': + handler = formatted_content_handler + children = children + handler(field, value) + + return children + +class Overview(object): + OVERVIEW_XML_TEMPLATE = """%s""" + + def __init__(self): + self.ordered_content = OrderedContent() + + def append(self, field, value): + self.ordered_content.append(field,value) + + def get_as_params(self, label='Overview'): + return { label : self.get_as_xml() } + + def get_as_xml(self): + ret = Overview.OVERVIEW_XML_TEMPLATE % (self.ordered_content.get_as_xml()) + + return ret + + +class QuestionForm(object): + + QUESTIONFORM_SCHEMA_LOCATION = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionForm.xsd" + QUESTIONFORM_XML_TEMPLATE = """%s""" # % (ns, questions_xml) + + def __init__(self, questions=None, overview=None): + if questions is None or type(questions) is not list: + raise ValueError("Must pass a list of Question instances to QuestionForm constructor") + else: + self.questions = questions + self.overview = overview + + + def get_as_xml(self): + if self.overview: + overview_xml = self.overview.get_as_xml() + questions_xml = "".join([q.get_as_xml() for q in self.questions]) + qf_xml = overview_xml + questions_xml + return QuestionForm.QUESTIONFORM_XML_TEMPLATE % (QuestionForm.QUESTIONFORM_SCHEMA_LOCATION, qf_xml) + + #def startElement(self, name, attrs, connection): + # return None + # + #def endElement(self, name, value, connection): + # + # #if name == 'Amount': + # # self.amount = float(value) + # #elif name == 'CurrencyCode': + # # self.currency_code = value + # #elif name == 'FormattedPrice': + # # self.formatted_price = value + # + # pass # What's this method for? I don't get it. + +class QuestionContent(object): + QUESTIONCONTENT_XML_TEMPLATE = """%s""" + + def __init__(self): + self.ordered_content = OrderedContent() + + def append(self, field, value): + self.ordered_content.append(field,value) + + def get_as_xml(self): + ret = QuestionContent.QUESTIONCONTENT_XML_TEMPLATE % (self.ordered_content.get_as_xml()) + + return ret + + +class AnswerSpecification(object): + + ANSWERSPECIFICATION_XML_TEMPLATE = """%s""" + + def __init__(self, spec): + self.spec = spec + def get_as_xml(self): + values = () # TODO + return AnswerSpecification.ANSWERSPECIFICATION_XML_TEMPLATE % self.spec.get_as_xml() + +class FreeTextAnswer(object): + + FREETEXTANSWER_XML_TEMPLATE = """%s%s""" # (constraints, default) + FREETEXTANSWER_CONSTRAINTS_XML_TEMPLATE = """%s%s%s""" # (is_numeric_xml, length_xml, regex_xml) + FREETEXTANSWER_LENGTH_XML_TEMPLATE = """""" # (min_length_attr, max_length_attr) + FREETEXTANSWER_ISNUMERIC_XML_TEMPLATE = """""" # (min_value_attr, max_value_attr) + FREETEXTANSWER_DEFAULTTEXT_XML_TEMPLATE = """%s""" # (default) + + def __init__(self, default=None, min_length=None, max_length=None, is_numeric=False, min_value=None, max_value=None, format_regex=None): + self.default = default + self.min_length = min_length + self.max_length = max_length + self.is_numeric = is_numeric + self.min_value = min_value + self.max_value = max_value + self.format_regex = format_regex + + def get_as_xml(self): + is_numeric_xml = "" + if self.is_numeric: + min_value_attr = "" + max_value_attr = "" + if self.min_value: + min_value_attr = """minValue="%d" """ % self.min_value + if self.max_value: + max_value_attr = """maxValue="%d" """ % self.max_value + is_numeric_xml = FreeTextAnswer.FREETEXTANSWER_ISNUMERIC_XML_TEMPLATE % (min_value_attr, max_value_attr) + + length_xml = "" + if self.min_length or self.max_length: + min_length_attr = "" + max_length_attr = "" + if self.min_length: + min_length_attr = """minLength="%d" """ + if self.max_length: + max_length_attr = """maxLength="%d" """ + length_xml = FreeTextAnswer.FREETEXTANSWER_LENGTH_XML_TEMPLATE % (min_length_attr, max_length_attr) + + regex_xml = "" + if self.format_regex: + format_regex_attribs = '''regex="%s"''' %self.format_regex['regex'] + + error_text = self.format_regex.get('error_text', None) + if error_text: + format_regex_attribs += ' errorText="%s"' %error_text + + flags = self.format_regex.get('flags', None) + if flags: + format_regex_attribs += ' flags="%s"' %flags + + regex_xml = """""" %format_regex_attribs + + constraints_xml = "" + if is_numeric_xml or length_xml or regex_xml: + constraints_xml = FreeTextAnswer.FREETEXTANSWER_CONSTRAINTS_XML_TEMPLATE % (is_numeric_xml, length_xml, regex_xml) + + default_xml = "" + if self.default is not None: + default_xml = FreeTextAnswer.FREETEXTANSWER_DEFAULTTEXT_XML_TEMPLATE % self.default + + return FreeTextAnswer.FREETEXTANSWER_XML_TEMPLATE % (constraints_xml, default_xml) + +class FileUploadAnswer(object): + FILEUPLOADANSWER_XML_TEMLPATE = """%d%d""" # (min, max) + DEFAULT_MIN_SIZE = 1024 # 1K (completely arbitrary!) + DEFAULT_MAX_SIZE = 5 * 1024 * 1024 # 5MB (completely arbitrary!) + + def __init__(self, min=None, max=None): + self.min = min + self.max = max + if self.min is None: + self.min = FileUploadAnswer.DEFAULT_MIN_SIZE + if self.max is None: + self.max = FileUploadAnswer.DEFAULT_MAX_SIZE + + def get_as_xml(self): + return FileUploadAnswer.FILEUPLOADANSWER_XML_TEMLPATE % (self.min, self.max) + +class SelectionAnswer(object): + """ + A class to generate SelectionAnswer XML data structures. + Does not yet implement Binary selection options. + """ + SELECTIONANSWER_XML_TEMPLATE = """%s%s%s""" # % (count_xml, style_xml, selections_xml) + SELECTION_XML_TEMPLATE = """%s%s""" # (identifier, value_xml) + SELECTION_VALUE_XML_TEMPLATE = """<%s>%s""" # (type, value, type) + STYLE_XML_TEMPLATE = """%s""" # (style) + MIN_SELECTION_COUNT_XML_TEMPLATE = """%s""" # count + MAX_SELECTION_COUNT_XML_TEMPLATE = """%s""" # count + ACCEPTED_STYLES = ['radiobutton', 'dropdown', 'checkbox', 'list', 'combobox', 'multichooser'] + OTHER_SELECTION_ELEMENT_NAME = 'OtherSelection' + + def __init__(self, min=1, max=1, style=None, selections=None, type='text', other=False): + + if style is not None: + if style in SelectionAnswer.ACCEPTED_STYLES: + self.style_suggestion = style + else: + raise ValueError("style '%s' not recognized; should be one of %s" % (style, ', '.join(SelectionAnswer.ACCEPTED_STYLES))) + else: + self.style_suggestion = None + + if selections is None: + raise ValueError("SelectionAnswer.__init__(): selections must be a non-empty list of (content, identifier) tuples") + else: + self.selections = selections + + self.min_selections = min + self.max_selections = max + + assert len(selections) >= self.min_selections, "# of selections is less than minimum of %d" % self.min_selections + #assert len(selections) <= self.max_selections, "# of selections exceeds maximum of %d" % self.max_selections + + self.type = type + + self.other = other + + def get_as_xml(self): + xml = "" + if self.type == 'text': + TYPE_TAG = "Text" + elif self.type == 'binary': + TYPE_TAG = "Binary" + else: + raise ValueError("illegal type: %s; must be either 'text' or 'binary'" % str(self.type)) + + # build list of elements + selections_xml = "" + for tpl in self.selections: + value_xml = SelectionAnswer.SELECTION_VALUE_XML_TEMPLATE % (TYPE_TAG, tpl[0], TYPE_TAG) + selection_xml = SelectionAnswer.SELECTION_XML_TEMPLATE % (tpl[1], value_xml) + selections_xml += selection_xml + + if self.other: + # add OtherSelection element as xml if available + if hasattr(self.other, 'get_as_xml'): + assert type(self.other) == FreeTextAnswer, 'OtherSelection can only be a FreeTextAnswer' + selections_xml += self.other.get_as_xml().replace('FreeTextAnswer', 'OtherSelection') + else: + selections_xml += "" + + if self.style_suggestion is not None: + style_xml = SelectionAnswer.STYLE_XML_TEMPLATE % self.style_suggestion + else: + style_xml = "" + + if self.style_suggestion != 'radiobutton': + count_xml = SelectionAnswer.MIN_SELECTION_COUNT_XML_TEMPLATE %self.min_selections + count_xml += SelectionAnswer.MAX_SELECTION_COUNT_XML_TEMPLATE %self.max_selections + else: + count_xml = "" + + ret = SelectionAnswer.SELECTIONANSWER_XML_TEMPLATE % (count_xml, style_xml, selections_xml) + + # return XML + return ret + diff --git a/api/boto/mturk/test/all_tests.py b/api/boto/mturk/test/all_tests.py new file mode 100644 index 0000000..a8f291a --- /dev/null +++ b/api/boto/mturk/test/all_tests.py @@ -0,0 +1,8 @@ +import doctest + +# doctest.testfile("create_hit.doctest") +# doctest.testfile("create_hit_binary.doctest") +doctest.testfile("create_free_text_question_regex.doctest") +# doctest.testfile("create_hit_from_hit_type.doctest") +# doctest.testfile("search_hits.doctest") +# doctest.testfile("reviewable_hits.doctest") diff --git a/api/boto/mturk/test/cleanup_tests.py b/api/boto/mturk/test/cleanup_tests.py new file mode 100644 index 0000000..7bdff90 --- /dev/null +++ b/api/boto/mturk/test/cleanup_tests.py @@ -0,0 +1,67 @@ +from boto.mturk.connection import MTurkConnection + +def cleanup(): + """Remove any boto test related HIT's""" + + conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + current_page = 1 + page_size = 10 + total_disabled = 0 + ignored = [] + + while True: + # reset the total for this loop + disabled_count = 0 + + # search all the hits in the sandbox + search_rs = conn.search_hits(page_size=page_size, page_number=current_page) + + # success? + if search_rs.status: + for hit in search_rs: + # delete any with Boto in the description + print 'hit id:%s Status:%s, desc:%s' %(hit.HITId, hit.HITStatus, hit.Description) + if hit.Description.find('Boto') != -1: + if hit.HITStatus != 'Reviewable': + print 'Disabling hit id:%s %s' %(hit.HITId, hit.Description) + disable_rs = conn.disable_hit(hit.HITId) + if disable_rs.status: + disabled_count += 1 + # update the running total + total_disabled += 1 + else: + print 'Error when disabling, code:%s, message:%s' %(disable_rs.Code, disable_rs.Message) + else: + print 'Disposing hit id:%s %s' %(hit.HITId, hit.Description) + dispose_rs = conn.dispose_hit(hit.HITId) + if dispose_rs.status: + disabled_count += 1 + # update the running total + total_disabled += 1 + else: + print 'Error when disposing, code:%s, message:%s' %(dispose_rs.Code, dispose_rs.Message) + + else: + if hit.HITId not in ignored: + print 'ignored:%s' %hit.HITId + ignored.append(hit.HITId) + + # any more results? + if int(search_rs.TotalNumResults) > current_page*page_size: + # if we have disabled any HITs on this page + # then we don't need to go to a new page + # otherwise we do + if not disabled_count: + current_page += 1 + else: + # no, we're done + break + else: + print 'Error performing search, code:%s, message:%s' %(search_rs.Code, search_rs.Message) + break + + total_ignored = len(ignored) + print 'Processed: %d HITs, disabled/disposed: %d, ignored: %d' %(total_ignored + total_disabled, total_disabled, total_ignored) + +if __name__ == '__main__': + cleanup() diff --git a/api/boto/mturk/test/create_free_text_question_regex.doctest b/api/boto/mturk/test/create_free_text_question_regex.doctest new file mode 100644 index 0000000..a10b7ed --- /dev/null +++ b/api/boto/mturk/test/create_free_text_question_regex.doctest @@ -0,0 +1,92 @@ +>>> import uuid +>>> import datetime +>>> from boto.mturk.connection import MTurkConnection +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer + +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + +# create content for a question +>>> qn_content = QuestionContent(title='Boto no hit type question content', +... text='What is a boto no hit type?') + +# create a free text answer that is not quite so free! +>>> ft_answer = FreeTextAnswer(format_regex=dict(regex="^[12][0-9]{3}-[01]?\d-[0-3]?\d$", +... error_text="You must enter a date with the format yyyy-mm-dd.", +... flags="i"), +... default="This is not a valid format") + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(ft_answer)) + +# now, create the actual HIT for the question without using a HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> keywords=['boto', 'test', 'doctest'] +>>> create_hit_rs = conn.create_hit(question=qn, +... lifetime=60*65, +... max_assignments=2, +... title='Boto create_hit title', +... description='Boto create_hit description', +... keywords=keywords, +... reward=0.23, +... duration=60*6, +... approval_delay=60*60, +... annotation='An annotation from boto create_hit test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +# for the requested hit type id +# the HIT Type Id is a unicode string +>>> hit_type_id = create_hit_rs.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +>>> create_hit_rs.MaxAssignments +u'2' + +>>> create_hit_rs.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime in seconds +>>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) +>>> expiration_datetime = datetime.datetime.strptime(create_hit_rs.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> delta.seconds < 5 +True + +# duration is as specified for the HIT type +>>> create_hit_rs.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly (allow for float error here) +>>> int(create_hit_rs[0].amount * 100) +23 + +>>> create_hit_rs[0].formatted_price +u'$0.23' + +# only US currency supported at present +>>> create_hit_rs[0].currency_code +u'USD' + +# title is the HIT type title +>>> create_hit_rs.Title +u'Boto create_hit title' + +# title is the HIT type description +>>> create_hit_rs.Description +u'Boto create_hit description' + +# annotation is correct +>>> create_hit_rs.RequesterAnnotation +u'An annotation from boto create_hit test' + +>>> create_hit_rs.HITReviewStatus +u'NotReviewed' diff --git a/api/boto/mturk/test/create_hit.doctest b/api/boto/mturk/test/create_hit.doctest new file mode 100644 index 0000000..22209d6 --- /dev/null +++ b/api/boto/mturk/test/create_hit.doctest @@ -0,0 +1,86 @@ +>>> import uuid +>>> import datetime +>>> from boto.mturk.connection import MTurkConnection +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer + +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + +# create content for a question +>>> qn_content = QuestionContent(title='Boto no hit type question content', +... text='What is a boto no hit type?') + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(FreeTextAnswer())) + +# now, create the actual HIT for the question without using a HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> keywords=['boto', 'test', 'doctest'] +>>> create_hit_rs = conn.create_hit(question=qn, +... lifetime=60*65, +... max_assignments=2, +... title='Boto create_hit title', +... description='Boto create_hit description', +... keywords=keywords, +... reward=0.23, +... duration=60*6, +... approval_delay=60*60, +... annotation='An annotation from boto create_hit test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +# for the requested hit type id +# the HIT Type Id is a unicode string +>>> hit_type_id = create_hit_rs.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +>>> create_hit_rs.MaxAssignments +u'2' + +>>> create_hit_rs.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime in seconds +>>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) +>>> expiration_datetime = datetime.datetime.strptime(create_hit_rs.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> delta.seconds < 5 +True + +# duration is as specified for the HIT type +>>> create_hit_rs.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly (allow for float error here) +>>> int(create_hit_rs[0].amount * 100) +23 + +>>> create_hit_rs[0].formatted_price +u'$0.23' + +# only US currency supported at present +>>> create_hit_rs[0].currency_code +u'USD' + +# title is the HIT type title +>>> create_hit_rs.Title +u'Boto create_hit title' + +# title is the HIT type description +>>> create_hit_rs.Description +u'Boto create_hit description' + +# annotation is correct +>>> create_hit_rs.RequesterAnnotation +u'An annotation from boto create_hit test' + +>>> create_hit_rs.HITReviewStatus +u'NotReviewed' diff --git a/api/boto/mturk/test/create_hit_binary.doctest b/api/boto/mturk/test/create_hit_binary.doctest new file mode 100644 index 0000000..3096083 --- /dev/null +++ b/api/boto/mturk/test/create_hit_binary.doctest @@ -0,0 +1,87 @@ +>>> import uuid +>>> import datetime +>>> from boto.mturk.connection import MTurkConnection +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer + +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + +# create content for a question +>>> qn_content = QuestionContent(title='Boto no hit type question content', +... text='What is a boto no hit type?', +... binary='http://www.example.com/test1.jpg') + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(FreeTextAnswer())) + +# now, create the actual HIT for the question without using a HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> keywords=['boto', 'test', 'doctest'] +>>> create_hit_rs = conn.create_hit(question=qn, +... lifetime=60*65, +... max_assignments=2, +... title='Boto create_hit title', +... description='Boto create_hit description', +... keywords=keywords, +... reward=0.23, +... duration=60*6, +... approval_delay=60*60, +... annotation='An annotation from boto create_hit test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +# for the requested hit type id +# the HIT Type Id is a unicode string +>>> hit_type_id = create_hit_rs.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +>>> create_hit_rs.MaxAssignments +u'2' + +>>> create_hit_rs.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime in seconds +>>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) +>>> expiration_datetime = datetime.datetime.strptime(create_hit_rs.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> delta.seconds < 5 +True + +# duration is as specified for the HIT type +>>> create_hit_rs.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly (allow for float error here) +>>> int(create_hit_rs[0].amount * 100) +23 + +>>> create_hit_rs[0].formatted_price +u'$0.23' + +# only US currency supported at present +>>> create_hit_rs[0].currency_code +u'USD' + +# title is the HIT type title +>>> create_hit_rs.Title +u'Boto create_hit title' + +# title is the HIT type description +>>> create_hit_rs.Description +u'Boto create_hit description' + +# annotation is correct +>>> create_hit_rs.RequesterAnnotation +u'An annotation from boto create_hit test' + +>>> create_hit_rs.HITReviewStatus +u'NotReviewed' diff --git a/api/boto/mturk/test/create_hit_external.py b/api/boto/mturk/test/create_hit_external.py new file mode 100644 index 0000000..e7425d6 --- /dev/null +++ b/api/boto/mturk/test/create_hit_external.py @@ -0,0 +1,14 @@ +import uuid +import datetime +from boto.mturk.connection import MTurkConnection +from boto.mturk.question import ExternalQuestion + +def test(): + q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800) + conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + keywords=['boto', 'test', 'doctest'] + create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', response_groups=['Minimal','HITDetail','HITQuestion','HITAssignmentSummary',]) + assert(create_hit_rs.status == True) + +if __name__ == "__main__": + test() diff --git a/api/boto/mturk/test/create_hit_from_hit_type.doctest b/api/boto/mturk/test/create_hit_from_hit_type.doctest new file mode 100644 index 0000000..144a677 --- /dev/null +++ b/api/boto/mturk/test/create_hit_from_hit_type.doctest @@ -0,0 +1,97 @@ +>>> import uuid +>>> import datetime +>>> from boto.mturk.connection import MTurkConnection +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer +>>> +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') +>>> keywords=['boto', 'test', 'doctest'] +>>> hit_type_rs = conn.register_hit_type('Boto Test HIT type', +... 'HIT Type for testing Boto', +... 0.12, +... 60*6, +... keywords=keywords, +... approval_delay=60*60) + +# this was a valid request +>>> hit_type_rs.status +True + +# the HIT Type Id is a unicode string +>>> hit_type_id = hit_type_rs.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +# create content for a question +>>> qn_content = QuestionContent(title='Boto question content create_hit_from_hit_type', +... text='What is a boto create_hit_from_hit_type?') + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(FreeTextAnswer())) + +# now, create the actual HIT for the question using the HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> create_hit_rs = conn.create_hit(hit_type=hit_type_rs.HITTypeId, +... question=qn, +... lifetime=60*65, +... max_assignments=2, +... annotation='An annotation from boto create_hit_from_hit_type test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +# for the requested hit type id +>>> create_hit_rs.HITTypeId == hit_type_id +True + +# with the correct number of maximum assignments +>>> create_hit_rs.MaxAssignments +u'2' + +# and the approval delay +>>> create_hit_rs.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime in seconds +>>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) +>>> expiration_datetime = datetime.datetime.strptime(create_hit_rs.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> delta.seconds < 5 +True + +# duration is as specified for the HIT type +>>> create_hit_rs.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly +>>> create_hit_rs[0].amount +0.12 + +>>> create_hit_rs[0].formatted_price +u'$0.12' + +# only US currency supported at present +>>> create_hit_rs[0].currency_code +u'USD' + +# title is the HIT type title +>>> create_hit_rs.Title +u'Boto Test HIT type' + +# title is the HIT type description +>>> create_hit_rs.Description +u'HIT Type for testing Boto' + +# annotation is correct +>>> create_hit_rs.RequesterAnnotation +u'An annotation from boto create_hit_from_hit_type test' + +# not reviewed yet +>>> create_hit_rs.HITReviewStatus +u'NotReviewed' diff --git a/api/boto/mturk/test/create_hit_with_qualifications.py b/api/boto/mturk/test/create_hit_with_qualifications.py new file mode 100644 index 0000000..f2149ee --- /dev/null +++ b/api/boto/mturk/test/create_hit_with_qualifications.py @@ -0,0 +1,18 @@ +import uuid +import datetime +from boto.mturk.connection import MTurkConnection +from boto.mturk.question import ExternalQuestion +from boto.mturk.qualification import Qualifications, PercentAssignmentsApprovedRequirement + +def test(): + q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800) + conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + keywords=['boto', 'test', 'doctest'] + qualifications = Qualifications() + qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95")) + create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications) + assert(create_hit_rs.status == True) + print create_hit_rs.HITTypeId + +if __name__ == "__main__": + test() diff --git a/api/boto/mturk/test/reviewable_hits.doctest b/api/boto/mturk/test/reviewable_hits.doctest new file mode 100644 index 0000000..0305901 --- /dev/null +++ b/api/boto/mturk/test/reviewable_hits.doctest @@ -0,0 +1,71 @@ +>>> from boto.mturk.connection import MTurkConnection +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + +# should have some reviewable HIT's returned, especially if returning all HIT type's +# NOTE: but only if your account has existing HIT's in the reviewable state +>>> reviewable_rs = conn.get_reviewable_hits() + +# this is a valid request +>>> reviewable_rs.status +True + +>>> len(reviewable_rs) > 1 +True + +# should contain at least one HIT object +>>> reviewable_rs # doctest: +ELLIPSIS +[>> hit_id = reviewable_rs[0].HITId + +# check that we can retrieve the assignments for a HIT +>>> assignments_rs = conn.get_assignments(hit_id) + +# this is a valid request +>>> assignments_rs.status +True + +>>> assignments_rs.NumResults >= 1 +True + +>>> len(assignments_rs) == int(assignments_rs.NumResults) +True + +>>> assignments_rs.PageNumber +u'1' + +>>> assignments_rs.TotalNumResults >= 1 +True + +# should contain at least one Assignment object +>>> assignments_rs # doctest: +ELLIPSIS +[>> assignment = assignments_rs[0] + +>>> assignment.HITId == hit_id +True + +# should have a valid status +>>> assignment.AssignmentStatus in ['Submitted', 'Approved', 'Rejected'] +True + +# should have returned at least one answer +>>> len(assignment.answers) > 0 +True + +# should contain at least one set of QuestionFormAnswer objects +>>> assignment.answers # doctest: +ELLIPSIS +[[>> answer = assignment.answers[0][0] + +# answer should be a FreeTextAnswer +>>> answer.FreeText # doctest: +ELLIPSIS +u'...' + +# question identifier should be a unicode string +>>> answer.QuestionIdentifier # doctest: +ELLIPSIS +u'...' + diff --git a/api/boto/mturk/test/search_hits.doctest b/api/boto/mturk/test/search_hits.doctest new file mode 100644 index 0000000..a2547ea --- /dev/null +++ b/api/boto/mturk/test/search_hits.doctest @@ -0,0 +1,16 @@ +>>> from boto.mturk.connection import MTurkConnection +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + +# should have some HIT's returned by a search (but only if your account has existing HIT's) +>>> search_rs = conn.search_hits() + +# this is a valid request +>>> search_rs.status +True + +>>> len(search_rs) > 1 +True + +>>> search_rs # doctest: +ELLIPSIS +[= 0: + method, version = update.split(':') + version = '-r%s' % version + else: + version = '-rHEAD' + location = boto.config.get('Boto', 'boto_location', '/usr/local/boto') + self.run('svn update %s %s' % (version, location)) + else: + # first remove the symlink needed when running from subversion + self.run('rm /usr/local/lib/python2.5/site-packages/boto') + self.run('easy_install %s' % update) + + def fetch_s3_file(self, s3_file): + try: + if s3_file.startswith('s3:'): + bucket_name, key_name = s3_file[len('s3:'):].split('/') + c = boto.connect_s3() + bucket = c.get_bucket(bucket_name) + key = bucket.get_key(key_name) + boto.log.info('Fetching %s/%s' % (bucket.name, key.name)) + path = os.path.join(self.working_dir, key.name) + key.get_contents_to_filename(path) + except: + boto.log.exception('Problem Retrieving file: %s' % s3_file) + path = None + return path + + def load_packages(self): + package_str = boto.config.get('Pyami', 'packages') + if package_str: + packages = package_str.split(',') + for package in packages: + package = package.strip() + if package.startswith('s3:'): + package = self.fetch_s3_file(package) + if package: + # if the "package" is really a .py file, it doesn't have to + # be installed, just being in the working dir is enough + if not package.endswith('.py'): + self.run('easy_install -Z %s' % package, exit_on_error=False) + + def main(self): + self.create_working_dir() + self.load_boto() + self.load_packages() + self.notify('Bootstrap Completed for %s' % boto.config.get_instance('instance-id')) + +if __name__ == "__main__": + # because bootstrap starts before any logging configuration can be loaded from + # the boto config files, we will manually enable logging to /var/log/boto.log + boto.set_file_logger('bootstrap', '/var/log/boto.log') + bs = Bootstrap() + bs.main() diff --git a/api/boto/pyami/config.py b/api/boto/pyami/config.py new file mode 100644 index 0000000..831acc4 --- /dev/null +++ b/api/boto/pyami/config.py @@ -0,0 +1,204 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import StringIO, os, re +import ConfigParser +import boto + +BotoConfigPath = '/etc/boto.cfg' +BotoConfigLocations = [BotoConfigPath] +if 'HOME' in os.environ: + UserConfigPath = os.path.expanduser('~/.boto') + BotoConfigLocations.append(UserConfigPath) +else: + UserConfigPath = None +if 'BOTO_CONFIG' in os.environ: + BotoConfigLocations.append(os.path.expanduser(os.environ['BOTO_CONFIG'])) + +class Config(ConfigParser.SafeConfigParser): + + def __init__(self, path=None, fp=None, do_load=True): + ConfigParser.SafeConfigParser.__init__(self, {'working_dir' : '/mnt/pyami', + 'debug' : '0'}) + if do_load: + if path: + self.load_from_path(path) + elif fp: + self.readfp(fp) + else: + self.read(BotoConfigLocations) + if "AWS_CREDENTIAL_FILE" in os.environ: + self.load_credential_file(os.path.expanduser(os.environ['AWS_CREDENTIAL_FILE'])) + + def load_credential_file(self, path): + """Load a credential file as is setup like the Java utilities""" + config = ConfigParser.ConfigParser() + c_data = StringIO.StringIO() + c_data.write("[Credentials]\n") + for line in open(path, "r").readlines(): + c_data.write(line.replace("AWSAccessKeyId", "aws_access_key_id").replace("AWSSecretKey", "aws_secret_access_key")) + c_data.seek(0) + self.readfp(c_data) + + def load_from_path(self, path): + file = open(path) + for line in file.readlines(): + match = re.match("^#import[\s\t]*([^\s^\t]*)[\s\t]*$", line) + if match: + extended_file = match.group(1) + (dir, file) = os.path.split(path) + self.load_from_path(os.path.join(dir, extended_file)) + self.read(path) + + def save_option(self, path, section, option, value): + """ + Write the specified Section.Option to the config file specified by path. + Replace any previous value. If the path doesn't exist, create it. + Also add the option the the in-memory config. + """ + config = ConfigParser.SafeConfigParser() + config.read(path) + if not config.has_section(section): + config.add_section(section) + config.set(section, option, value) + fp = open(path, 'w') + config.write(fp) + fp.close() + if not self.has_section(section): + self.add_section(section) + self.set(section, option, value) + + def save_user_option(self, section, option, value): + self.save_option(UserConfigPath, section, option, value) + + def save_system_option(self, section, option, value): + self.save_option(BotoConfigPath, section, option, value) + + def get_instance(self, name, default=None): + try: + val = self.get('Instance', name) + except: + val = default + return val + + def get_user(self, name, default=None): + try: + val = self.get('User', name) + except: + val = default + return val + + def getint_user(self, name, default=0): + try: + val = self.getint('User', name) + except: + val = default + return val + + def get_value(self, section, name, default=None): + return self.get(section, name, default) + + def get(self, section, name, default=None): + try: + val = ConfigParser.SafeConfigParser.get(self, section, name) + except: + val = default + return val + + def getint(self, section, name, default=0): + try: + val = ConfigParser.SafeConfigParser.getint(self, section, name) + except: + val = int(default) + return val + + def getfloat(self, section, name, default=0.0): + try: + val = ConfigParser.SafeConfigParser.getfloat(self, section, name) + except: + val = float(default) + return val + + def getbool(self, section, name, default=False): + if self.has_option(section, name): + val = self.get(section, name) + if val.lower() == 'true': + val = True + else: + val = False + else: + val = default + return val + + def setbool(self, section, name, value): + if value: + self.set(section, name, 'true') + else: + self.set(section, name, 'false') + + def dump(self): + s = StringIO.StringIO() + self.write(s) + print s.getvalue() + + def dump_safe(self, fp=None): + if not fp: + fp = StringIO.StringIO() + for section in self.sections(): + fp.write('[%s]\n' % section) + for option in self.options(section): + if option == 'aws_secret_access_key': + fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option) + else: + fp.write('%s = %s\n' % (option, self.get(section, option))) + + def dump_to_sdb(self, domain_name, item_name): + import simplejson + sdb = boto.connect_sdb() + domain = sdb.lookup(domain_name) + if not domain: + domain = sdb.create_domain(domain_name) + item = domain.new_item(item_name) + item.active = False + for section in self.sections(): + d = {} + for option in self.options(section): + d[option] = self.get(section, option) + item[section] = simplejson.dumps(d) + item.save() + + def load_from_sdb(self, domain_name, item_name): + import simplejson + sdb = boto.connect_sdb() + domain = sdb.lookup(domain_name) + item = domain.get_item(item_name) + for section in item.keys(): + if not self.has_section(section): + self.add_section(section) + d = simplejson.loads(item[section]) + for attr_name in d.keys(): + attr_value = d[attr_name] + if attr_value == None: + attr_value = 'None' + if isinstance(attr_value, bool): + self.setbool(section, attr_name, attr_value) + else: + self.set(section, attr_name, attr_value) diff --git a/api/boto/pyami/copybot.cfg b/api/boto/pyami/copybot.cfg new file mode 100644 index 0000000..cbfdc5a --- /dev/null +++ b/api/boto/pyami/copybot.cfg @@ -0,0 +1,60 @@ +# +# Your AWS Credentials +# +[Credentials] +aws_access_key_id = +aws_secret_access_key = + +# +# If you want to use a separate set of credentials when writing +# to the destination bucket, put them here +#dest_aws_access_key_id = +#dest_aws_secret_access_key = + +# +# Fill out this section if you want emails from CopyBot +# when it starts and stops +# +[Notification] +#smtp_host = +#smtp_user = +#smtp_pass = +#smtp_from = +#smtp_to = + +# +# If you leave this section as is, it will automatically +# update boto from subversion upon start up. +# If you don't want that to happen, comment this out +# +[Boto] +boto_location = /usr/local/boto +boto_update = svn:HEAD + +# +# This tells the Pyami code in boto what scripts +# to run during startup +# +[Pyami] +scripts = boto.pyami.copybot.CopyBot + +# +# Source bucket and Destination Bucket, obviously. +# If the Destination bucket does not exist, it will +# attempt to create it. +# If exit_on_completion is false, the instance +# will keep running after the copy operation is +# complete which might be handy for debugging. +# If copy_acls is false, the ACL's will not be +# copied with the objects to the new bucket. +# If replace_dst is false, copybot will not +# will only store the source file in the dest if +# that file does not already exist. If it's true +# it will replace it even if it does exist. +# +[CopyBot] +src_bucket = +dst_bucket = +exit_on_completion = true +copy_acls = true +replace_dst = true diff --git a/api/boto/pyami/copybot.py b/api/boto/pyami/copybot.py new file mode 100644 index 0000000..ed397cb --- /dev/null +++ b/api/boto/pyami/copybot.py @@ -0,0 +1,97 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import boto +from boto.pyami.scriptbase import ScriptBase +import os, StringIO + +class CopyBot(ScriptBase): + + def __init__(self): + ScriptBase.__init__(self) + self.wdir = boto.config.get('Pyami', 'working_dir') + self.log_file = '%s.log' % self.instance_id + self.log_path = os.path.join(self.wdir, self.log_file) + boto.set_file_logger(self.name, self.log_path) + self.src_name = boto.config.get(self.name, 'src_bucket') + self.dst_name = boto.config.get(self.name, 'dst_bucket') + self.replace = boto.config.getbool(self.name, 'replace_dst', True) + s3 = boto.connect_s3() + self.src = s3.lookup(self.src_name) + if not self.src: + boto.log.error('Source bucket does not exist: %s' % self.src_name) + dest_access_key = boto.config.get(self.name, 'dest_aws_access_key_id', None) + if dest_access_key: + dest_secret_key = boto.config.get(self.name, 'dest_aws_secret_access_key', None) + s3 = boto.connect(dest_access_key, dest_secret_key) + self.dst = s3.lookup(self.dst_name) + if not self.dst: + self.dst = s3.create_bucket(self.dst_name) + + def copy_bucket_acl(self): + if boto.config.get(self.name, 'copy_acls', True): + acl = self.src.get_xml_acl() + self.dst.set_xml_acl(acl) + + def copy_key_acl(self, src, dst): + if boto.config.get(self.name, 'copy_acls', True): + acl = src.get_xml_acl() + dst.set_xml_acl(acl) + + def copy_keys(self): + boto.log.info('src=%s' % self.src.name) + boto.log.info('dst=%s' % self.dst.name) + try: + for key in self.src: + if not self.replace: + exists = self.dst.lookup(key.name) + if exists: + boto.log.info('key=%s already exists in %s, skipping' % (key.name, self.dst.name)) + continue + boto.log.info('copying %d bytes from key=%s' % (key.size, key.name)) + prefix, base = os.path.split(key.name) + path = os.path.join(self.wdir, base) + key.get_contents_to_filename(path) + new_key = self.dst.new_key(key.name) + new_key.set_contents_from_filename(path) + self.copy_key_acl(key, new_key) + os.unlink(path) + except: + boto.log.exception('Error copying key: %s' % key.name) + + def copy_log(self): + key = self.dst.new_key(self.log_file) + key.set_contents_from_filename(self.log_path) + + def main(self): + fp = StringIO.StringIO() + boto.config.dump_safe(fp) + self.notify('%s (%s) Starting' % (self.name, self.instance_id), fp.getvalue()) + if self.src and self.dst: + self.copy_keys() + if self.dst: + self.copy_log() + self.notify('%s (%s) Stopping' % (self.name, self.instance_id), + 'Copy Operation Complete') + if boto.config.getbool(self.name, 'exit_on_completion', True): + ec2 = boto.connect_ec2() + ec2.terminate_instances([self.instance_id]) + diff --git a/api/boto/pyami/helloworld.py b/api/boto/pyami/helloworld.py new file mode 100644 index 0000000..680873c --- /dev/null +++ b/api/boto/pyami/helloworld.py @@ -0,0 +1,28 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.scriptbase import ScriptBase + +class HelloWorld(ScriptBase): + + def main(self): + self.log('Hello World!!!') + diff --git a/api/boto/pyami/installers/__init__.py b/api/boto/pyami/installers/__init__.py new file mode 100644 index 0000000..4dcf2f4 --- /dev/null +++ b/api/boto/pyami/installers/__init__.py @@ -0,0 +1,64 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.scriptbase import ScriptBase +import os + +class Installer(ScriptBase): + """ + Abstract base class for installers + """ + + def add_cron(self, name, minute, hour, mday, month, wday, who, command, env=None): + """ + Add an entry to the system crontab. + """ + raise NotImplimented() + + def add_init_script(self, file): + """ + Add this file to the init.d directory + """ + + def add_env(self, key, value): + """ + Add an environemnt variable + """ + raise NotImplimented() + + def stop(self, service_name): + """ + Stop a service. + """ + raise NotImplimented() + + def start(self, service_name): + """ + Start a service. + """ + raise NotImplimented() + + def install(self): + """ + Do whatever is necessary to "install" the package. + """ + raise NotImplimented() + diff --git a/api/boto/pyami/installers/ubuntu/__init__.py b/api/boto/pyami/installers/ubuntu/__init__.py new file mode 100644 index 0000000..60ee658 --- /dev/null +++ b/api/boto/pyami/installers/ubuntu/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/api/boto/pyami/installers/ubuntu/apache.py b/api/boto/pyami/installers/ubuntu/apache.py new file mode 100644 index 0000000..febc2df --- /dev/null +++ b/api/boto/pyami/installers/ubuntu/apache.py @@ -0,0 +1,43 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.installers.ubuntu.installer import Installer + +class Apache(Installer): + """ + Install apache2, mod_python, and libapache2-svn + """ + + def install(self): + self.run("apt-get update") + self.run('apt-get -y install apache2', notify=True, exit_on_error=True) + self.run('apt-get -y install libapache2-mod-python', notify=True, exit_on_error=True) + self.run('a2enmod rewrite', notify=True, exit_on_error=True) + self.run('a2enmod ssl', notify=True, exit_on_error=True) + self.run('a2enmod proxy', notify=True, exit_on_error=True) + self.run('a2enmod proxy_ajp', notify=True, exit_on_error=True) + + # Hard reboot the apache2 server to enable these module + self.stop("apache2") + self.start("apache2") + + def main(self): + self.install() diff --git a/api/boto/pyami/installers/ubuntu/ebs.py b/api/boto/pyami/installers/ubuntu/ebs.py new file mode 100644 index 0000000..2cf0f22 --- /dev/null +++ b/api/boto/pyami/installers/ubuntu/ebs.py @@ -0,0 +1,203 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +Automated installer to attach, format and mount an EBS volume. +This installer assumes that you want the volume formatted as +an XFS file system. To drive this installer, you need the +following section in the boto config passed to the new instance. +You also need to install dateutil by listing python-dateutil +in the list of packages to be installed in the Pyami seciont +of your boto config file. + +If there is already a device mounted at the specified mount point, +the installer assumes that it is the ephemeral drive and unmounts +it, remounts it as /tmp and chmods it to 777. + +Config file section:: + + [EBS] + volume_id = + logical_volume_name = + device = + mount_point = + +""" +import boto +from boto.manage.volume import Volume +import os, time +from boto.pyami.installers.ubuntu.installer import Installer +from string import Template + +BackupScriptTemplate = """#!/usr/bin/env python +# Backup EBS volume +import boto +from boto.pyami.scriptbase import ScriptBase +import traceback + +class Backup(ScriptBase): + + def main(self): + try: + ec2 = boto.connect_ec2() + self.run("/usr/sbin/xfs_freeze -f ${mount_point}") + snapshot = ec2.create_snapshot('${volume_id}') + boto.log.info("Snapshot created: %s " % snapshot) + except Exception, e: + self.notify(subject="${instance_id} Backup Failed", body=traceback.format_exc()) + boto.log.info("Snapshot created: ${volume_id}") + except Exception, e: + self.notify(subject="${instance_id} Backup Failed", body=traceback.format_exc()) + finally: + self.run("/usr/sbin/xfs_freeze -u ${mount_point}") + +if __name__ == "__main__": + b = Backup() + b.main() +""" + +BackupCleanupScript= """#!/usr/bin/env python +# Cleans Backups of EBS volumes + +for v in Volume.all(): + v.trim_snapshot(True) +""" + +class EBSInstaller(Installer): + """ + Set up the EBS stuff + """ + + def __init__(self, config_file=None): + Installer.__init__(self, config_file) + self.instance_id = boto.config.get('Instance', 'instance-id') + self.device = boto.config.get('EBS', 'device', '/dev/sdp') + self.volume_id = boto.config.get('EBS', 'volume_id') + self.logical_volume_name = boto.config.get('EBS', 'logical_volume_name') + self.mount_point = boto.config.get('EBS', 'mount_point', '/ebs') + + def attach(self): + ec2 = boto.connect_ec2() + if self.logical_volume_name: + # if a logical volume was specified, override the specified volume_id + # (if there was one) with the current AWS volume for the logical volume: + logical_volume = Volume.find(name = self.logical_volume_name).next() + self.volume_id = logical_volume._volume_id + volume = ec2.get_all_volumes([self.volume_id])[0] + # wait for the volume to be available. The volume may still be being created + # from a snapshot. + while volume.update() != 'available': + boto.log.info('Volume %s not yet available. Current status = %s.' % (volume.id, volume.status)) + time.sleep(5) + ec2.attach_volume(self.volume_id, self.instance_id, self.device) + # now wait for the volume device to appear + while not os.path.exists(self.device): + boto.log.info('%s still does not exist, waiting 10 seconds' % self.device) + time.sleep(10) + + def make_fs(self): + boto.log.info('make_fs...') + has_fs = self.run('fsck %s' % self.device) + if has_fs != 0: + self.run('mkfs -t xfs %s' % self.device) + + def create_backup_script(self): + t = Template(BackupScriptTemplate) + s = t.substitute(volume_id=self.volume_id, instance_id=self.instance_id, + mount_point=self.mount_point) + fp = open('/usr/local/bin/ebs_backup', 'w') + fp.write(s) + fp.close() + self.run('chmod +x /usr/local/bin/ebs_backup') + + def create_backup_cleanup_script(self): + fp = open('/usr/local/bin/ebs_backup_cleanup', 'w') + fp.write(BackupCleanupScript) + fp.close() + self.run('chmod +x /usr/local/bin/ebs_backup_cleanup') + + def handle_mount_point(self): + boto.log.info('handle_mount_point') + if not os.path.isdir(self.mount_point): + boto.log.info('making directory') + # mount directory doesn't exist so create it + self.run("mkdir %s" % self.mount_point) + else: + boto.log.info('directory exists already') + self.run('mount -l') + lines = self.last_command.output.split('\n') + for line in lines: + t = line.split() + if t and t[2] == self.mount_point: + # something is already mounted at the mount point + # unmount that and mount it as /tmp + if t[0] != self.device: + self.run('umount %s' % self.mount_point) + self.run('mount %s /tmp' % t[0]) + self.run('chmod 777 /tmp') + break + # Mount up our new EBS volume onto mount_point + self.run("mount %s %s" % (self.device, self.mount_point)) + self.run('xfs_growfs %s' % self.mount_point) + + def update_fstab(self): + f = open("/etc/fstab", "a") + f.write('%s\t%s\txfs\tdefaults 0 0\n' % (self.mount_point, self.device)) + f.close() + + def install(self): + # First, find and attach the volume + self.attach() + + # Install the xfs tools + self.run('apt-get -y install xfsprogs xfsdump') + + # Check to see if the filesystem was created or not + self.make_fs() + + # create the /ebs directory for mounting + self.handle_mount_point() + + # create the backup script + self.create_backup_script() + + # Set up the backup script + minute = boto.config.get('EBS', 'backup_cron_minute', '0') + hour = boto.config.get('EBS', 'backup_cron_hour', '4,16') + self.add_cron("ebs_backup", "/usr/local/bin/ebs_backup", minute=minute, hour=hour) + + # Set up the backup cleanup script + minute = boto.config.get('EBS', 'backup_cleanup_cron_minute') + hour = boto.config.get('EBS', 'backup_cleanup_cron_hour') + if (minute != None) and (hour != None): + self.create_backup_cleanup_script(); + self.add_cron("ebs_backup_cleanup", "/usr/local/bin/ebs_backup_cleanup", minute=minute, hour=hour) + + # Set up the fstab + self.update_fstab() + + def main(self): + if not os.path.exists(self.device): + self.install() + else: + boto.log.info("Device %s is already attached, skipping EBS Installer" % self.device) diff --git a/api/boto/pyami/installers/ubuntu/installer.py b/api/boto/pyami/installers/ubuntu/installer.py new file mode 100644 index 0000000..0169950 --- /dev/null +++ b/api/boto/pyami/installers/ubuntu/installer.py @@ -0,0 +1,96 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import boto.pyami.installers +import os +import os.path +import stat +import boto +import random +from pwd import getpwnam + +class Installer(boto.pyami.installers.Installer): + """ + Base Installer class for Ubuntu-based AMI's + """ + def add_cron(self, name, command, minute="*", hour="*", mday="*", month="*", wday="*", who="root", env=None): + """ + Write a file to /etc/cron.d to schedule a command + env is a dict containing environment variables you want to set in the file + name will be used as the name of the file + """ + if minute == 'random': + minute = str(random.randrange(60)) + if hour == 'random': + hour = str(random.randrange(24)) + fp = open('/etc/cron.d/%s' % name, "w") + if env: + for key, value in env.items(): + fp.write('%s=%s\n' % (key, value)) + fp.write('%s %s %s %s %s %s %s\n' % (minute, hour, mday, month, wday, who, command)) + fp.close() + + def add_init_script(self, file, name): + """ + Add this file to the init.d directory + """ + f_path = os.path.join("/etc/init.d", name) + f = open(f_path, "w") + f.write(file) + f.close() + os.chmod(f_path, stat.S_IREAD| stat.S_IWRITE | stat.S_IEXEC) + self.run("/usr/sbin/update-rc.d %s defaults" % name) + + def add_env(self, key, value): + """ + Add an environemnt variable + For Ubuntu, the best place is /etc/environment. Values placed here do + not need to be exported. + """ + boto.log.info('Adding env variable: %s=%s' % (key, value)) + if not os.path.exists("/etc/environment.orig"): + self.run('cp /etc/environment /etc/environment.orig', notify=False, exit_on_error=False) + fp = open('/etc/environment', 'a') + fp.write('\n%s="%s"' % (key, value)) + fp.close() + os.environ[key] = value + + def stop(self, service_name): + self.run('/etc/init.d/%s stop' % service_name) + + def start(self, service_name): + self.run('/etc/init.d/%s start' % service_name) + + def create_user(self, user): + """ + Create a user on the local system + """ + self.run("useradd -m %s" % user) + usr = getpwnam(user) + return usr + + + def install(self): + """ + This is the only method you need to override + """ + raise NotImplimented() + diff --git a/api/boto/pyami/installers/ubuntu/mysql.py b/api/boto/pyami/installers/ubuntu/mysql.py new file mode 100644 index 0000000..490e5db --- /dev/null +++ b/api/boto/pyami/installers/ubuntu/mysql.py @@ -0,0 +1,109 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This installer will install mysql-server on an Ubuntu machine. +In addition to the normal installation done by apt-get, it will +also configure the new MySQL server to store it's data files in +a different location. By default, this is /mnt but that can be +configured in the [MySQL] section of the boto config file passed +to the instance. +""" +from boto.pyami.installers.ubuntu.installer import Installer +import os +import boto +from boto.utils import ShellCommand +from ConfigParser import SafeConfigParser +import time + +ConfigSection = """ +[MySQL] +root_password = +data_dir = +""" + +class MySQL(Installer): + + def install(self): + self.run('apt-get update') + self.run('apt-get -y install mysql-server', notify=True, exit_on_error=True) + +# def set_root_password(self, password=None): +# if not password: +# password = boto.config.get('MySQL', 'root_password') +# if password: +# self.run('mysqladmin -u root password %s' % password) +# return password + + def change_data_dir(self, password=None): + data_dir = boto.config.get('MySQL', 'data_dir', '/mnt') + fresh_install = False; + is_mysql_running_command = ShellCommand('mysqladmin ping') # exit status 0 if mysql is running + is_mysql_running_command.run() + if is_mysql_running_command.getStatus() == 0: + # mysql is running. This is the state apt-get will leave it in. If it isn't running, + # that means mysql was already installed on the AMI and there's no need to stop it, + # saving 40 seconds on instance startup. + time.sleep(10) #trying to stop mysql immediately after installing it fails + # We need to wait until mysql creates the root account before we kill it + # or bad things will happen + i = 0 + while self.run("echo 'quit' | mysql -u root") != 0 and i<5: + time.sleep(5) + i = i + 1 + self.run('/etc/init.d/mysql stop') + self.run("pkill -9 mysql") + + mysql_path = os.path.join(data_dir, 'mysql') + if not os.path.exists(mysql_path): + self.run('mkdir %s' % mysql_path) + fresh_install = True; + self.run('chown -R mysql:mysql %s' % mysql_path) + fp = open('/etc/mysql/conf.d/use_mnt.cnf', 'w') + fp.write('# created by pyami\n') + fp.write('# use the %s volume for data\n' % data_dir) + fp.write('[mysqld]\n') + fp.write('datadir = %s\n' % mysql_path) + fp.write('log_bin = %s\n' % os.path.join(mysql_path, 'mysql-bin.log')) + fp.close() + if fresh_install: + self.run('cp -pr /var/lib/mysql/* %s/' % mysql_path) + self.start('mysql') + else: + #get the password ubuntu expects to use: + config_parser = SafeConfigParser() + config_parser.read('/etc/mysql/debian.cnf') + password = config_parser.get('client', 'password') + # start the mysql deamon, then mysql with the required grant statement piped into it: + self.start('mysql') + time.sleep(10) #time for mysql to start + grant_command = "echo \"GRANT ALL PRIVILEGES ON *.* TO 'debian-sys-maint'@'localhost' IDENTIFIED BY '%s' WITH GRANT OPTION;\" | mysql" % password + while self.run(grant_command) != 0: + time.sleep(5) + # leave mysqld running + + def main(self): + self.install() + # change_data_dir runs 'mysql -u root' which assumes there is no mysql password, i + # and changing that is too ugly to be worth it: + #self.set_root_password() + self.change_data_dir() + diff --git a/api/boto/pyami/installers/ubuntu/trac.py b/api/boto/pyami/installers/ubuntu/trac.py new file mode 100644 index 0000000..c97ddd2 --- /dev/null +++ b/api/boto/pyami/installers/ubuntu/trac.py @@ -0,0 +1,137 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.installers.ubuntu.installer import Installer +import boto +import os + +class Trac(Installer): + """ + Install Trac and DAV-SVN + Sets up a Vhost pointing to [Trac]->home + Using the config parameter [Trac]->hostname + Sets up a trac environment for every directory found under [Trac]->data_dir + + [Trac] + name = My Foo Server + hostname = trac.foo.com + home = /mnt/sites/trac + data_dir = /mnt/trac + svn_dir = /mnt/subversion + server_admin = root@foo.com + sdb_auth_domain = users + # Optional + SSLCertificateFile = /mnt/ssl/foo.crt + SSLCertificateKeyFile = /mnt/ssl/foo.key + SSLCertificateChainFile = /mnt/ssl/FooCA.crt + + """ + + def install(self): + self.run('apt-get -y install trac', notify=True, exit_on_error=True) + self.run('apt-get -y install libapache2-svn', notify=True, exit_on_error=True) + self.run("a2enmod ssl") + self.run("a2enmod python") + self.run("a2enmod dav_svn") + self.run("a2enmod rewrite") + + def setup_vhost(self): + domain = boto.config.get("Trac", "hostname").strip() + if domain: + cnf = open("/etc/apache2/sites-available/%s" % domain, "w") + cnf.write("NameVirtualHost *:80\n") + if boto.config.get("Trac", "SSLCertificateFile"): + cnf.write("NameVirtualHost *:443\n\n") + cnf.write("\n") + cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) + cnf.write("\tServerName %s\n" % domain) + cnf.write("\tRewriteEngine On\n") + cnf.write("\tRewriteRule ^(.*)$ https://%s$1\n" % domain) + cnf.write("\n\n") + + cnf.write("\n") + else: + cnf.write("\n") + + cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) + cnf.write("\tServerName %s\n" % domain) + cnf.write("\tDocumentRoot %s\n" % boto.config.get("Trac", "home").strip()) + + cnf.write("\t\n" % boto.config.get("Trac", "home").strip()) + cnf.write("\t\tOptions FollowSymLinks Indexes MultiViews\n") + cnf.write("\t\tAllowOverride All\n") + cnf.write("\t\tOrder allow,deny\n") + cnf.write("\t\tallow from all\n") + cnf.write("\t\n") + + cnf.write("\t\n") + cnf.write("\t\tAuthType Basic\n") + cnf.write("\t\tAuthName \"%s\"\n" % boto.config.get("Trac", "name")) + cnf.write("\t\tRequire valid-user\n") + cnf.write("\t\tAuthBasicAuthoritative off\n") + cnf.write("\t\tAuthUserFile /dev/null\n") + cnf.write("\t\tPythonAuthenHandler marajo.web.authen_handler\n") + cnf.write("\t\tPythonOption SDBDomain %s\n" % boto.config.get("Trac", "sdb_auth_domain")) + cnf.write("\t\n") + + data_dir = boto.config.get("Trac", "data_dir") + for env in os.listdir(data_dir): + if(env[0] != "."): + cnf.write("\t\n" % env) + cnf.write("\t\tSetHandler mod_python\n") + cnf.write("\t\tPythonInterpreter main_interpreter\n") + cnf.write("\t\tPythonHandler trac.web.modpython_frontend\n") + cnf.write("\t\tPythonOption TracEnv %s/%s\n" % (data_dir, env)) + cnf.write("\t\tPythonOption TracUriRoot /trac%s\n" % env) + cnf.write("\t\n") + + svn_dir = boto.config.get("Trac", "svn_dir") + for env in os.listdir(svn_dir): + if(env[0] != "."): + cnf.write("\t\n" % env) + cnf.write("\t\tDAV svn\n") + cnf.write("\t\tSVNPath %s/%s\n" % (svn_dir, env)) + cnf.write("\t\n") + + cnf.write("\tErrorLog /var/log/apache2/error.log\n") + cnf.write("\tLogLevel warn\n") + cnf.write("\tCustomLog /var/log/apache2/access.log combined\n") + cnf.write("\tServerSignature On\n") + SSLCertificateFile = boto.config.get("Trac", "SSLCertificateFile") + if SSLCertificateFile: + cnf.write("\tSSLEngine On\n") + cnf.write("\tSSLCertificateFile %s\n" % SSLCertificateFile) + + SSLCertificateKeyFile = boto.config.get("Trac", "SSLCertificateKeyFile") + if SSLCertificateKeyFile: + cnf.write("\tSSLCertificateKeyFile %s\n" % SSLCertificateKeyFile) + + SSLCertificateChainFile = boto.config.get("Trac", "SSLCertificateChainFile") + if SSLCertificateChainFile: + cnf.write("\tSSLCertificateChainFile %s\n" % SSLCertificateChainFile) + cnf.write("\n") + cnf.close() + self.run("a2ensite %s" % domain) + self.run("/etc/init.d/apache2 force-reload") + + def main(self): + self.install() + self.setup_vhost() diff --git a/api/boto/pyami/launch_ami.py b/api/boto/pyami/launch_ami.py new file mode 100755 index 0000000..c49c2a3 --- /dev/null +++ b/api/boto/pyami/launch_ami.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import getopt, sys, imp, time +import boto +from boto.utils import get_instance_userdata + +usage_string = """ +SYNOPSIS + launch_ami.py -a ami_id [-b script_bucket] [-s script_name] + [-m module] [-c class_name] [-r] + [-g group] [-k key_name] [-n num_instances] + [-w] [extra_data] + Where: + ami_id - the id of the AMI you wish to launch + module - The name of the Python module containing the class you + want to run when the instance is started. If you use this + option the Python module must already be stored on the + instance in a location that is on the Python path. + script_file - The name of a local Python module that you would like + to have copied to S3 and then run on the instance + when it is started. The specified module must be + import'able (i.e. in your local Python path). It + will then be copied to the specified bucket in S3 + (see the -b option). Once the new instance(s) + start up the script will be copied from S3 and then + run locally on the instance. + class_name - The name of the class to be instantiated within the + module or script file specified. + script_bucket - the name of the bucket in which the script will be + stored + group - the name of the security group the instance will run in + key_name - the name of the keypair to use when launching the AMI + num_instances - how many instances of the AMI to launch (default 1) + input_queue_name - Name of SQS to read input messages from + output_queue_name - Name of SQS to write output messages to + extra_data - additional name-value pairs that will be passed as + userdata to the newly launched instance. These should + be of the form "name=value" + The -r option reloads the Python module to S3 without launching + another instance. This can be useful during debugging to allow + you to test a new version of your script without shutting down + your instance and starting up another one. + The -w option tells the script to run synchronously, meaning to + wait until the instance is actually up and running. It then prints + the IP address and internal and external DNS names before exiting. +""" + +def usage(): + print usage_string + sys.exit() + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'a:b:c:g:hi:k:m:n:o:rs:w', + ['ami', 'bucket', 'class', 'group', 'help', + 'inputqueue', 'keypair', 'module', + 'numinstances', 'outputqueue', + 'reload', 'script_name', 'wait']) + except: + usage() + params = {'module_name' : None, + 'script_name' : None, + 'class_name' : None, + 'script_bucket' : None, + 'group' : 'default', + 'keypair' : None, + 'ami' : None, + 'num_instances' : 1, + 'input_queue_name' : None, + 'output_queue_name' : None} + reload = None + wait = None + for o, a in opts: + if o in ('-a', '--ami'): + params['ami'] = a + if o in ('-b', '--bucket'): + params['script_bucket'] = a + if o in ('-c', '--class'): + params['class_name'] = a + if o in ('-g', '--group'): + params['group'] = a + if o in ('-h', '--help'): + usage() + if o in ('-i', '--inputqueue'): + params['input_queue_name'] = a + if o in ('-k', '--keypair'): + params['keypair'] = a + if o in ('-m', '--module'): + params['module_name'] = a + if o in ('-n', '--num_instances'): + params['num_instances'] = int(a) + if o in ('-o', '--outputqueue'): + params['output_queue_name'] = a + if o in ('-r', '--reload'): + reload = True + if o in ('-s', '--script'): + params['script_name'] = a + if o in ('-w', '--wait'): + wait = True + + # check required fields + required = ['ami'] + for pname in required: + if not params.get(pname, None): + print '%s is required' % pname + usage() + if params['script_name']: + # first copy the desired module file to S3 bucket + if reload: + print 'Reloading module %s to S3' % params['script_name'] + else: + print 'Copying module %s to S3' % params['script_name'] + l = imp.find_module(params['script_name']) + c = boto.connect_s3() + bucket = c.get_bucket(params['script_bucket']) + key = bucket.new_key(params['script_name']+'.py') + key.set_contents_from_file(l[0]) + params['script_md5'] = key.md5 + # we have everything we need, now build userdata string + l = [] + for k, v in params.items(): + if v: + l.append('%s=%s' % (k, v)) + c = boto.connect_ec2() + l.append('aws_access_key_id=%s' % c.aws_access_key_id) + l.append('aws_secret_access_key=%s' % c.aws_secret_access_key) + for kv in args: + l.append(kv) + s = '|'.join(l) + if not reload: + rs = c.get_all_images([params['ami']]) + img = rs[0] + r = img.run(user_data=s, key_name=params['keypair'], + security_groups=[params['group']], + max_count=params.get('num_instances', 1)) + print 'AMI: %s - %s (Started)' % (params['ami'], img.location) + print 'Reservation %s contains the following instances:' % r.id + for i in r.instances: + print '\t%s' % i.id + if wait: + running = False + while not running: + time.sleep(30) + [i.update() for i in r.instances] + status = [i.state for i in r.instances] + print status + if status.count('running') == len(r.instances): + running = True + for i in r.instances: + print 'Instance: %s' % i.ami_launch_index + print 'Public DNS Name: %s' % i.public_dns_name + print 'Private DNS Name: %s' % i.private_dns_name + +if __name__ == "__main__": + main() + diff --git a/api/boto/pyami/scriptbase.py b/api/boto/pyami/scriptbase.py new file mode 100644 index 0000000..6fe92aa --- /dev/null +++ b/api/boto/pyami/scriptbase.py @@ -0,0 +1,44 @@ +import os, sys, time, traceback +import smtplib +from boto.utils import ShellCommand, get_ts +import boto +import boto.utils + +class ScriptBase: + + def __init__(self, config_file=None): + self.instance_id = boto.config.get('Instance', 'instance-id', 'default') + self.name = self.__class__.__name__ + self.ts = get_ts() + if config_file: + boto.config.read(config_file) + + def notify(self, subject, body=''): + boto.utils.notify(subject, body) + + def mkdir(self, path): + if not os.path.isdir(path): + try: + os.mkdir(path) + except: + boto.log.error('Error creating directory: %s' % path) + + def umount(self, path): + if os.path.ismount(path): + self.run('umount %s' % path) + + def run(self, command, notify=True, exit_on_error=False): + self.last_command = ShellCommand(command) + if self.last_command.status != 0: + boto.log.error('Error running command: "%s". Output: "%s"' % (command, self.last_command.output)) + if notify: + self.notify('Error encountered', \ + 'Error running the following command:\n\t%s\n\nCommand output:\n\t%s' % \ + (command, self.last_command.output)) + if exit_on_error: + sys.exit(-1) + return self.last_command.status + + def main(self): + pass + diff --git a/api/boto/pyami/startup.py b/api/boto/pyami/startup.py new file mode 100644 index 0000000..d6f1376 --- /dev/null +++ b/api/boto/pyami/startup.py @@ -0,0 +1,59 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import os, sys, traceback, StringIO +import boto +from boto.utils import find_class +from boto import config +from boto.pyami.scriptbase import ScriptBase +from boto.utils import find_class + +class Startup(ScriptBase): + + def run_scripts(self): + scripts = config.get('Pyami', 'scripts') + if scripts: + for script in scripts.split(','): + script = script.strip(" ") + try: + pos = script.rfind('.') + if pos > 0: + mod_name = script[0:pos] + cls_name = script[pos+1:] + cls = find_class(mod_name, cls_name) + boto.log.info('Running Script: %s' % script) + s = cls() + s.main() + else: + boto.log.warning('Trouble parsing script: %s' % script) + except Exception, e: + boto.log.exception('Problem Running Script: %s' % script) + + def main(self): + self.run_scripts() + self.notify('Startup Completed for %s' % config.get('Instance', 'instance-id')) + +if __name__ == "__main__": + if not config.has_section('loggers'): + boto.set_file_logger('startup', '/var/log/boto.log') + sys.path.append(config.get('Pyami', 'working_dir')) + su = Startup() + su.main() diff --git a/api/boto/rds/__init__.py b/api/boto/rds/__init__.py new file mode 100644 index 0000000..92b7199 --- /dev/null +++ b/api/boto/rds/__init__.py @@ -0,0 +1,813 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import xml.sax +import base64 +import time +import boto +import boto.utils +import urllib +from boto.connection import AWSQueryConnection +from boto import handler +from boto.resultset import ResultSet +from boto.rds.dbinstance import DBInstance +from boto.rds.dbsecuritygroup import DBSecurityGroup +from boto.rds.parametergroup import ParameterGroup +from boto.rds.dbsnapshot import DBSnapshot +from boto.rds.event import Event + +#boto.set_stream_logger('rds') + +class RDSConnection(AWSQueryConnection): + + DefaultHost = 'rds.amazonaws.com' + APIVersion = '2009-10-16' + SignatureVersion = '2' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=DefaultHost, debug=0, + https_connection_factory=None, path='/'): + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, + proxy_pass, self.DefaultHost, debug, + https_connection_factory, path) + + # DB Instance methods + + def get_all_dbinstances(self, instance_id=None, max_records=None, + marker=None): + """ + Retrieve all the DBInstances in your account. + + :type instance_id: str + :param instance_id: DB Instance identifier. If supplied, only information + this instance will be returned. Otherwise, info + about all DB Instances will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.rds.dbinstance.DBInstance` + """ + params = {} + if instance_id: + params['DBInstanceIdentifier'] = instance_id + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBInstances', params, [('DBInstance', DBInstance)]) + + def create_dbinstance(self, id, allocated_storage, instance_class, + master_username, master_password, port=3306, + engine='MySQL5.1', db_name=None, param_group=None, + security_groups=None, availability_zone=None, + preferred_maintenance_window=None, + backup_retention_period=None, + preferred_backup_window=None): + """ + Create a new DBInstance. + + :type id: str + :param id: Unique identifier for the new instance. + Must contain 1-63 alphanumeric characters. + First character must be a letter. + May not end with a hyphen or contain two consecutive hyphens + + :type allocated_storage: int + :param allocated_storage: Initially allocated storage size, in GBs. + Valid values are [5-1024] + + :type instance_class: str + :param instance_class: The compute and memory capacity of the DBInstance. + Valid values are: + db.m1.small | db.m1.large | db.m1.xlarge | + db.m2.2xlarge | db.m2.4xlarge + + :type engine: str +. :param engine: Name of database engine. Must be MySQL5.1 for now. + + :type master_username: str + :param master_username: Name of master user for the DBInstance. + Must be 1-15 alphanumeric characters, first must be + a letter. + + :type master_password: str + :param master_password: Password of master user for the DBInstance. + Must be 4-16 alphanumeric characters. + + :type port: int + :param port: Port number on which database accepts connections. + Valid values [1115-65535]. Defaults to 3306. + + :type db_name: str + :param db_name: Name of a database to create when the DBInstance + is created. Default is to create no databases. + + :type param_group: str + :param param_group: Name of DBParameterGroup to associate with + this DBInstance. If no groups are specified + no parameter groups will be used. + + :type security_groups: list of str or list of DBSecurityGroup objects + :param security_groups: List of names of DBSecurityGroup to authorize on + this DBInstance. + + :type availability_zone: str + :param availability_zone: Name of the availability zone to place + DBInstance into. + + :type preferred_maintenance_window: str + :param preferred_maintenance_window: The weekly time range (in UTC) during + which maintenance can occur. + Default is Sun:05:00-Sun:09:00 + + :type backup_retention_period: int + :param backup_retention_period: The number of days for which automated + backups are retained. Setting this to + zero disables automated backups. + + :type preferred_backup_window: str + :param preferred_backup_window: The daily time range during which + automated backups are created (if + enabled). Must be in h24:mi-hh24:mi + format (UTC). + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The new db instance. + """ + params = {'DBInstanceIdentifier' : id, + 'AllocatedStorage' : allocated_storage, + 'DBInstanceClass' : instance_class, + 'Engine' : engine, + 'MasterUsername' : master_username, + 'MasterUserPassword' : master_password} + if port: + params['Port'] = port + if db_name: + params['DBName'] = db_name + if param_group: + params['DBParameterGroup'] = param_group + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, DBSecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, 'DBSecurityGroups.member') + if availability_zone: + params['AvailabilityZone'] = availability_zone + if preferred_maintenance_window: + params['PreferredMaintenanceWindow'] = preferred_maintenance_window + if backup_retention_period: + params['BackupRetentionPeriod'] = backup_retention_period + if preferred_backup_window: + params['PreferredBackupWindow'] = preferred_backup_window + + return self.get_object('CreateDBInstance', params, DBInstance) + + def modify_dbinstance(self, id, param_group=None, security_groups=None, + preferred_maintenance_window=None, + master_password=None, allocated_storage=None, + backup_retention_period=None, + preferred_backup_window=None, + apply_immediately=False): + """ + Modify an existing DBInstance. + + :type id: str + :param id: Unique identifier for the new instance. + + :type security_groups: list of str or list of DBSecurityGroup objects + :param security_groups: List of names of DBSecurityGroup to authorize on + this DBInstance. + + :type preferred_maintenance_window: str + :param preferred_maintenance_window: The weekly time range (in UTC) during + which maintenance can occur. + Default is Sun:05:00-Sun:09:00 + + :type master_password: str + :param master_password: Password of master user for the DBInstance. + Must be 4-15 alphanumeric characters. + + :type allocated_storage: int + :param allocated_storage: The new allocated storage size, in GBs. + Valid values are [5-1024] + + :type instance_class: str + :param instance_class: The compute and memory capacity of the DBInstance. + Changes will be applied at next maintenance + window unless apply_immediately is True. + Valid values are: + db.m1.small | db.m1.large | db.m1.xlarge | + db.m2.2xlarge | db.m2.4xlarge + + :type apply_immediately: bool + :param apply_immediately: If true, the modifications will be applied + as soon as possible rather than waiting for + the next preferred maintenance window. + + :type backup_retention_period: int + :param backup_retention_period: The number of days for which automated + backups are retained. Setting this to + zero disables automated backups. + + :type preferred_backup_window: str + :param preferred_backup_window: The daily time range during which + automated backups are created (if + enabled). Must be in h24:mi-hh24:mi + format (UTC). + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The modified db instance. + """ + params = {'DBInstanceIdentifier' : id} + if param_group: + params['DBParameterGroupName'] = param_group + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, DBSecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, 'DBSecurityGroups.member') + if preferred_maintenance_window: + params['PreferredMaintenanceWindow'] = preferred_maintenance_window + if master_password: + params['MasterUserPassword'] = master_password + if allocated_storage: + params['AllocatedStorage'] = allocated_storage + if backup_retention_period: + params['BackupRetentionPeriod'] = backup_retention_period + if preferred_backup_window: + params['PreferredBackupWindow'] = preferred_backup_window + if apply_immediately: + params['ApplyImmediately'] = 'true' + + return self.get_object('ModifyDBInstance', params, DBInstance) + + def delete_dbinstance(self, id, skip_final_snapshot=False, + final_snapshot_id=''): + """ + Delete an existing DBInstance. + + :type id: str + :param id: Unique identifier for the new instance. + + :type skip_final_snapshot: bool + :param skip_final_snapshot: This parameter determines whether a final + db snapshot is created before the instance + is deleted. If True, no snapshot is created. + If False, a snapshot is created before + deleting the instance. + + :type final_snapshot_id: str + :param final_snapshot_id: If a final snapshot is requested, this + is the identifier used for that snapshot. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The deleted db instance. + """ + params = {'DBInstanceIdentifier' : id} + if skip_final_snapshot: + params['SkipFinalSnapshot'] = 'true' + else: + params['SkipFinalSnapshot'] = 'false' + params['FinalDBSnapshotIdentifier'] = final_snapshot_id + return self.get_object('DeleteDBInstance', params, DBInstance) + + # DBParameterGroup methods + + def get_all_dbparameter_groups(self, groupname=None, max_records=None, + marker=None): + """ + Get all parameter groups associated with your account in a region. + + :type groupname: str + :param groupname: The name of the DBParameter group to retrieve. + If not provided, all DBParameter groups will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.ec2.parametergroup.ParameterGroup` + """ + params = {} + if groupname: + params['DBParameterGroupName'] = groupname + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBParameterGroups', params, + [('DBParameterGroup', ParameterGroup)]) + + def get_all_dbparameters(self, groupname, source=None, + max_records=None, marker=None): + """ + Get all parameters associated with a ParameterGroup + + :type groupname: str + :param groupname: The name of the DBParameter group to retrieve. + + :type source: str + :param source: Specifies which parameters to return. + If not specified, all parameters will be returned. + Valid values are: user|system|engine-default + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: :class:`boto.ec2.parametergroup.ParameterGroup` + :return: The ParameterGroup + """ + params = {'DBParameterGroupName' : groupname} + if source: + params['Source'] = source + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + pg = self.get_object('DescribeDBParameters', params, ParameterGroup) + pg.name = groupname + return pg + + def create_parameter_group(self, name, engine='MySQL5.1', description=''): + """ + Create a new dbparameter group for your account. + + :type name: string + :param name: The name of the new dbparameter group + + :type engine: str + :param engine: Name of database engine. Must be MySQL5.1 for now. + + :type description: string + :param description: The description of the new security group + + :rtype: :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` + :return: The newly created DBSecurityGroup + """ + params = {'DBParameterGroupName': name, + 'Engine': engine, + 'Description' : description} + return self.get_object('CreateDBParameterGroup', params, ParameterGroup) + + def modify_parameter_group(self, name, parameters=None): + """ + Modify a parameter group for your account. + + :type name: string + :param name: The name of the new parameter group + + :type parameters: list of :class:`boto.rds.parametergroup.Parameter` + :param parameters: The new parameters + + :rtype: :class:`boto.rds.parametergroup.ParameterGroup` + :return: The newly created ParameterGroup + """ + params = {'DBParameterGroupName': name} + for i in range(0, len(parameters)): + parameter = parameters[i] + parameter.merge(params, i+1) + return self.get_list('ModifyDBParameterGroup', params, ParameterGroup) + + def reset_parameter_group(self, name, reset_all_params=False, parameters=None): + """ + Resets some or all of the parameters of a ParameterGroup to the + default value + + :type key_name: string + :param key_name: The name of the ParameterGroup to reset + + :type parameters: list of :class:`boto.rds.parametergroup.Parameter` + :param parameters: The parameters to reset. If not supplied, all parameters + will be reset. + """ + params = {'DBParameterGroupName':name} + if reset_all_params: + params['ResetAllParameters'] = 'true' + else: + params['ResetAllParameters'] = 'false' + for i in range(0, len(parameters)): + parameter = parameters[i] + parameter.merge(params, i+1) + return self.get_status('ResetDBParameterGroup', params) + + def delete_parameter_group(self, name): + """ + Delete a DBSecurityGroup from your account. + + :type key_name: string + :param key_name: The name of the DBSecurityGroup to delete + """ + params = {'DBParameterGroupName':name} + return self.get_status('DeleteDBParameterGroup', params) + + # DBSecurityGroup methods + + def get_all_dbsecurity_groups(self, groupname=None, max_records=None, + marker=None): + """ + Get all security groups associated with your account in a region. + + :type groupnames: list + :param groupnames: A list of the names of security groups to retrieve. + If not provided, all security groups will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` + """ + params = {} + if groupname: + params['DBSecurityGroupName'] = groupname + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBSecurityGroups', params, + [('DBSecurityGroup', DBSecurityGroup)]) + + def create_dbsecurity_group(self, name, description=None): + """ + Create a new security group for your account. + This will create the security group within the region you + are currently connected to. + + :type name: string + :param name: The name of the new security group + + :type description: string + :param description: The description of the new security group + + :rtype: :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` + :return: The newly created DBSecurityGroup + """ + params = {'DBSecurityGroupName':name} + if description: + params['DBSecurityGroupDescription'] = description + group = self.get_object('CreateDBSecurityGroup', params, DBSecurityGroup) + group.name = name + group.description = description + return group + + def delete_dbsecurity_group(self, name): + """ + Delete a DBSecurityGroup from your account. + + :type key_name: string + :param key_name: The name of the DBSecurityGroup to delete + """ + params = {'DBSecurityGroupName':name} + return self.get_status('DeleteDBSecurityGroup', params) + + def authorize_dbsecurity_group(self, group_name, cidr_ip=None, + ec2_security_group_name=None, + ec2_security_group_owner_id=None): + """ + Add a new rule to an existing security group. + You need to pass in either src_security_group_name and + src_security_group_owner_id OR a CIDR block but not both. + + :type group_name: string + :param group_name: The name of the security group you are adding + the rule to. + + :type ec2_security_group_name: string + :param ec2_security_group_name: The name of the EC2 security group you are + granting access to. + + :type ec2_security_group_owner_id: string + :param ec2_security_group_owner_id: The ID of the owner of the EC2 security + group you are granting access to. + + :type cidr_ip: string + :param cidr_ip: The CIDR block you are providing access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + params = {'DBSecurityGroupName':group_name} + if ec2_security_group_name: + params['EC2SecurityGroupName'] = ec2_security_group_name + if ec2_security_group_owner_id: + params['EC2SecurityGroupOwnerId'] = ec2_security_group_owner_id + if cidr_ip: + params['CIDRIP'] = urllib.quote(cidr_ip) + return self.get_object('AuthorizeDBSecurityGroupIngress', params, DBSecurityGroup) + + def revoke_security_group(self, group_name, ec2_security_group_name=None, + ec2_security_group_owner_id=None, cidr_ip=None): + """ + Remove an existing rule from an existing security group. + You need to pass in either ec2_security_group_name and + ec2_security_group_owner_id OR a CIDR block. + + :type group_name: string + :param group_name: The name of the security group you are removing + the rule from. + + :type ec2_security_group_name: string + :param ec2_security_group_name: The name of the EC2 security group you are + granting access to. + + :type ec2_security_group_owner_id: string + :param ec2_security_group_owner_id: The ID of the owner of the EC2 security + group you are granting access to. + + :type cidr_ip: string + :param cidr_ip: The CIDR block you are providing access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + params = {'DBSecurityGroupName':group_name} + if ec2_security_group_name: + params['EC2SecurityGroupName'] = ec2_security_group_name + if ec2_security_group_owner_id: + params['EC2SecurityGroupOwnerId'] = ec2_security_group_owner_id + if cidr_ip: + params['CIDRIP'] = cidr_ip + return self.get_object('RevokeDBSecurityGroupIngress', params, DBSecurityGroup) + + # DBSnapshot methods + + def get_all_dbsnapshots(self, snapshot_id=None, instance_id=None, + max_records=None, marker=None): + """ + Get information about DB Snapshots. + + :type snapshot_id: str + :param snapshot_id: The unique identifier of an RDS snapshot. + If not provided, all RDS snapshots will be returned. + + :type instance_id: str + :param instance_id: The identifier of a DBInstance. If provided, + only the DBSnapshots related to that instance will + be returned. + If not provided, all RDS snapshots will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.rds.dbsnapshot.DBSnapshot` + """ + params = {} + if snapshot_id: + params['DBSnapshotIdentifier'] = snapshot_id + if instance_id: + params['DBInstanceIdentifier'] = instance_id + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBSnapshots', params, + [('DBSnapshots', DBSnapshot)]) + + def create_dbsnapshot(self, snapshot_id, dbinstance_id): + """ + Create a new DB snapshot. + + :type snapshot_id: string + :param snapshot_id: The identifier for the DBSnapshot + + :type dbinstance_id: string + :param dbinstance_id: The source identifier for the RDS instance from + which the snapshot is created. + + :rtype: :class:`boto.rds.dbsnapshot.DBSnapshot` + :return: The newly created DBSnapshot + """ + params = {'DBSnapshotIdentifier' : snapshot_id, + 'DBInstanceIdentifier' : dbinstance_id} + return self.get_object('CreateDBSnapshot', params, DBSnapshot) + + def delete_dbsnapshot(self, identifier): + """ + Delete a DBSnapshot + + :type identifier: string + :param identifier: The identifier of the DBSnapshot to delete + """ + params = {'DBSnapshotIdentifier' : identifier} + return self.get_object('DeleteDBSnapshot', params, DBSnapshot) + + def restore_dbinstance_from_dbsnapshot(self, identifier, instance_id, + instance_class, port=None, + availability_zone=None): + + """ + Create a new DBInstance from a DB snapshot. + + :type identifier: string + :param identifier: The identifier for the DBSnapshot + + :type instance_id: string + :param instance_id: The source identifier for the RDS instance from + which the snapshot is created. + + :type instance_class: str + :param instance_class: The compute and memory capacity of the DBInstance. + Valid values are: + db.m1.small | db.m1.large | db.m1.xlarge | + db.m2.2xlarge | db.m2.4xlarge + + :type port: int + :param port: Port number on which database accepts connections. + Valid values [1115-65535]. Defaults to 3306. + + :type availability_zone: str + :param availability_zone: Name of the availability zone to place + DBInstance into. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The newly created DBInstance + """ + params = {'DBSnapshotIdentifier' : identifier, + 'DBInstanceIdentifier' : instance_id, + 'DBInstanceClass' : instance_class} + if port: + params['Port'] = port + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('RestoreDBInstanceFromDBSnapshot', + params, DBInstance) + + def restore_dbinstance_from_point_in_time(self, source_instance_id, + target_instance_id, + use_latest=False, + restore_time=None, + dbinstance_class=None, + port=None, + availability_zone=None): + + """ + Create a new DBInstance from a point in time. + + :type source_instance_id: string + :param source_instance_id: The identifier for the source DBInstance. + + :type target_instance_id: string + :param target_instance_id: The identifier of the new DBInstance. + + :type use_latest: bool + :param use_latest: If True, the latest snapshot availabile will + be used. + + :type restore_time: datetime + :param restore_time: The date and time to restore from. Only + used if use_latest is False. + + :type instance_class: str + :param instance_class: The compute and memory capacity of the DBInstance. + Valid values are: + db.m1.small | db.m1.large | db.m1.xlarge | + db.m2.2xlarge | db.m2.4xlarge + + :type port: int + :param port: Port number on which database accepts connections. + Valid values [1115-65535]. Defaults to 3306. + + :type availability_zone: str + :param availability_zone: Name of the availability zone to place + DBInstance into. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The newly created DBInstance + """ + params = {'SourceDBInstanceIdentifier' : source_instance_id, + 'TargetDBInstanceIdentifier' : target_instance_id} + if use_latest: + params['UseLatestRestorableTime'] = 'true' + elif restore_time: + params['RestoreTime'] = restore_time.isoformat() + if instance_class: + params['DBInstanceClass'] = instance_class + if port: + params['Port'] = port + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('RestoreDBInstanceToPointInTime', + params, DBInstance) + + # Events + + def get_all_events(self, source_identifier=None, source_type=None, + start_time=None, end_time=None, + max_records=None, marker=None): + """ + Get information about events related to your DBInstances, + DBSecurityGroups and DBParameterGroups. + + :type source_identifier: str + :param source_identifier: If supplied, the events returned will be + limited to those that apply to the identified + source. The value of this parameter depends + on the value of source_type. If neither + parameter is specified, all events in the time + span will be returned. + + :type source_type: str + :param source_type: Specifies how the source_identifier should + be interpreted. Valid values are: + b-instance | db-security-group | + db-parameter-group | db-snapshot + + :type start_time: datetime + :param start_time: The beginning of the time interval for events. + If not supplied, all available events will + be returned. + + :type end_time: datetime + :param end_time: The ending of the time interval for events. + If not supplied, all available events will + be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of class:`boto.rds.event.Event` + """ + params = {} + if source_identifier and source_type: + params['SourceIdentifier'] = source_identifier + params['SourceType'] = source_type + if start_time: + params['StartTime'] = start_time.isoformat() + if end_time: + params['EndTime'] = end_time.isoformat() + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeEvents', params, [('Event', Event)]) + + diff --git a/api/boto/rds/dbinstance.py b/api/boto/rds/dbinstance.py new file mode 100644 index 0000000..23e1c98 --- /dev/null +++ b/api/boto/rds/dbinstance.py @@ -0,0 +1,136 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.rds.dbsecuritygroup import DBSecurityGroup +from boto.rds.parametergroup import ParameterGroup + +class DBInstance(object): + """ + Represents a RDS DBInstance + """ + + def __init__(self, connection=None, id=None): + self.connection = connection + self.id = id + self.create_time = None + self.engine = None + self.status = None + self.allocated_storage = None + self.endpoint = None + self.instance_class = None + self.master_username = None + self.parameter_group = None + self.security_group = None + self.availability_zone = None + self.backup_retention_period = None + self.preferred_backup_window = None + self.preferred_maintenance_window = None + self.latest_restorable_time = None + self._in_endpoint = False + self._port = None + self._address = None + + def __repr__(self): + return 'DBInstance:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'Endpoint': + self._in_endpoint = True + elif name == 'DBParameterGroup': + self.parameter_group = ParameterGroup(self.connection) + return self.parameter_group + elif name == 'DBSecurityGroup': + self.security_group = DBSecurityGroup(self.connection) + return self.security_group + return None + + def endElement(self, name, value, connection): + if name == 'DBInstanceIdentifier': + self.id = value + elif name == 'DBInstanceStatus': + self.status = value + elif name == 'InstanceCreateTime': + self.create_time = value + elif name == 'Engine': + self.engine = value + elif name == 'DBInstanceStatus': + self.status = value + elif name == 'AllocatedStorage': + self.allocated_storage = int(value) + elif name == 'DBInstanceClass': + self.instance_class = value + elif name == 'MasterUsername': + self.master_username = value + elif name == 'Port': + if self._in_endpoint: + self._port = int(value) + elif name == 'Address': + if self._in_endpoint: + self._address = value + elif name == 'Endpoint': + self.endpoint = (self._address, self._port) + self._in_endpoint = False + elif name == 'AvailabilityZone': + self.availability_zone = value + elif name == 'BackupRetentionPeriod': + self.backup_retention_period = value + elif name == 'LatestRestorableTime': + self.latest_restorable_time = value + elif name == 'PreferredMaintenanceWindow': + self.preferred_maintenance_window = value + elif name == 'PreferredBackupWindow': + self.preferred_backup_window = value + else: + setattr(self, name, value) + + def snapshot(self, snapshot_id): + """ + Create a new DB snapshot of this DBInstance. + + :type identifier: string + :param identifier: The identifier for the DBSnapshot + + :rtype: :class:`boto.rds.dbsnapshot.DBSnapshot` + :return: The newly created DBSnapshot + """ + return self.connection.create_dbsnapshot(snapshot_id, self.id) + + def stop(self, skip_final_snapshot, final_snapshot_id): + """ + Delete this DBInstance. + + :type skip_final_snapshot: bool + :param skip_final_snapshot: This parameter determines whether a final + db snapshot is created before the instance + is deleted. If True, no snapshot is created. + If False, a snapshot is created before + deleting the instance. + + :type final_snapshot_id: str + :param final_snapshot_id: If a final snapshot is requested, this + is the identifier used for that snapshot. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The deleted db instance. + """ + return self.connection.delete_dbinstance(self.id, + skip_final_snapshot, + final_snapshot_id) diff --git a/api/boto/rds/dbsecuritygroup.py b/api/boto/rds/dbsecuritygroup.py new file mode 100644 index 0000000..9ec6cc0 --- /dev/null +++ b/api/boto/rds/dbsecuritygroup.py @@ -0,0 +1,159 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an DBSecurityGroup +""" +from boto.ec2.securitygroup import SecurityGroup + +class DBSecurityGroup(object): + + def __init__(self, connection=None, owner_id=None, + name=None, description=None): + self.connection = connection + self.owner_id = owner_id + self.name = name + self.description = description + self.ec2_groups = [] + self.ip_ranges = [] + + def __repr__(self): + return 'DBSecurityGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'IPRange': + cidr = IPRange(self) + self.ip_ranges.append(cidr) + return cidr + elif name == 'EC2SecurityGroup': + ec2_grp = EC2SecurityGroup(self) + self.ec2_groups.append(ec2_grp) + return ec2_grp + else: + return None + + def endElement(self, name, value, connection): + if name == 'OwnerId': + self.owner_id = value + elif name == 'DBSecurityGroupName': + self.name = value + elif name == 'DBSecurityGroupDescription': + self.description = value + elif name == 'IPRanges': + pass + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_dbsecurity_group(self.name) + + def authorize(self, cidr_ip=None, ec2_group=None): + """ + Add a new rule to this DBSecurity group. + You need to pass in either a CIDR block to authorize or + and EC2 SecurityGroup. + + @type cidr_ip: string + @param cidr_ip: A valid CIDR IP range to authorize + + @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>`b + + @rtype: bool + @return: True if successful. + """ + if isinstance(ec2_group, SecurityGroup): + group_name = ec2_group.name + group_owner_id = ec2_group.owner_id + else: + group_name = None + group_owner_id = None + return self.connection.authorize_dbsecurity_group(self.name, + cidr_ip, + group_name, + group_owner_id) + + def revoke(self, cidr_ip=None, ec2_group=None): + """ + Revoke access to a CIDR range or EC2 SecurityGroup + You need to pass in either a CIDR block to authorize or + and EC2 SecurityGroup. + + @type cidr_ip: string + @param cidr_ip: A valid CIDR IP range to authorize + + @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>`b + + @rtype: bool + @return: True if successful. + """ + if isinstance(ec2_group, SecurityGroup): + group_name = ec2_group.name + group_owner_id = ec2_group.owner_id + else: + group_name = None + group_owner_id = None + return self.connection.revoke_dbsecurity_group(self.name, + cidr_ip, + group_name, + group_owner_id) + +class IPRange(object): + + def __init__(self, parent=None): + self.parent = parent + self.cidr_ip = None + self.status = None + + def __repr__(self): + return 'IPRange:%s' % self.cidr_ip + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'CIDRIP': + self.cidr_ip = value + elif name == 'Status': + self.status = value + else: + setattr(self, name, value) + +class EC2SecurityGroup(object): + + def __init__(self, parent=None): + self.parent = parent + self.name = None + self.owner_id = None + + def __repr__(self): + return 'EC2SecurityGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'EC2SecurityGroupName': + self.name = value + elif name == 'EC2SecurityGroupOwnerId': + self.owner_id = value + else: + setattr(self, name, value) + diff --git a/api/boto/rds/dbsnapshot.py b/api/boto/rds/dbsnapshot.py new file mode 100644 index 0000000..78d0230 --- /dev/null +++ b/api/boto/rds/dbsnapshot.py @@ -0,0 +1,74 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class DBSnapshot(object): + """ + Represents a RDS DB Snapshot + """ + + def __init__(self, connection=None, id=None): + self.connection = connection + self.id = id + self.engine = None + self.snapshot_create_time = None + self.instance_create_time = None + self.port = None + self.status = None + self.availability_zone = None + self.master_username = None + self.allocated_storage = None + self.instance_id = None + self.availability_zone = None + + def __repr__(self): + return 'DBSnapshot:%s' % self.id + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'Engine': + self.engine = value + elif name == 'InstanceCreateTime': + self.instance_create_time = value + elif name == 'SnapshotCreateTime': + self.snapshot_create_time = value + elif name == 'DBInstanceIdentifier': + self.instance_id = value + elif name == 'DBSnapshotIdentifier': + self.id = value + elif name == 'Port': + self.port = int(value) + elif name == 'Status': + self.status = value + elif name == 'AvailabilityZone': + self.availability_zone = value + elif name == 'MasterUsername': + self.master_username = value + elif name == 'AllocatedStorage': + self.allocated_storage = int(value) + elif name == 'SnapshotTime': + self.time = value + else: + setattr(self, name, value) + + + diff --git a/api/boto/rds/event.py b/api/boto/rds/event.py new file mode 100644 index 0000000..a91f8f0 --- /dev/null +++ b/api/boto/rds/event.py @@ -0,0 +1,49 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Event(object): + + def __init__(self, connection=None): + self.connection = connection + self.message = None + self.source_identifier = None + self.source_type = None + self.engine = None + self.date = None + + def __repr__(self): + return '"%s"' % self.message + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'SourceIdentifier': + self.source_identifier = value + elif name == 'SourceType': + self.source_type = value + elif name == 'Message': + self.message = value + elif name == 'Date': + self.date = value + else: + setattr(self, name, value) + diff --git a/api/boto/rds/parametergroup.py b/api/boto/rds/parametergroup.py new file mode 100644 index 0000000..081e263 --- /dev/null +++ b/api/boto/rds/parametergroup.py @@ -0,0 +1,201 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class ParameterGroup(dict): + + def __init__(self, connection=None): + dict.__init__(self) + self.connection = connection + self.name = None + self.description = None + self.engine = None + self._current_param = None + + def __repr__(self): + return 'ParameterGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'Parameter': + if self._current_param: + self[self._current_param.name] = self._current_param + self._current_param = Parameter(self) + return self._current_param + + def endElement(self, name, value, connection): + if name == 'DBParameterGroupName': + self.name = value + elif name == 'Description': + self.description = value + elif name == 'Engine': + self.engine = value + else: + setattr(self, name, value) + + def modifiable(self): + mod = [] + for key in self: + p = self[key] + if p.is_modifiable: + mod.append(p) + return mod + + def get_params(self): + pg = self.connection.get_all_dbparameters(self.name) + self.update(pg) + + def add_param(self, name, value, apply_method): + param = Parameter() + param.name = name + param.value = value + param.apply_method = apply_method + self.params.append(param) + +class Parameter(object): + """ + Represents a RDS Parameter + """ + + ValidTypes = {'integer' : int, + 'string' : str, + 'boolean' : bool} + ValidSources = ['user', 'system', 'engine-default'] + ValidApplyTypes = ['static', 'dynamic'] + ValidApplyMethods = ['immediate', 'pending-reboot'] + + def __init__(self, group=None, name=None): + self.group = group + self.name = name + self._value = None + self.type = str + self.source = None + self.is_modifiable = True + self.description = None + self.apply_method = None + self.allowed_values = None + + def __repr__(self): + return 'Parameter:%s' % self.name + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'ParameterName': + self.name = value + elif name == 'ParameterValue': + self._value = value + elif name == 'DataType': + if value in self.ValidTypes: + self.type = value + elif name == 'Source': + if value in self.ValidSources: + self.source = value + elif name == 'IsModifiable': + if value.lower() == 'true': + self.is_modifiable = True + else: + self.is_modifiable = False + elif name == 'Description': + self.description = value + elif name == 'ApplyType': + if value in self.ValidApplyTypes: + self.apply_type = value + elif name == 'AllowedValues': + self.allowed_values = value + else: + setattr(self, name, value) + + def merge(self, d, i): + prefix = 'Parameters.member.%d.' % i + if self.name: + d[prefix+'ParameterName'] = self.name + if self._value: + d[prefix+'ParameterValue'] = self._value + if self.apply_type: + d[prefix+'ApplyMethod'] = self.apply_method + + def _set_string_value(self, value): + if not isinstance(value, str) or isinstance(value, unicode): + raise ValueError, 'value must be of type str' + if self.allowed_values: + choices = self.allowed_values.split(',') + if value not in choices: + raise ValueError, 'value must be in %s' % self.allowed_values + set._value = value + + def _set_integer_value(self, value): + if isinstance(value, str) or isinstance(value, unicode): + value = int(value) + if isinstance(value, int) or isinstance(value, long): + if self.allowed_values: + min, max = self.allowed_values.split('-') + if value < int(min) or value > int(max): + raise ValueError, 'range is %s' % self.allowed_values + self._value = value + else: + raise ValueError, 'value must be integer' + + def _set_boolean_value(self, value): + if isinstance(value, bool): + self._value = value + elif isinstance(value, str) or isinstance(value, unicode): + if value.lower() == 'true': + self._value = True + else: + self._value = False + else: + raise ValueError, 'value must be boolean' + + def set_value(self, value): + if self.type == 'string': + self._set_string_value(value) + elif self.type == 'integer': + self._set_integer_value(value) + elif self.type == 'boolean': + self._set_boolean_value(value) + else: + raise TypeError, 'unknown type (%s)' % self.type + + def get_value(self): + if self._value == None: + return self._value + if self.type == 'string': + return self._value + elif self.type == 'integer': + if not isinstance(self._value, int) and not isinstance(self._value, long): + self._set_integer_value(self._value) + return self._value + elif self.type == 'boolean': + if not isinstance(self._value, bool): + self._set_boolean_value(self._value) + return self._value + else: + raise TypeError, 'unknown type (%s)' % self.type + + value = property(get_value, set_value, 'The value of the parameter') + + def apply(self, immediate=False): + if immediate: + self.apply_method = 'immediate' + else: + self.apply_method = 'pending-reboot' + self.group.connection.modify_parameter_group(self.group.name, [self]) + diff --git a/api/boto/resultset.py b/api/boto/resultset.py new file mode 100644 index 0000000..aab1b68 --- /dev/null +++ b/api/boto/resultset.py @@ -0,0 +1,130 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class ResultSet(list): + """ + The ResultSet is used to pass results back from the Amazon services + to the client. It has an ugly but workable mechanism for parsing + the XML results from AWS. Because I don't really want any dependencies + on external libraries, I'm using the standard SAX parser that comes + with Python. The good news is that it's quite fast and efficient but + it makes some things rather difficult. + + You can pass in, as the marker_elem parameter, a list of tuples. + Each tuple contains a string as the first element which represents + the XML element that the resultset needs to be on the lookout for + and a Python class as the second element of the tuple. Each time the + specified element is found in the XML, a new instance of the class + will be created and popped onto the stack. + + """ + + def __init__(self, marker_elem=None): + list.__init__(self) + if isinstance(marker_elem, list): + self.markers = marker_elem + else: + self.markers = [] + self.marker = None + self.is_truncated = False + self.next_token = None + self.status = True + + def startElement(self, name, attrs, connection): + for t in self.markers: + if name == t[0]: + obj = t[1](connection) + self.append(obj) + return obj + return None + + def to_boolean(self, value, true_value='true'): + if value == true_value: + return True + else: + return False + + def endElement(self, name, value, connection): + if name == 'IsTruncated': + self.is_truncated = self.to_boolean(value) + elif name == 'Marker': + self.marker = value + elif name == 'Prefix': + self.prefix = value + elif name == 'return': + self.status = self.to_boolean(value) + elif name == 'StatusCode': + self.status = self.to_boolean(value, 'Success') + elif name == 'ItemName': + self.append(value) + elif name == 'NextToken': + self.next_token = value + elif name == 'BoxUsage': + try: + connection.box_usage += float(value) + except: + pass + elif name == 'IsValid': + self.status = self.to_boolean(value, 'True') + else: + setattr(self, name, value) + +class BooleanResult(object): + + def __init__(self, marker_elem=None): + self.status = True + self.request_id = None + self.box_usage = None + + def __repr__(self): + if self.status: + return 'True' + else: + return 'False' + + def __nonzero__(self): + return self.status + + def startElement(self, name, attrs, connection): + return None + + def to_boolean(self, value, true_value='true'): + if value == true_value: + return True + else: + return False + + def endElement(self, name, value, connection): + if name == 'return': + self.status = self.to_boolean(value) + elif name == 'StatusCode': + self.status = self.to_boolean(value, 'Success') + elif name == 'IsValid': + self.status = self.to_boolean(value, 'True') + elif name == 'RequestId': + self.request_id = value + elif name == 'requestId': + self.request_id = value + elif name == 'BoxUsage': + self.request_id = value + else: + setattr(self, name, value) + diff --git a/api/boto/s3/__init__.py b/api/boto/s3/__init__.py new file mode 100644 index 0000000..be2de1d --- /dev/null +++ b/api/boto/s3/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto + +boto.check_extensions(__name__, __path__) + +from connection import S3Connection as Connection +from key import Key +from bucket import Bucket + +__all__ = ['Connection', 'Key', 'Bucket'] diff --git a/api/boto/s3/acl.py b/api/boto/s3/acl.py new file mode 100644 index 0000000..702551e --- /dev/null +++ b/api/boto/s3/acl.py @@ -0,0 +1,161 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.s3.user import User +import StringIO + +CannedACLStrings = ['private', 'public-read', + 'public-read-write', 'authenticated-read'] + +class Policy: + + def __init__(self, parent=None): + self.parent = parent + self.acl = None + + def __repr__(self): + grants = [] + for g in self.acl.grants: + if g.id == self.owner.id: + grants.append("%s (owner) = %s" % (g.display_name, g.permission)) + else: + if g.type == 'CanonicalUser': + u = g.display_name + elif g.type == 'Group': + u = g.uri + else: + u = g.email + grants.append("%s = %s" % (u, g.permission)) + return "" % ", ".join(grants) + + def startElement(self, name, attrs, connection): + if name == 'Owner': + self.owner = User(self) + return self.owner + elif name == 'AccessControlList': + self.acl = ACL(self) + return self.acl + else: + return None + + def endElement(self, name, value, connection): + if name == 'Owner': + pass + elif name == 'AccessControlList': + pass + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + s += self.owner.to_xml() + s += self.acl.to_xml() + s += '' + return s + +class ACL: + + def __init__(self, policy=None): + self.policy = policy + self.grants = [] + + def add_grant(self, grant): + self.grants.append(grant) + + def add_email_grant(self, permission, email_address): + grant = Grant(permission=permission, type='AmazonCustomerByEmail', + email_address=email_address) + self.grants.append(grant) + + def add_user_grant(self, permission, user_id): + grant = Grant(permission=permission, type='CanonicalUser', id=user_id) + self.grants.append(grant) + + def startElement(self, name, attrs, connection): + if name == 'Grant': + self.grants.append(Grant(self)) + return self.grants[-1] + else: + return None + + def endElement(self, name, value, connection): + if name == 'Grant': + pass + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + for grant in self.grants: + s += grant.to_xml() + s += '' + return s + +class Grant: + + NameSpace = 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' + + def __init__(self, permission=None, type=None, id=None, + display_name=None, uri=None, email_address=None): + self.permission = permission + self.id = id + self.display_name = display_name + self.uri = uri + self.email_address = email_address + self.type = type + + def startElement(self, name, attrs, connection): + if name == 'Grantee': + self.type = attrs['xsi:type'] + return None + + def endElement(self, name, value, connection): + if name == 'ID': + self.id = value + elif name == 'DisplayName': + self.display_name = value + elif name == 'URI': + self.uri = value + elif name == 'EmailAddress': + self.email_address = value + elif name == 'Grantee': + pass + elif name == 'Permission': + self.permission = value + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + s += '' % (self.NameSpace, self.type) + if self.type == 'CanonicalUser': + s += '%s' % self.id + s += '%s' % self.display_name + elif self.type == 'Group': + s += '%s' % self.uri + else: + s += '%s' % self.email_address + s += '' + s += '%s' % self.permission + s += '' + return s + + diff --git a/api/boto/s3/bucket.py b/api/boto/s3/bucket.py new file mode 100644 index 0000000..297f0a2 --- /dev/null +++ b/api/boto/s3/bucket.py @@ -0,0 +1,495 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto import handler +from boto.resultset import ResultSet +from boto.s3.acl import Policy, CannedACLStrings, ACL, Grant +from boto.s3.user import User +from boto.s3.key import Key +from boto.s3.prefix import Prefix +from boto.exception import S3ResponseError, S3PermissionsError, S3CopyError +from boto.s3.bucketlistresultset import BucketListResultSet +import boto.utils +import xml.sax +import urllib + +S3Permissions = ['READ', 'WRITE', 'READ_ACP', 'WRITE_ACP', 'FULL_CONTROL'] + +class Bucket: + + BucketLoggingBody = """ + + + %s + %s + + """ + + EmptyBucketLoggingBody = """ + + """ + + LoggingGroup = 'http://acs.amazonaws.com/groups/s3/LogDelivery' + + BucketPaymentBody = """ + + %s + """ + + def __init__(self, connection=None, name=None, key_class=Key): + self.name = name + self.connection = connection + self.key_class = key_class + + def __repr__(self): + return '' % self.name + + def __iter__(self): + return iter(BucketListResultSet(self)) + + def __contains__(self, key_name): + return not (self.get_key(key_name) is None) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Name': + self.name = value + elif name == 'CreationDate': + self.creation_date = value + else: + setattr(self, name, value) + + def set_key_class(self, key_class): + """ + Set the Key class associated with this bucket. By default, this + would be the boto.s3.key.Key class but if you want to subclass that + for some reason this allows you to associate your new class with a + bucket so that when you call bucket.new_key() or when you get a listing + of keys in the bucket you will get an instances of your key class + rather than the default. + + :type key_class: class + :param key_class: A subclass of Key that can be more specific + """ + self.key_class = key_class + + def lookup(self, key_name, headers=None): + """ + Deprecated: Please use get_key method. + + :type key_name: string + :param key_name: The name of the key to retrieve + + :rtype: :class:`boto.s3.key.Key` + :returns: A Key object from this bucket. + """ + return self.get_key(key_name, headers=headers) + + def get_key(self, key_name, headers=None): + """ + Check to see if a particular key exists within the bucket. This + method uses a HEAD request to check for the existance of the key. + Returns: An instance of a Key object or None + + :type key_name: string + :param key_name: The name of the key to retrieve + + :rtype: :class:`boto.s3.key.Key` + :returns: A Key object from this bucket. + """ + response = self.connection.make_request('HEAD', self.name, key_name, headers=headers) + if response.status == 200: + body = response.read() + k = self.key_class(self) + k.metadata = boto.utils.get_aws_metadata(response.msg) + k.etag = response.getheader('etag') + k.content_type = response.getheader('content-type') + k.content_encoding = response.getheader('content-encoding') + k.last_modified = response.getheader('last-modified') + k.size = int(response.getheader('content-length')) + k.name = key_name + return k + else: + if response.status == 404: + body = response.read() + return None + else: + raise S3ResponseError(response.status, response.reason, '') + + def list(self, prefix='', delimiter='', marker='', headers=None): + """ + List key objects within a bucket. This returns an instance of an + BucketListResultSet that automatically handles all of the result + paging, etc. from S3. You just need to keep iterating until + there are no more results. + Called with no arguments, this will return an iterator object across + all keys within the bucket. + + :type prefix: string + :param prefix: allows you to limit the listing to a particular + prefix. For example, if you call the method with prefix='/foo/' + then the iterator will only cycle through the keys that begin with + the string '/foo/'. + + :type delimiter: string + :param delimiter: can be used in conjunction with the prefix + to allow you to organize and browse your keys hierarchically. See: + http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ + for more details. + + :type marker: string + :param marker: The "marker" of where you are in the result set + + :rtype: :class:`boto.s3.bucketlistresultset.BucketListResultSet` + :return: an instance of a BucketListResultSet that handles paging, etc + """ + return BucketListResultSet(self, prefix, delimiter, marker, headers) + + def get_all_keys(self, headers=None, **params): + """ + A lower-level method for listing contents of a bucket. This closely models the actual S3 + API and requires you to manually handle the paging of results. For a higher-level method + that handles the details of paging for you, you can use the list method. + + :type maxkeys: int + :param maxkeys: The maximum number of keys to retrieve + + :type prefix: string + :param prefix: The prefix of the keys you want to retrieve + + :type marker: string + :param marker: The "marker" of where you are in the result set + + :type delimiter: string + :param delimiter: "If this optional, Unicode string parameter is included with your request, then keys that contain the same string between the prefix and the first occurrence of the delimiter will be rolled up into a single result element in the CommonPrefixes collection. These rolled-up keys are not returned elsewhere in the response." + + :rtype: ResultSet + :return: The result from S3 listing the keys requested + + """ + l = [] + for k,v in params.items(): + if k == 'maxkeys': + k = 'max-keys' + if isinstance(v, unicode): + v = v.encode('utf-8') + if v is not None: + l.append('%s=%s' % (urllib.quote(k), urllib.quote(str(v)))) + if len(l): + s = '&'.join(l) + else: + s = None + response = self.connection.make_request('GET', self.name, + headers=headers, query_args=s) + body = response.read() + boto.log.debug(body) + if response.status == 200: + rs = ResultSet([('Contents', self.key_class), + ('CommonPrefixes', Prefix)]) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise S3ResponseError(response.status, response.reason, body) + + def new_key(self, key_name=None): + """ + Creates a new key + + :type key_name: string + :param key_name: The name of the key to create + + :rtype: :class:`boto.s3.key.Key` or subclass + :returns: An instance of the newly created key object + """ + return self.key_class(self, key_name) + + def generate_url(self, expires_in, method='GET', headers=None, force_http=False): + return self.connection.generate_url(expires_in, method, self.name, headers=headers, + force_http=force_http) + + def delete_key(self, key_name, headers=None): + """ + Deletes a key from the bucket. + + :type key_name: string + :param key_name: The key name to delete + """ + response = self.connection.make_request('DELETE', self.name, key_name, headers=headers) + body = response.read() + if response.status != 204: + raise S3ResponseError(response.status, response.reason, body) + + def copy_key(self, new_key_name, src_bucket_name, src_key_name, metadata=None): + """ + Create a new key in the bucket by copying another existing key. + + :type new_key_name: string + :param new_key_name: The name of the new key + + :type src_bucket_name: string + :param src_bucket_name: The name of the source bucket + + :type src_key_name: string + :param src_key_name: The name of the source key + + :type metadata: dict + :param metadata: Metadata to be associated with new key. + If metadata is supplied, it will replace the + metadata of the source key being copied. + If no metadata is supplied, the source key's + metadata will be copied to the new key. + + :rtype: :class:`boto.s3.key.Key` or subclass + :returns: An instance of the newly created key object + """ + src = '%s/%s' % (src_bucket_name, urllib.quote(src_key_name)) + if metadata: + headers = {'x-amz-copy-source' : src, + 'x-amz-metadata-directive' : 'REPLACE'} + headers = boto.utils.merge_meta(headers, metadata) + else: + headers = {'x-amz-copy-source' : src, + 'x-amz-metadata-directive' : 'COPY'} + response = self.connection.make_request('PUT', self.name, new_key_name, + headers=headers) + body = response.read() + if response.status == 200: + key = self.new_key(new_key_name) + h = handler.XmlHandler(key, self) + xml.sax.parseString(body, h) + if hasattr(key, 'Error'): + raise S3CopyError(key.Code, key.Message, body) + return key + else: + raise S3ResponseError(response.status, response.reason, body) + + def set_canned_acl(self, acl_str, key_name='', headers=None): + assert acl_str in CannedACLStrings + + if headers: + headers['x-amz-acl'] = acl_str + else: + headers={'x-amz-acl': acl_str} + + response = self.connection.make_request('PUT', self.name, key_name, + headers=headers, query_args='acl') + body = response.read() + if response.status != 200: + raise S3ResponseError(response.status, response.reason, body) + + def get_xml_acl(self, key_name='', headers=None): + response = self.connection.make_request('GET', self.name, key_name, + query_args='acl', headers=headers) + body = response.read() + if response.status != 200: + raise S3ResponseError(response.status, response.reason, body) + return body + + def set_xml_acl(self, acl_str, key_name='', headers=None): + response = self.connection.make_request('PUT', self.name, key_name, + data=acl_str, query_args='acl', headers=headers) + body = response.read() + if response.status != 200: + raise S3ResponseError(response.status, response.reason, body) + + def set_acl(self, acl_or_str, key_name='', headers=None): + if isinstance(acl_or_str, Policy): + self.set_xml_acl(acl_or_str.to_xml(), key_name, headers=headers) + else: + self.set_canned_acl(acl_or_str, key_name, headers=headers) + + def get_acl(self, key_name='', headers=None): + response = self.connection.make_request('GET', self.name, key_name, + query_args='acl', headers=headers) + body = response.read() + if response.status == 200: + policy = Policy(self) + h = handler.XmlHandler(policy, self) + xml.sax.parseString(body, h) + return policy + else: + raise S3ResponseError(response.status, response.reason, body) + + def make_public(self, recursive=False, headers=None): + self.set_canned_acl('public-read', headers=headers) + if recursive: + for key in self: + self.set_canned_acl('public-read', key.name, headers=headers) + + def add_email_grant(self, permission, email_address, recursive=False, headers=None): + """ + Convenience method that provides a quick way to add an email grant to a bucket. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + + :param permission: The permission being granted. Should be one of: (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html for more details on permissions. + :type permission: string + + :param email_address: The email address associated with the AWS account your are granting + the permission to. + :type email_address: string + + :param recursive: A boolean value to controls whether the command will apply the + grant to all keys within the bucket or not. The default value is False. + By passing a True value, the call will iterate through all keys in the + bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take a long time! + :type recursive: boolean + """ + if permission not in S3Permissions: + raise S3PermissionsError('Unknown Permission: %s' % permission) + policy = self.get_acl(headers=headers) + policy.acl.add_email_grant(permission, email_address) + self.set_acl(policy, headers=headers) + if recursive: + for key in self: + key.add_email_grant(permission, email_address, headers=headers) + + def add_user_grant(self, permission, user_id, recursive=False, headers=None): + """ + Convenience method that provides a quick way to add a canonical user grant to a bucket. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + + :type permission: string + :param permission: The permission being granted. Should be one of: + READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html + for more details on permissions. + + :type user_id: string + :param user_id: The canonical user id associated with the AWS account your are granting + the permission to. + + :type recursive: bool + :param recursive: A boolean value that controls whether the command will apply the + grant to all keys within the bucket or not. The default value is False. + By passing a True value, the call will iterate through all keys in the + bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take a long time! + """ + if permission not in S3Permissions: + raise S3PermissionsError('Unknown Permission: %s' % permission) + policy = self.get_acl(headers=headers) + policy.acl.add_user_grant(permission, user_id) + self.set_acl(policy, headers=headers) + if recursive: + for key in self: + key.add_user_grant(permission, user_id, headers=headers) + + def list_grants(self, headers=None): + policy = self.get_acl(headers=headers) + return policy.acl.grants + + def get_location(self): + """ + Returns the LocationConstraint for the bucket. + + :rtype: str + :return: The LocationConstraint for the bucket or the empty string if + no constraint was specified when bucket was created. + """ + response = self.connection.make_request('GET', self.name, + query_args='location') + body = response.read() + if response.status == 200: + rs = ResultSet(self) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.LocationConstraint + else: + raise S3ResponseError(response.status, response.reason, body) + + def enable_logging(self, target_bucket, target_prefix='', headers=None): + if isinstance(target_bucket, Bucket): + target_bucket = target_bucket.name + body = self.BucketLoggingBody % (target_bucket, target_prefix) + response = self.connection.make_request('PUT', self.name, data=body, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise S3ResponseError(response.status, response.reason, body) + + def disable_logging(self, headers=None): + body = self.EmptyBucketLoggingBody + response = self.connection.make_request('PUT', self.name, data=body, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise S3ResponseError(response.status, response.reason, body) + + def get_logging_status(self, headers=None): + response = self.connection.make_request('GET', self.name, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return body + else: + raise S3ResponseError(response.status, response.reason, body) + + def set_as_logging_target(self, headers=None): + policy = self.get_acl(headers=headers) + g1 = Grant(permission='WRITE', type='Group', uri=self.LoggingGroup) + g2 = Grant(permission='READ_ACP', type='Group', uri=self.LoggingGroup) + policy.acl.add_grant(g1) + policy.acl.add_grant(g2) + self.set_acl(policy, headers=headers) + + def disable_logging(self, headers=None): + body = self.EmptyBucketLoggingBody + response = self.connection.make_request('PUT', self.name, data=body, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise S3ResponseError(response.status, response.reason, body) + + def get_request_payment(self, headers=None): + response = self.connection.make_request('GET', self.name, + query_args='requestPayment', headers=headers) + body = response.read() + if response.status == 200: + return body + else: + raise S3ResponseError(response.status, response.reason, body) + + def set_request_payment(self, payer='BucketOwner', headers=None): + body = self.BucketPaymentBody % payer + response = self.connection.make_request('PUT', self.name, data=body, + query_args='requestPayment', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise S3ResponseError(response.status, response.reason, body) + + def delete(self, headers=None): + return self.connection.delete_bucket(self.name, headers=headers) diff --git a/api/boto/s3/bucketlistresultset.py b/api/boto/s3/bucketlistresultset.py new file mode 100644 index 0000000..66ed4ee --- /dev/null +++ b/api/boto/s3/bucketlistresultset.py @@ -0,0 +1,57 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +def bucket_lister(bucket, prefix='', delimiter='', marker='', headers=None): + """ + A generator function for listing keys in a bucket. + """ + more_results = True + k = None + while more_results: + rs = bucket.get_all_keys(prefix=prefix, marker=marker, + delimiter=delimiter, headers=headers) + for k in rs: + yield k + if k: + marker = k.name + more_results= rs.is_truncated + +class BucketListResultSet: + """ + A resultset for listing keys within a bucket. Uses the bucket_lister + generator function and implements the iterator interface. This + transparently handles the results paging from S3 so even if you have + many thousands of keys within the bucket you can iterate over all + keys in a reasonably efficient manner. + """ + + def __init__(self, bucket=None, prefix='', delimiter='', marker='', headers=None): + self.bucket = bucket + self.prefix = prefix + self.delimiter = delimiter + self.marker = marker + self.headers = headers + + def __iter__(self): + return bucket_lister(self.bucket, prefix=self.prefix, + delimiter=self.delimiter, marker=self.marker, headers=self.headers) + + diff --git a/api/boto/s3/connection.py b/api/boto/s3/connection.py new file mode 100644 index 0000000..e366f7e --- /dev/null +++ b/api/boto/s3/connection.py @@ -0,0 +1,343 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax +import urllib, base64 +import time +import boto.utils +import types +from boto.connection import AWSAuthConnection +from boto import handler +from boto.s3.bucket import Bucket +from boto.s3.key import Key +from boto.resultset import ResultSet +from boto.exception import S3ResponseError, S3CreateError, BotoClientError + +def assert_case_insensitive(f): + def wrapper(*args, **kwargs): + if len(args) == 3 and not (args[2].islower() or args[2].isalnum()): + raise BotoClientError("Bucket names cannot contain upper-case " \ + "characters when using either the sub-domain or virtual " \ + "hosting calling format.") + return f(*args, **kwargs) + return wrapper + +class _CallingFormat: + def build_url_base(self, protocol, server, bucket, key=''): + url_base = '%s://' % protocol + url_base += self.build_host(server, bucket) + url_base += self.build_path_base(bucket, key) + return url_base + + def build_host(self, server, bucket): + if bucket == '': + return server + else: + return self.get_bucket_server(server, bucket) + + def build_auth_path(self, bucket, key=''): + path = '' + if bucket != '': + path = '/' + bucket + return path + '/%s' % urllib.quote(key) + + def build_path_base(self, bucket, key=''): + return '/%s' % urllib.quote(key) + +class SubdomainCallingFormat(_CallingFormat): + @assert_case_insensitive + def get_bucket_server(self, server, bucket): + return '%s.%s' % (bucket, server) + +class VHostCallingFormat(_CallingFormat): + @assert_case_insensitive + def get_bucket_server(self, server, bucket): + return bucket + +class OrdinaryCallingFormat(_CallingFormat): + def get_bucket_server(self, server, bucket): + return server + + def build_path_base(self, bucket, key=''): + path_base = '/' + if bucket: + path_base += "%s/" % bucket + return path_base + urllib.quote(key) + +class Location: + DEFAULT = '' + EU = 'EU' + +class S3Connection(AWSAuthConnection): + + DefaultHost = 's3.amazonaws.com' + QueryString = 'Signature=%s&Expires=%d&AWSAccessKeyId=%s' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, + host=DefaultHost, debug=0, https_connection_factory=None, + calling_format=SubdomainCallingFormat(), path='/'): + self.calling_format = calling_format + AWSAuthConnection.__init__(self, host, + aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + debug=debug, https_connection_factory=https_connection_factory, + path=path) + + def __iter__(self): + return self.get_all_buckets() + + def __contains__(self, bucket_name): + return not (self.lookup(bucket_name) is None) + + def build_post_policy(self, expiration_time, conditions): + """ + Taken from the AWS book Python examples and modified for use with boto + """ + if type(expiration_time) != time.struct_time: + raise 'Policy document must include a valid expiration Time object' + + # Convert conditions object mappings to condition statements + + return '{"expiration": "%s",\n"conditions": [%s]}' % \ + (time.strftime(boto.utils.ISO8601, expiration_time), ",".join(conditions)) + + + def build_post_form_args(self, bucket_name, key, expires_in = 6000, + acl = None, success_action_redirect = None, max_content_length = None, + http_method = "http", fields=None, conditions=None): + """ + Taken from the AWS book Python examples and modified for use with boto + This only returns the arguments required for the post form, not the actual form + This does not return the file input field which also needs to be added + + :param bucket_name: Bucket to submit to + :type bucket_name: string + + :param key: Key name, optionally add ${filename} to the end to attach the submitted filename + :type key: string + + :param expires_in: Time (in seconds) before this expires, defaults to 6000 + :type expires_in: integer + + :param acl: ACL rule to use, if any + :type acl: :class:`boto.s3.acl.ACL` + + :param success_action_redirect: URL to redirect to on success + :type success_action_redirect: string + + :param max_content_length: Maximum size for this file + :type max_content_length: integer + + :type http_method: string + :param http_method: HTTP Method to use, "http" or "https" + + + :rtype: dict + :return: A dictionary containing field names/values as well as a url to POST to + + .. code-block:: python + + { + "action": action_url_to_post_to, + "fields": [ + { + "name": field_name, + "value": field_value + }, + { + "name": field_name2, + "value": field_value2 + } + ] + } + + """ + if fields == None: + fields = [] + if conditions == None: + conditions = [] + expiration = time.gmtime(int(time.time() + expires_in)) + + # Generate policy document + conditions.append('{"bucket": "%s"}' % bucket_name) + if key.endswith("${filename}"): + conditions.append('["starts-with", "$key", "%s"]' % key[:-len("${filename}")]) + else: + conditions.append('{"key": "%s"}' % key) + if acl: + conditions.append('{"acl": "%s"}' % acl) + fields.append({ "name": "acl", "value": acl}) + if success_action_redirect: + conditions.append('{"success_action_redirect": "%s"}' % success_action_redirect) + fields.append({ "name": "success_action_redirect", "value": success_action_redirect}) + if max_content_length: + conditions.append('["content-length-range", 0, %i]' % max_content_length) + fields.append({"name":'content-length-range', "value": "0,%i" % max_content_length}) + + policy = self.build_post_policy(expiration, conditions) + + # Add the base64-encoded policy document as the 'policy' field + policy_b64 = base64.b64encode(policy) + fields.append({"name": "policy", "value": policy_b64}) + + # Add the AWS access key as the 'AWSAccessKeyId' field + fields.append({"name": "AWSAccessKeyId", "value": self.aws_access_key_id}) + + # Add signature for encoded policy document as the 'AWSAccessKeyId' field + hmac_copy = self.hmac.copy() + hmac_copy.update(policy_b64) + signature = base64.encodestring(hmac_copy.digest()).strip() + fields.append({"name": "signature", "value": signature}) + fields.append({"name": "key", "value": key}) + + # HTTPS protocol will be used if the secure HTTP option is enabled. + url = '%s://%s.s3.amazonaws.com/' % (http_method, bucket_name) + + return {"action": url, "fields": fields} + + + def generate_url(self, expires_in, method, bucket='', key='', + headers=None, query_auth=True, force_http=False): + if not headers: + headers = {} + expires = int(time.time() + expires_in) + auth_path = self.calling_format.build_auth_path(bucket, key) + canonical_str = boto.utils.canonical_string(method, auth_path, + headers, expires) + hmac_copy = self.hmac.copy() + hmac_copy.update(canonical_str) + b64_hmac = base64.encodestring(hmac_copy.digest()).strip() + encoded_canonical = urllib.quote_plus(b64_hmac) + path = self.calling_format.build_path_base(bucket, key) + if query_auth: + query_part = '?' + self.QueryString % (encoded_canonical, expires, + self.aws_access_key_id) + if 'x-amz-security-token' in headers: + query_part += '&x-amz-security-token=%s' % urllib.quote(headers['x-amz-security-token']); + else: + query_part = '' + if force_http: + protocol = 'http' + port = 80 + else: + protocol = self.protocol + port = self.port + return self.calling_format.build_url_base(protocol, self.server_name(port), + bucket, key) + query_part + + def get_all_buckets(self, headers=None): + response = self.make_request('GET') + body = response.read() + if response.status > 300: + raise S3ResponseError(response.status, response.reason, body, headers=headers) + rs = ResultSet([('Bucket', Bucket)]) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def get_canonical_user_id(self, headers=None): + """ + Convenience method that returns the "CanonicalUserID" of the user who's credentials + are associated with the connection. The only way to get this value is to do a GET + request on the service which returns all buckets associated with the account. As part + of that response, the canonical userid is returned. This method simply does all of + that and then returns just the user id. + + :rtype: string + :return: A string containing the canonical user id. + """ + rs = self.get_all_buckets(headers=headers) + return rs.ID + + def get_bucket(self, bucket_name, validate=True, headers=None): + bucket = Bucket(self, bucket_name) + if validate: + rs = bucket.get_all_keys(headers, maxkeys=0) + return bucket + + def lookup(self, bucket_name, validate=True, headers=None): + try: + bucket = self.get_bucket(bucket_name, validate, headers=headers) + except: + bucket = None + return bucket + + def create_bucket(self, bucket_name, headers=None, location=Location.DEFAULT, policy=None): + """ + Creates a new located bucket. By default it's in the USA. You can pass + Location.EU to create an European bucket. + + :type bucket_name: string + :param bucket_name: The name of the new bucket + + :type headers: dict + :param headers: Additional headers to pass along with the request to AWS. + + :type location: :class:`boto.s3.connection.Location` + :param location: The location of the new bucket + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + """ + if policy: + if headers: + headers['x-amz-acl'] = policy + else: + headers = {'x-amz-acl' : policy} + if location == Location.DEFAULT: + data = '' + else: + data = '' + \ + location + '' + response = self.make_request('PUT', bucket_name, headers=headers, + data=data) + body = response.read() + if response.status == 409: + raise S3CreateError(response.status, response.reason, body) + if response.status == 200: + return Bucket(self, bucket_name) + else: + raise S3ResponseError(response.status, response.reason, body) + + def delete_bucket(self, bucket, headers=None): + response = self.make_request('DELETE', bucket, headers=headers) + body = response.read() + if response.status != 204: + raise S3ResponseError(response.status, response.reason, body) + + def make_request(self, method, bucket='', key='', headers=None, data='', + query_args=None, sender=None): + if isinstance(bucket, Bucket): + bucket = bucket.name + if isinstance(key, Key): + key = key.name + path = self.calling_format.build_path_base(bucket, key) + auth_path = self.calling_format.build_auth_path(bucket, key) + host = self.calling_format.build_host(self.server_name(), bucket) + if query_args: + path += '?' + query_args + auth_path += '?' + query_args + return AWSAuthConnection.make_request(self, method, path, headers, + data, host, auth_path, sender) + diff --git a/api/boto/s3/key.py b/api/boto/s3/key.py new file mode 100644 index 0000000..ada4352 --- /dev/null +++ b/api/boto/s3/key.py @@ -0,0 +1,764 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import mimetypes +import os +import rfc822 +import StringIO +import base64 +import boto.utils +from boto.exception import S3ResponseError, S3DataError, BotoClientError +from boto.s3.user import User +from boto import UserAgent, config +try: + from hashlib import md5 +except ImportError: + from md5 import md5 + +class Key(object): + + DefaultContentType = 'application/octet-stream' + + BufferSize = 8192 + + def __init__(self, bucket=None, name=None): + self.bucket = bucket + self.name = name + self.metadata = {} + self.content_type = self.DefaultContentType + self.content_encoding = None + self.filename = None + self.etag = None + self.last_modified = None + self.owner = None + self.storage_class = None + self.md5 = None + self.base64md5 = None + self.path = None + self.resp = None + self.mode = None + self.size = None + + def __repr__(self): + if self.bucket: + return '' % (self.bucket.name, self.name) + else: + return '' % self.name + + def __getattr__(self, name): + if name == 'key': + return self.name + else: + raise AttributeError + + def __setattr__(self, name, value): + if name == 'key': + self.__dict__['name'] = value + else: + self.__dict__[name] = value + + def __iter__(self): + return self + + def open_read(self, headers=None, query_args=None): + """ + Open this key for reading + + :type headers: dict + :param headers: Headers to pass in the web request + + :type query_args: string + :param query_args: Arguments to pass in the query string (ie, 'torrent') + """ + if self.resp == None: + self.mode = 'r' + + self.resp = self.bucket.connection.make_request('GET', self.bucket.name, self.name, headers, query_args=query_args) + if self.resp.status < 199 or self.resp.status > 299: + raise S3ResponseError(self.resp.status, self.resp.reason) + response_headers = self.resp.msg + self.metadata = boto.utils.get_aws_metadata(response_headers) + for name,value in response_headers.items(): + if name.lower() == 'content-length': + self.size = int(value) + elif name.lower() == 'etag': + self.etag = value + elif name.lower() == 'content-type': + self.content_type = value + elif name.lower() == 'content-encoding': + self.content_encoding = value + elif name.lower() == 'last-modified': + self.last_modified = value + + def open_write(self, headers=None): + """ + Open this key for writing. + Not yet implemented + + :type headers: dict + :param headers: Headers to pass in the write request + """ + raise BotoClientError('Not Implemented') + + def open(self, mode='r', headers=None, query_args=None): + if mode == 'r': + self.mode = 'r' + self.open_read(headers=headers, query_args=query_args) + elif mode == 'w': + self.mode = 'w' + self.open_write(headers=headers) + else: + raise BotoClientError('Invalid mode: %s' % mode) + + closed = False + def close(self): + if self.resp: + self.resp.read() + self.resp = None + self.mode = None + self.closed = True + + def next(self): + """ + By providing a next method, the key object supports use as an iterator. + For example, you can now say: + + for bytes in key: + write bytes to a file or whatever + + All of the HTTP connection stuff is handled for you. + """ + self.open_read() + data = self.resp.read(self.BufferSize) + if not data: + self.close() + raise StopIteration + return data + + def read(self, size=0): + if size == 0: + size = self.BufferSize + self.open_read() + data = self.resp.read(size) + if not data: + self.close() + return data + + def copy(self, dst_bucket, dst_key, metadata=None): + """ + Copy this Key to another bucket. + + :type dst_bucket: string + :param dst_bucket: The name of the destination bucket + + :type dst_key: string + :param dst_key: The name of the destinatino key + + :type metadata: dict + :param metadata: Metadata to be associated with new key. + If metadata is supplied, it will replace the + metadata of the source key being copied. + If no metadata is supplied, the source key's + metadata will be copied to the new key. + + :rtype: :class:`boto.s3.key.Key` or subclass + :returns: An instance of the newly created key object + """ + dst_bucket = self.bucket.connection.lookup(dst_bucket) + return dst_bucket.copy_key(dst_key, self.bucket.name, self.name, metadata) + + def startElement(self, name, attrs, connection): + if name == 'Owner': + self.owner = User(self) + return self.owner + else: + return None + + def endElement(self, name, value, connection): + if name == 'Key': + self.name = value.encode('utf-8') + elif name == 'ETag': + self.etag = value + elif name == 'LastModified': + self.last_modified = value + elif name == 'Size': + self.size = int(value) + elif name == 'StorageClass': + self.storage_class = value + elif name == 'Owner': + pass + else: + setattr(self, name, value) + + def exists(self): + """ + Returns True if the key exists + + :rtype: bool + :return: Whether the key exists on S3 + """ + return bool(self.bucket.lookup(self.name)) + + def delete(self): + """ + Delete this key from S3 + """ + return self.bucket.delete_key(self.name) + + def get_metadata(self, name): + return self.metadata.get(name) + + def set_metadata(self, name, value): + self.metadata[name] = value + + def update_metadata(self, d): + self.metadata.update(d) + + # convenience methods for setting/getting ACL + def set_acl(self, acl_str, headers=None): + if self.bucket != None: + self.bucket.set_acl(acl_str, self.name, headers=headers) + + def get_acl(self, headers=None): + if self.bucket != None: + return self.bucket.get_acl(self.name, headers=headers) + + def get_xml_acl(self, headers=None): + if self.bucket != None: + return self.bucket.get_xml_acl(self.name, headers=headers) + + def set_xml_acl(self, acl_str, headers=None): + if self.bucket != None: + return self.bucket.set_xml_acl(acl_str, self.name, headers=headers) + + def set_canned_acl(self, acl_str, headers=None): + return self.bucket.set_canned_acl(acl_str, self.name, headers) + + def make_public(self, headers=None): + return self.bucket.set_canned_acl('public-read', self.name, headers) + + def generate_url(self, expires_in, method='GET', headers=None, + query_auth=True, force_http=False): + """ + Generate a URL to access this key. + + :type expires_in: int + :param expires_in: How long the url is valid for, in seconds + + :type method: string + :param method: The method to use for retrieving the file (default is GET) + + :type headers: dict + :param headers: Any headers to pass along in the request + + :type query_auth: bool + :param query_auth: + + :rtype: string + :return: The URL to access the key + """ + return self.bucket.connection.generate_url(expires_in, method, + self.bucket.name, self.name, + headers, query_auth, force_http) + + def send_file(self, fp, headers=None, cb=None, num_cb=10): + """ + Upload a file to a key into a bucket on S3. + + :type fp: file + :param fp: The file pointer to upload + + :type headers: dict + :param headers: The headers to pass along with the PUT request + + :type cb: function + :param cb: a callback function that will be called to report + progress on the upload. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted to S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + """ + def sender(http_conn, method, path, data, headers): + http_conn.putrequest(method, path) + for key in headers: + http_conn.putheader(key, headers[key]) + http_conn.endheaders() + fp.seek(0) + save_debug = self.bucket.connection.debug + self.bucket.connection.debug = 0 + if cb: + if num_cb > 2: + cb_count = self.size / self.BufferSize / (num_cb-2) + else: + cb_count = 0 + i = total_bytes = 0 + cb(total_bytes, self.size) + l = fp.read(self.BufferSize) + while len(l) > 0: + http_conn.send(l) + if cb: + total_bytes += len(l) + i += 1 + if i == cb_count: + cb(total_bytes, self.size) + i = 0 + l = fp.read(self.BufferSize) + if cb: + cb(total_bytes, self.size) + response = http_conn.getresponse() + body = response.read() + fp.seek(0) + self.bucket.connection.debug = save_debug + if response.status == 500 or response.status == 503 or \ + response.getheader('location'): + # we'll try again + return response + elif response.status >= 200 and response.status <= 299: + self.etag = response.getheader('etag') + if self.etag != '"%s"' % self.md5: + raise S3DataError('ETag from S3 did not match computed MD5') + return response + else: + raise S3ResponseError(response.status, response.reason, body) + + if not headers: + headers = {} + else: + headers = headers.copy() + headers['User-Agent'] = UserAgent + headers['Content-MD5'] = self.base64md5 + if headers.has_key('Content-Type'): + self.content_type = headers['Content-Type'] + elif self.path: + self.content_type = mimetypes.guess_type(self.path)[0] + if self.content_type == None: + self.content_type = self.DefaultContentType + headers['Content-Type'] = self.content_type + else: + headers['Content-Type'] = self.content_type + headers['Content-Length'] = str(self.size) + headers['Expect'] = '100-Continue' + headers = boto.utils.merge_meta(headers, self.metadata) + return self.bucket.connection.make_request('PUT', self.bucket.name, + self.name, headers, sender=sender) + + def compute_md5(self, fp): + """ + :type fp: file + :param fp: File pointer to the file to MD5 hash. The file pointer will be + reset to the beginning of the file before the method returns. + + :rtype: tuple + :return: A tuple containing the hex digest version of the MD5 hash + as the first element and the base64 encoded version of the + plain digest as the second element. + """ + m = md5() + fp.seek(0) + s = fp.read(self.BufferSize) + while s: + m.update(s) + s = fp.read(self.BufferSize) + hex_md5 = m.hexdigest() + base64md5 = base64.encodestring(m.digest()) + if base64md5[-1] == '\n': + base64md5 = base64md5[0:-1] + self.size = fp.tell() + fp.seek(0) + return (hex_md5, base64md5) + + def set_contents_from_file(self, fp, headers=None, replace=True, cb=None, num_cb=10, + policy=None, md5=None): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the contents of the file pointed to by 'fp' as the + contents. + + :type fp: file + :param fp: the file whose contents to upload + + :type headers: dict + :param headers: additional HTTP headers that will be sent with the PUT request. + + :type replace: bool + :param replace: If this parameter is False, the method + will first check to see if an object exists in the + bucket with the same key. If it does, it won't + overwrite it. The default value is True which will + overwrite the object. + + :type cb: function + :param cb: a callback function that will be called to report + progress on the upload. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted to S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum of the + file as the first element and the Base64-encoded version of the plain + checksum as the second element. This is the same format returned by + the compute_md5 method. + :param md5: If you need to compute the MD5 for any reason prior to upload, + it's silly to have to do it twice so this param, if present, will be + used as the MD5 values of the file. Otherwise, the checksum will be computed. + """ + if policy: + if headers: + headers['x-amz-acl'] = policy + else: + headers = {'x-amz-acl' : policy} + if hasattr(fp, 'name'): + self.path = fp.name + if self.bucket != None: + if not md5: + md5 = self.compute_md5(fp) + self.md5 = md5[0] + self.base64md5 = md5[1] + if self.name == None: + self.name = self.md5 + if not replace: + k = self.bucket.lookup(self.name) + if k: + return + self.send_file(fp, headers, cb, num_cb) + + def set_contents_from_filename(self, filename, headers=None, replace=True, cb=None, num_cb=10, + policy=None, md5=None): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the contents of the file named by 'filename'. + See set_contents_from_file method for details about the + parameters. + + :type filename: string + :param filename: The name of the file that you want to put onto S3 + + :type headers: dict + :param headers: Additional headers to pass along with the request to AWS. + + :type replace: bool + :param replace: If True, replaces the contents of the file if it already exists. + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum of the + file as the first element and the Base64-encoded version of the plain + checksum as the second element. This is the same format returned by + the compute_md5 method. + :param md5: If you need to compute the MD5 for any reason prior to upload, + it's silly to have to do it twice so this param, if present, will be + used as the MD5 values of the file. Otherwise, the checksum will be computed. + """ + fp = open(filename, 'rb') + self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy) + fp.close() + + def set_contents_from_string(self, s, headers=None, replace=True, cb=None, num_cb=10, + policy=None, md5=None): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the string 's' as the contents. + See set_contents_from_file method for details about the + parameters. + + :type headers: dict + :param headers: Additional headers to pass along with the request to AWS. + + :type replace: bool + :param replace: If True, replaces the contents of the file if it already exists. + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum of the + file as the first element and the Base64-encoded version of the plain + checksum as the second element. This is the same format returned by + the compute_md5 method. + :param md5: If you need to compute the MD5 for any reason prior to upload, + it's silly to have to do it twice so this param, if present, will be + used as the MD5 values of the file. Otherwise, the checksum will be computed. + """ + fp = StringIO.StringIO(s) + self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy) + fp.close() + + def get_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieves a file from an S3 Key + + :type fp: file + :param fp: File pointer to put the data into + + :type headers: string + :param: headers to send when retrieving the files + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type torrent: bool + :param torrent: Flag for whether to get a torrent for the file + """ + if cb: + if num_cb > 2: + cb_count = self.size / self.BufferSize / (num_cb-2) + else: + cb_count = 0 + i = total_bytes = 0 + cb(total_bytes, self.size) + save_debug = self.bucket.connection.debug + if self.bucket.connection.debug == 1: + self.bucket.connection.debug = 0 + + if torrent: torrent = "torrent" + self.open('r', headers, query_args=torrent) + for bytes in self: + fp.write(bytes) + if cb: + total_bytes += len(bytes) + i += 1 + if i == cb_count: + cb(total_bytes, self.size) + i = 0 + if cb: + cb(total_bytes, self.size) + self.close() + self.bucket.connection.debug = save_debug + + def get_torrent_file(self, fp, headers=None, cb=None, num_cb=10): + """ + Get a torrent file (see to get_file) + + :type fp: file + :param fp: The file pointer of where to put the torrent + + :type headers: dict + :param headers: Headers to be passed + + :type cb: function + :param cb: Callback function to call on retrieved data + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + """ + return self.get_file(fp, headers, cb, num_cb, torrent=True) + + def get_contents_to_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Write the contents of the object to the file pointed + to by 'fp'. + + :type fp: File -like object + :param fp: + + :type headers: dict + :param headers: additional HTTP headers that will be sent with the GET request. + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type torrent: bool + :param torrent: If True, returns the contents of a torrent file as a string. + + """ + if self.bucket != None: + self.get_file(fp, headers, cb, num_cb, torrent=torrent) + + def get_contents_to_filename(self, filename, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Store contents of the object to a file named by 'filename'. + See get_contents_to_file method for details about the + parameters. + + :type filename: string + :param filename: The filename of where to put the file contents + + :type headers: dict + :param headers: Any additional headers to send in the request + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type torrent: bool + :param torrent: If True, returns the contents of a torrent file as a string. + + """ + fp = open(filename, 'wb') + self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent) + fp.close() + # if last_modified date was sent from s3, try to set file's timestamp + if self.last_modified != None: + try: + modified_tuple = rfc822.parsedate_tz(self.last_modified) + modified_stamp = int(rfc822.mktime_tz(modified_tuple)) + os.utime(fp.name, (modified_stamp, modified_stamp)) + except Exception: pass + + def get_contents_as_string(self, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Return the contents of the object as a string. + See get_contents_to_file method for details about the + parameters. + + :type headers: dict + :param headers: Any additional headers to send in the request + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type torrent: bool + :param torrent: If True, returns the contents of a torrent file as a string. + + :rtype: string + :returns: The contents of the file as a string + """ + fp = StringIO.StringIO() + self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent) + return fp.getvalue() + + def add_email_grant(self, permission, email_address): + """ + Convenience method that provides a quick way to add an email grant to a key. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + + :type permission: string + :param permission: The permission being granted. Should be one of: + READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html + for more details on permissions. + + :type email_address: string + :param email_address: The email address associated with the AWS account your are granting + the permission to. + """ + policy = self.get_acl() + policy.acl.add_email_grant(permission, email_address) + self.set_acl(policy) + + def add_user_grant(self, permission, user_id): + """ + Convenience method that provides a quick way to add a canonical user grant to a key. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + + :type permission: string + :param permission: The permission being granted. Should be one of: + READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html + for more details on permissions. + + :type user_id: string + :param user_id: The canonical user id associated with the AWS account your are granting + the permission to. + """ + policy = self.get_acl() + policy.acl.add_user_grant(permission, user_id) + self.set_acl(policy) diff --git a/api/boto/s3/prefix.py b/api/boto/s3/prefix.py new file mode 100644 index 0000000..fc0f26a --- /dev/null +++ b/api/boto/s3/prefix.py @@ -0,0 +1,35 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Prefix: + def __init__(self, bucket=None, name=None): + self.bucket = bucket + self.name = name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Prefix': + self.name = value + else: + setattr(self, name, value) + diff --git a/api/boto/s3/user.py b/api/boto/s3/user.py new file mode 100644 index 0000000..f45f038 --- /dev/null +++ b/api/boto/s3/user.py @@ -0,0 +1,49 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class User: + def __init__(self, parent=None, id='', display_name=''): + if parent: + parent.owner = self + self.type = None + self.id = id + self.display_name = display_name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'DisplayName': + self.display_name = value + elif name == 'ID': + self.id = value + else: + setattr(self, name, value) + + def to_xml(self, element_name='Owner'): + if self.type: + s = '<%s xsi:type="%s">' % (element_name, self.type) + else: + s = '<%s>' % element_name + s += '%s' % self.id + s += '%s' % self.display_name + s += '' % element_name + return s diff --git a/api/boto/sdb/__init__.py b/api/boto/sdb/__init__.py new file mode 100644 index 0000000..42af6a9 --- /dev/null +++ b/api/boto/sdb/__init__.py @@ -0,0 +1,41 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto +from regioninfo import SDBRegionInfo + +def regions(): + """ + Get all available regions for the SDB service. + + :rtype: list + :return: A list of :class:`boto.sdb.regioninfo.RegionInfo` + """ + return [SDBRegionInfo(name='us-east-1', endpoint='sdb.amazonaws.com'), + SDBRegionInfo(name='eu-west-1', endpoint='sdb.eu-west-1.amazonaws.com'), + SDBRegionInfo(name='us-west-1', endpoint='sdb.us-west-1.amazonaws.com')] + +def connect_to_region(region_name): + for region in regions(): + if region.name == region_name: + return region.connect() + return None diff --git a/api/boto/sdb/connection.py b/api/boto/sdb/connection.py new file mode 100644 index 0000000..28e130a --- /dev/null +++ b/api/boto/sdb/connection.py @@ -0,0 +1,471 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import urllib +import xml.sax +import threading +import boto +from boto import handler +from boto.connection import AWSQueryConnection +from boto.sdb.domain import Domain, DomainMetaData +from boto.sdb.item import Item +from boto.sdb.regioninfo import SDBRegionInfo +from boto.exception import SDBResponseError +from boto.resultset import ResultSet +import warnings + +class ItemThread(threading.Thread): + + def __init__(self, name, domain_name, item_names): + threading.Thread.__init__(self, name=name) + print 'starting %s with %d items' % (name, len(item_names)) + self.domain_name = domain_name + self.conn = SDBConnection() + self.item_names = item_names + self.items = [] + + def run(self): + for item_name in self.item_names: + item = self.conn.get_attributes(self.domain_name, item_name) + self.items.append(item) + +class SDBConnection(AWSQueryConnection): + + DefaultRegionName = 'us-east-1' + DefaultRegionEndpoint = 'sdb.amazonaws.com' + APIVersion = '2007-11-07' + SignatureVersion = '2' + ResponseError = SDBResponseError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/', converter=None): + if not region: + region = SDBRegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + self.region.endpoint, debug, https_connection_factory, path) + self.box_usage = 0.0 + self.converter = converter + self.item_cls = Item + + def set_item_cls(self, cls): + self.item_cls = cls + + def build_name_value_list(self, params, attributes, replace=False): + keys = attributes.keys() + keys.sort() + i = 1 + for key in keys: + value = attributes[key] + if isinstance(value, list): + for v in value: + params['Attribute.%d.Name'%i] = key + if self.converter: + v = self.converter.encode(v) + params['Attribute.%d.Value'%i] = v + if replace: + params['Attribute.%d.Replace'%i] = 'true' + i += 1 + else: + params['Attribute.%d.Name'%i] = key + if self.converter: + value = self.converter.encode(value) + params['Attribute.%d.Value'%i] = value + if replace: + params['Attribute.%d.Replace'%i] = 'true' + i += 1 + + def build_batch_list(self, params, items, replace=False): + item_names = items.keys() + i = 0 + for item_name in item_names: + j = 0 + item = items[item_name] + attr_names = item.keys() + params['Item.%d.ItemName' % i] = item_name + for attr_name in attr_names: + value = item[attr_name] + if isinstance(value, list): + for v in value: + if self.converter: + v = self.converter.encode(v) + params['Item.%d.Attribute.%d.Name' % (i,j)] = attr_name + params['Item.%d.Attribute.%d.Value' % (i,j)] = v + if replace: + params['Item.%d.Attribute.%d.Replace' % (i,j)] = 'true' + j += 1 + else: + params['Item.%d.Attribute.%d.Name' % (i,j)] = attr_name + if self.converter: + value = self.converter.encode(value) + params['Item.%d.Attribute.%d.Value' % (i,j)] = value + if replace: + params['Item.%d.Attribute.%d.Replace' % (i,j)] = 'true' + j += 1 + i += 1 + + def build_name_list(self, params, attribute_names): + i = 1 + attribute_names.sort() + for name in attribute_names: + params['Attribute.%d.Name'%i] = name + i += 1 + + def get_usage(self): + """ + Returns the BoxUsage accumulated on this SDBConnection object. + + :rtype: float + :return: The accumulated BoxUsage of all requests made on the connection. + """ + return self.box_usage + + def print_usage(self): + """ + Print the BoxUsage and approximate costs of all requests made on this connection. + """ + print 'Total Usage: %f compute seconds' % self.box_usage + cost = self.box_usage * 0.14 + print 'Approximate Cost: $%f' % cost + + def get_domain(self, domain_name, validate=True): + domain = Domain(self, domain_name) + if validate: + self.select(domain, """select * from `%s` limit 1""" % domain_name) + return domain + + def lookup(self, domain_name, validate=True): + """ + Lookup an existing SimpleDB domain + + :type domain_name: string + :param domain_name: The name of the new domain + + :rtype: :class:`boto.sdb.domain.Domain` object or None + :return: The Domain object or None if the domain does not exist. + """ + try: + domain = self.get_domain(domain_name, validate) + except: + domain = None + return domain + + def get_all_domains(self, max_domains=None, next_token=None): + params = {} + if max_domains: + params['MaxNumberOfDomains'] = max_domains + if next_token: + params['NextToken'] = next_token + return self.get_list('ListDomains', params, [('DomainName', Domain)]) + + def create_domain(self, domain_name): + """ + Create a SimpleDB domain. + + :type domain_name: string + :param domain_name: The name of the new domain + + :rtype: :class:`boto.sdb.domain.Domain` object + :return: The newly created domain + """ + params = {'DomainName':domain_name} + d = self.get_object('CreateDomain', params, Domain) + d.name = domain_name + return d + + def get_domain_and_name(self, domain_or_name): + if (isinstance(domain_or_name, Domain)): + return (domain_or_name, domain_or_name.name) + else: + return (self.get_domain(domain_or_name), domain_or_name) + + def delete_domain(self, domain_or_name): + """ + Delete a SimpleDB domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :rtype: bool + :return: True if successful + + B{Note:} This will delete the domain and all items within the domain. + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name} + return self.get_status('DeleteDomain', params) + + def domain_metadata(self, domain_or_name): + """ + Get the Metadata for a SimpleDB domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :rtype: :class:`boto.sdb.domain.DomainMetaData` object + :return: The newly created domain metadata object + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name} + d = self.get_object('DomainMetadata', params, DomainMetaData) + d.domain = domain + return d + + def put_attributes(self, domain_or_name, item_name, attributes, replace=True): + """ + Store attributes for a given item in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type item_name: string + :param item_name: The name of the item whose attributes are being stored. + + :type attribute_names: dict or dict-like object + :param attribute_names: The name/value pairs to store as attributes + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name, + 'ItemName' : item_name} + self.build_name_value_list(params, attributes, replace) + return self.get_status('PutAttributes', params) + + def batch_put_attributes(self, domain_or_name, items, replace=True): + """ + Store attributes for multiple items in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type items: dict or dict-like object + :param items: A dictionary-like object. The keys of the dictionary are + the item names and the values are themselves dictionaries + of attribute names/values, exactly the same as the + attribute_names parameter of the scalar put_attributes + call. + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name} + self.build_batch_list(params, items, replace) + return self.get_status('BatchPutAttributes', params, verb='POST') + + def get_attributes(self, domain_or_name, item_name, attribute_names=None, item=None): + """ + Retrieve attributes for a given item in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type item_name: string + :param item_name: The name of the item whose attributes are being retrieved. + + :type attribute_names: string or list of strings + :param attribute_names: An attribute name or list of attribute names. This + parameter is optional. If not supplied, all attributes + will be retrieved for the item. + + :rtype: :class:`boto.sdb.item.Item` + :return: An Item mapping type containing the requested attribute name/values + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name, + 'ItemName' : item_name} + if attribute_names: + if not isinstance(attribute_names, list): + attribute_names = [attribute_names] + self.build_list_params(params, attribute_names, 'AttributeName') + response = self.make_request('GetAttributes', params) + body = response.read() + if response.status == 200: + if item == None: + item = self.item_cls(domain, item_name) + h = handler.XmlHandler(item, self) + xml.sax.parseString(body, h) + return item + else: + raise SDBResponseError(response.status, response.reason, body) + + def delete_attributes(self, domain_or_name, item_name, attr_names=None): + """ + Delete attributes from a given item in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type item_name: string + :param item_name: The name of the item whose attributes are being deleted. + + :type attributes: dict, list or :class:`boto.sdb.item.Item` + :param attributes: Either a list containing attribute names which will cause + all values associated with that attribute name to be deleted or + a dict or Item containing the attribute names and keys and list + of values to delete as the value. If no value is supplied, + all attribute name/values for the item will be deleted. + + :rtype: bool + :return: True if successful + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name, + 'ItemName' : item_name} + if attr_names: + if isinstance(attr_names, list): + self.build_name_list(params, attr_names) + elif isinstance(attr_names, dict) or isinstance(attr_names, self.item_cls): + self.build_name_value_list(params, attr_names) + return self.get_status('DeleteAttributes', params) + + def query(self, domain_or_name, query='', max_items=None, next_token=None): + """ + Returns a list of item names within domain_name that match the query. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type query: string + :param query: The SimpleDB query to be performed. + + :type max_items: int + :param max_items: The maximum number of items to return. If not + supplied, the default is None which returns all + items matching the query. + + :rtype: ResultSet + :return: An iterator containing the results. + """ + warnings.warn('Query interface is deprecated', DeprecationWarning) + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name, + 'QueryExpression' : query} + if max_items: + params['MaxNumberOfItems'] = max_items + if next_token: + params['NextToken'] = next_token + return self.get_object('Query', params, ResultSet) + + def query_with_attributes(self, domain_or_name, query='', attr_names=None, + max_items=None, next_token=None): + """ + Returns a set of Attributes for item names within domain_name that match the query. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type query: string + :param query: The SimpleDB query to be performed. + + :type attr_names: list + :param attr_names: The name of the attributes to be returned. + If no attributes are specified, all attributes + will be returned. + + :type max_items: int + :param max_items: The maximum number of items to return. If not + supplied, the default is None which returns all + items matching the query. + + :rtype: ResultSet + :return: An iterator containing the results. + """ + warnings.warn('Query interface is deprecated', DeprecationWarning) + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name, + 'QueryExpression' : query} + if max_items: + params['MaxNumberOfItems'] = max_items + if next_token: + params['NextToken'] = next_token + if attr_names: + self.build_list_params(params, attr_names, 'AttributeName') + return self.get_list('QueryWithAttributes', params, [('Item', self.item_cls)], parent=domain) + + def select(self, domain_or_name, query='', next_token=None): + """ + Returns a set of Attributes for item names within domain_name that match the query. + The query must be expressed in using the SELECT style syntax rather than the + original SimpleDB query language. + Even though the select request does not require a domain object, a domain + object must be passed into this method so the Item objects returned can + point to the appropriate domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type query: string + :param query: The SimpleDB query to be performed. + + :rtype: ResultSet + :return: An iterator containing the results. + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'SelectExpression' : query} + if next_token: + params['NextToken'] = next_token + return self.get_list('Select', params, [('Item', self.item_cls)], parent=domain) + + def threaded_query(self, domain_or_name, query='', max_items=None, next_token=None, num_threads=6): + """ + Returns a list of fully populated items that match the query provided. + + The name/value pairs for all of the matching item names are retrieved in a number of separate + threads (specified by num_threads) to achieve maximum throughput. + The ResultSet that is returned has an attribute called next_token that can be used + to retrieve additional results for the same query. + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + if max_items and num_threads > max_items: + num_threads = max_items + rs = self.query(domain_or_name, query, max_items, next_token) + threads = [] + n = len(rs) / num_threads + for i in range(0, num_threads): + if i+1 == num_threads: + thread = ItemThread('Thread-%d' % i, domain_name, rs[n*i:]) + else: + thread = ItemThread('Thread-%d' % i, domain_name, rs[n*i:n*(i+1)]) + threads.append(thread) + thread.start() + del rs[0:] + for thread in threads: + thread.join() + for item in thread.items: + rs.append(item) + return rs + diff --git a/api/boto/sdb/db/__init__.py b/api/boto/sdb/db/__init__.py new file mode 100644 index 0000000..86044ed --- /dev/null +++ b/api/boto/sdb/db/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + diff --git a/api/boto/sdb/db/blob.py b/api/boto/sdb/db/blob.py new file mode 100644 index 0000000..d92eb65 --- /dev/null +++ b/api/boto/sdb/db/blob.py @@ -0,0 +1,64 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +class Blob(object): + """Blob object""" + def __init__(self, value=None, file=None, id=None): + self._file = file + self.id = id + self.value = value + + @property + def file(self): + from StringIO import StringIO + if self._file: + f = self._file + else: + f = StringIO(self.value) + return f + + def __str__(self): + if hasattr(self.file, "get_contents_as_string"): + return str(self.file.get_contents_as_string()) + else: + return str(self.file.getvalue()) + + def read(self): + return self.file.read() + + def readline(self): + return self.file.readline() + + def next(self): + return sefl.file.next() + + def __iter__(self): + return iter(self.file) + + @property + def size(self): + if self._file: + return self._file.size + elif self.value: + return len(self.value) + else: + return 0 diff --git a/api/boto/sdb/db/key.py b/api/boto/sdb/db/key.py new file mode 100644 index 0000000..42a9d8d --- /dev/null +++ b/api/boto/sdb/db/key.py @@ -0,0 +1,59 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Key(object): + + @classmethod + def from_path(cls, *args, **kwds): + raise NotImplementedError, "Paths are not currently supported" + + def __init__(self, encoded=None, obj=None): + self.name = None + if obj: + self.id = obj.id + self.kind = obj.kind() + else: + self.id = None + self.kind = None + + def app(self): + raise NotImplementedError, "Applications are not currently supported" + + def kind(self): + return self.kind + + def id(self): + return self.id + + def name(self): + raise NotImplementedError, "Key Names are not currently supported" + + def id_or_name(self): + return self.id + + def has_id_or_name(self): + return self.id != None + + def parent(self): + raise NotImplementedError, "Key parents are not currently supported" + + def __str__(self): + return self.id_or_name() diff --git a/api/boto/sdb/db/manager/__init__.py b/api/boto/sdb/db/manager/__init__.py new file mode 100644 index 0000000..1d75549 --- /dev/null +++ b/api/boto/sdb/db/manager/__init__.py @@ -0,0 +1,86 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import boto + +def get_manager(cls): + """ + Returns the appropriate Manager class for a given Model class. It does this by + looking in the boto config for a section like this:: + + [DB] + db_type = SimpleDB + db_user = + db_passwd = + db_name = my_domain + [DB_TestBasic] + db_type = SimpleDB + db_user = + db_passwd = + db_name = basic_domain + db_port = 1111 + + The values in the DB section are "generic values" that will be used if nothing more + specific is found. You can also create a section for a specific Model class that + gives the db info for that class. In the example above, TestBasic is a Model subclass. + """ + db_user = boto.config.get('DB', 'db_user', None) + db_passwd = boto.config.get('DB', 'db_passwd', None) + db_type = boto.config.get('DB', 'db_type', 'SimpleDB') + db_name = boto.config.get('DB', 'db_name', None) + db_table = boto.config.get('DB', 'db_table', None) + db_host = boto.config.get('DB', 'db_host', "sdb.amazonaws.com") + db_port = boto.config.getint('DB', 'db_port', 443) + enable_ssl = boto.config.getbool('DB', 'enable_ssl', True) + sql_dir = boto.config.get('DB', 'sql_dir', None) + debug = boto.config.getint('DB', 'debug', 0) + # first see if there is a fully qualified section name in the Boto config file + module_name = cls.__module__.replace('.', '_') + db_section = 'DB_' + module_name + '_' + cls.__name__ + if not boto.config.has_section(db_section): + db_section = 'DB_' + cls.__name__ + if boto.config.has_section(db_section): + db_user = boto.config.get(db_section, 'db_user', db_user) + db_passwd = boto.config.get(db_section, 'db_passwd', db_passwd) + db_type = boto.config.get(db_section, 'db_type', db_type) + db_name = boto.config.get(db_section, 'db_name', db_name) + db_table = boto.config.get(db_section, 'db_table', db_table) + db_host = boto.config.get(db_section, 'db_host', db_host) + db_port = boto.config.getint(db_section, 'db_port', db_port) + enable_ssl = boto.config.getint(db_section, 'enable_ssl', enable_ssl) + debug = boto.config.getint(db_section, 'debug', debug) + if db_type == 'SimpleDB': + from sdbmanager import SDBManager + return SDBManager(cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl) + elif db_type == 'PostgreSQL': + from pgmanager import PGManager + if db_table: + return PGManager(cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl) + else: + return None + elif db_type == 'XML': + from xmlmanager import XMLManager + return XMLManager(cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl) + else: + raise ValueError, 'Unknown db_type: %s' % db_type + diff --git a/api/boto/sdb/db/manager/pgmanager.py b/api/boto/sdb/db/manager/pgmanager.py new file mode 100644 index 0000000..4c7e3ad --- /dev/null +++ b/api/boto/sdb/db/manager/pgmanager.py @@ -0,0 +1,387 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +from boto.sdb.db.key import Key +from boto.sdb.db.model import Model +import psycopg2 +import psycopg2.extensions +import uuid, sys, os, string +from boto.exception import * + +psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) + +class PGConverter: + + def __init__(self, manager): + self.manager = manager + self.type_map = {Key : (self.encode_reference, self.decode_reference), + Model : (self.encode_reference, self.decode_reference)} + + def encode(self, type, value): + if type in self.type_map: + encode = self.type_map[type][0] + return encode(value) + return value + + def decode(self, type, value): + if type in self.type_map: + decode = self.type_map[type][1] + return decode(value) + return value + + def encode_prop(self, prop, value): + if isinstance(value, list): + if hasattr(prop, 'item_type'): + s = "{" + new_value = [] + for v in value: + item_type = getattr(prop, 'item_type') + if Model in item_type.mro(): + item_type = Model + new_value.append('%s' % self.encode(item_type, v)) + s += ','.join(new_value) + s += "}" + return s + else: + return value + return self.encode(prop.data_type, value) + + def decode_prop(self, prop, value): + if prop.data_type == list: + if value != None: + if not isinstance(value, list): + value = [value] + if hasattr(prop, 'item_type'): + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + if item_type != self.manager.cls: + return item_type._manager.decode_value(prop, value) + else: + item_type = Model + return [self.decode(item_type, v) for v in value] + return value + elif hasattr(prop, 'reference_class'): + ref_class = getattr(prop, 'reference_class') + if ref_class != self.manager.cls: + return ref_class._manager.decode_value(prop, value) + else: + return self.decode(prop.data_type, value) + elif hasattr(prop, 'calculated_type'): + calc_type = getattr(prop, 'calculated_type') + return self.decode(calc_type, value) + else: + return self.decode(prop.data_type, value) + + def encode_reference(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return value.id + + def decode_reference(self, value): + if not value: + return None + try: + return self.manager.get_object_from_id(value) + except: + raise ValueError, 'Unable to convert %s to Object' % value + +class PGManager(object): + + def __init__(self, cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl): + self.cls = cls + self.db_name = db_name + self.db_user = db_user + self.db_passwd = db_passwd + self.db_host = db_host + self.db_port = db_port + self.db_table = db_table + self.sql_dir = sql_dir + self.in_transaction = False + self.converter = PGConverter(self) + self._connect() + + def _build_connect_string(self): + cs = 'dbname=%s user=%s password=%s host=%s port=%d' + return cs % (self.db_name, self.db_user, self.db_passwd, + self.db_host, self.db_port) + + def _connect(self): + self.connection = psycopg2.connect(self._build_connect_string()) + self.connection.set_client_encoding('UTF8') + self.cursor = self.connection.cursor() + + def _object_lister(self, cursor): + try: + for row in cursor: + yield self._object_from_row(row, cursor.description) + except StopIteration: + cursor.close() + raise StopIteration + + def _dict_from_row(self, row, description): + d = {} + for i in range(0, len(row)): + d[description[i][0]] = row[i] + return d + + def _object_from_row(self, row, description=None): + if not description: + description = self.cursor.description + d = self._dict_from_row(row, description) + obj = self.cls(d['id']) + obj._manager = self + obj._auto_update = False + for prop in obj.properties(hidden=False): + if prop.data_type != Key: + v = self.decode_value(prop, d[prop.name]) + v = prop.make_value_from_datastore(v) + if hasattr(prop, 'calculated_type'): + prop._set_direct(obj, v) + elif not prop.empty(v): + setattr(obj, prop.name, v) + else: + setattr(obj, prop.name, prop.default_value()) + return obj + + def _build_insert_qs(self, obj, calculated): + fields = [] + values = [] + templs = [] + id_calculated = [p for p in calculated if p.name == 'id'] + for prop in obj.properties(hidden=False): + if prop not in calculated: + value = prop.get_value_for_datastore(obj) + if value != prop.default_value() or prop.required: + value = self.encode_value(prop, value) + values.append(value) + fields.append('"%s"' % prop.name) + templs.append('%s') + qs = 'INSERT INTO "%s" (' % self.db_table + if len(id_calculated) == 0: + qs += '"id",' + qs += ','.join(fields) + qs += ") VALUES (" + if len(id_calculated) == 0: + qs += "'%s'," % obj.id + qs += ','.join(templs) + qs += ')' + if calculated: + qs += ' RETURNING ' + calc_values = ['"%s"' % p.name for p in calculated] + qs += ','.join(calc_values) + qs += ';' + return qs, values + + def _build_update_qs(self, obj, calculated): + fields = [] + values = [] + for prop in obj.properties(hidden=False): + if prop not in calculated: + value = prop.get_value_for_datastore(obj) + if value != prop.default_value() or prop.required: + value = self.encode_value(prop, value) + values.append(value) + field = '"%s"=' % prop.name + field += '%s' + fields.append(field) + qs = 'UPDATE "%s" SET ' % self.db_table + qs += ','.join(fields) + qs += """ WHERE "id" = '%s'""" % obj.id + if calculated: + qs += ' RETURNING ' + calc_values = ['"%s"' % p.name for p in calculated] + qs += ','.join(calc_values) + qs += ';' + return qs, values + + def _get_sql(self, mapping=None): + print '_get_sql' + sql = None + if self.sql_dir: + path = os.path.join(self.sql_dir, self.cls.__name__ + '.sql') + print path + if os.path.isfile(path): + fp = open(path) + sql = fp.read() + fp.close() + t = string.Template(sql) + sql = t.safe_substitute(mapping) + return sql + + def start_transaction(self): + print 'start_transaction' + self.in_transaction = True + + def end_transaction(self): + print 'end_transaction' + self.in_transaction = False + self.commit() + + def commit(self): + if not self.in_transaction: + print '!!commit on %s' % self.db_table + try: + self.connection.commit() + + except psycopg2.ProgrammingError, err: + self.connection.rollback() + raise err + + def rollback(self): + print '!!rollback on %s' % self.db_table + self.connection.rollback() + + def delete_table(self): + self.cursor.execute('DROP TABLE "%s";' % self.db_table) + self.commit() + + def create_table(self, mapping=None): + self.cursor.execute(self._get_sql(mapping)) + self.commit() + + def encode_value(self, prop, value): + return self.converter.encode_prop(prop, value) + + def decode_value(self, prop, value): + return self.converter.decode_prop(prop, value) + + def execute_sql(self, query): + self.cursor.execute(query, None) + self.commit() + + def query_sql(self, query, vars=None): + self.cursor.execute(query, vars) + return self.cursor.fetchall() + + def lookup(self, cls, name, value): + values = [] + qs = 'SELECT * FROM "%s" WHERE ' % self.db_table + found = False + for property in cls.properties(hidden=False): + if property.name == name: + found = True + value = self.encode_value(property, value) + values.append(value) + qs += "%s=" % name + qs += "%s" + if not found: + raise SDBPersistenceError('%s is not a valid field' % key) + qs += ';' + print qs + self.cursor.execute(qs, values) + if self.cursor.rowcount == 1: + row = self.cursor.fetchone() + return self._object_from_row(row, self.cursor.description) + elif self.cursor.rowcount == 0: + raise KeyError, 'Object not found' + else: + raise LookupError, 'Multiple Objects Found' + + def query(self, cls, filters, limit=None, order_by=None): + parts = [] + qs = 'SELECT * FROM "%s"' % self.db_table + if filters: + qs += ' WHERE ' + properties = cls.properties(hidden=False) + for filter, value in filters: + name, op = filter.strip().split() + found = False + for property in properties: + if property.name == name: + found = True + value = self.encode_value(property, value) + parts.append(""""%s"%s'%s'""" % (name, op, value)) + if not found: + raise SDBPersistenceError('%s is not a valid field' % key) + qs += ','.join(parts) + qs += ';' + print qs + cursor = self.connection.cursor() + cursor.execute(qs) + return self._object_lister(cursor) + + def get_property(self, prop, obj, name): + qs = """SELECT "%s" FROM "%s" WHERE id='%s';""" % (name, self.db_table, obj.id) + print qs + self.cursor.execute(qs, None) + if self.cursor.rowcount == 1: + rs = self.cursor.fetchone() + for prop in obj.properties(hidden=False): + if prop.name == name: + v = self.decode_value(prop, rs[0]) + return v + raise AttributeError, '%s not found' % name + + def set_property(self, prop, obj, name, value): + pass + value = self.encode_value(prop, value) + qs = 'UPDATE "%s" SET ' % self.db_table + qs += "%s='%s'" % (name, self.encode_value(prop, value)) + qs += " WHERE id='%s'" % obj.id + qs += ';' + print qs + self.cursor.execute(qs) + self.commit() + + def get_object(self, cls, id): + qs = """SELECT * FROM "%s" WHERE id='%s';""" % (self.db_table, id) + self.cursor.execute(qs, None) + if self.cursor.rowcount == 1: + row = self.cursor.fetchone() + return self._object_from_row(row, self.cursor.description) + else: + raise SDBPersistenceError('%s object with id=%s does not exist' % (cls.__name__, id)) + + def get_object_from_id(self, id): + return self.get_object(self.cls, id) + + def _find_calculated_props(self, obj): + return [p for p in obj.properties() if hasattr(p, 'calculated_type')] + + def save_object(self, obj): + obj._auto_update = False + calculated = self._find_calculated_props(obj) + if not obj.id: + obj.id = str(uuid.uuid4()) + qs, values = self._build_insert_qs(obj, calculated) + else: + qs, values = self._build_update_qs(obj, calculated) + print qs + self.cursor.execute(qs, values) + if calculated: + calc_values = self.cursor.fetchone() + print calculated + print calc_values + for i in range(0, len(calculated)): + prop = calculated[i] + prop._set_direct(obj, calc_values[i]) + self.commit() + + def delete_object(self, obj): + qs = """DELETE FROM "%s" WHERE id='%s';""" % (self.db_table, obj.id) + print qs + self.cursor.execute(qs) + self.commit() + + diff --git a/api/boto/sdb/db/manager/sdbmanager.py b/api/boto/sdb/db/manager/sdbmanager.py new file mode 100644 index 0000000..2bb2440 --- /dev/null +++ b/api/boto/sdb/db/manager/sdbmanager.py @@ -0,0 +1,518 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import boto +import re +from boto.utils import find_class +import uuid +from boto.sdb.db.key import Key +from boto.sdb.db.model import Model +from boto.sdb.db.blob import Blob +from boto.sdb.db.property import ListProperty, MapProperty +from datetime import datetime +from boto.exception import SDBPersistenceError +from tempfile import TemporaryFile + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +class SDBConverter: + """ + Responsible for converting base Python types to format compatible with underlying + database. For SimpleDB, that means everything needs to be converted to a string + when stored in SimpleDB and from a string when retrieved. + + To convert a value, pass it to the encode or decode method. The encode method + will take a Python native value and convert to DB format. The decode method will + take a DB format value and convert it to Python native format. To find the appropriate + method to call, the generic encode/decode methods will look for the type-specific + method by searching for a method called "encode_" or "decode_". + """ + def __init__(self, manager): + self.manager = manager + self.type_map = { bool : (self.encode_bool, self.decode_bool), + int : (self.encode_int, self.decode_int), + long : (self.encode_long, self.decode_long), + float : (self.encode_float, self.decode_float), + Model : (self.encode_reference, self.decode_reference), + Key : (self.encode_reference, self.decode_reference), + datetime : (self.encode_datetime, self.decode_datetime), + Blob: (self.encode_blob, self.decode_blob), + } + + def encode(self, item_type, value): + if item_type in self.type_map: + encode = self.type_map[item_type][0] + return encode(value) + return value + + def decode(self, item_type, value): + if item_type in self.type_map: + decode = self.type_map[item_type][1] + return decode(value) + return value + + def encode_list(self, prop, value): + if not isinstance(value, list): + value = [value] + new_value = [] + for v in value: + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + item_type = Model + new_value.append(self.encode(item_type, v)) + return new_value + + def encode_map(self, prop, value): + if not isinstance(value, dict): + raise ValueError, 'Expected a dict value, got %s' % type(value) + new_value = [] + for key in value: + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + item_type = Model + encoded_value = self.encode(item_type, value[key]) + new_value.append('%s:%s' % (key, encoded_value)) + return new_value + + def encode_prop(self, prop, value): + if isinstance(prop, ListProperty): + return self.encode_list(prop, value) + elif isinstance(prop, MapProperty): + return self.encode_map(prop, value) + else: + return self.encode(prop.data_type, value) + + def decode_list(self, prop, value): + if not isinstance(value, list): + value = [value] + if hasattr(prop, 'item_type'): + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + return [item_type(id=v) for v in value] + return [self.decode(item_type, v) for v in value] + else: + return value + + def decode_map(self, prop, value): + if not isinstance(value, list): + value = [value] + ret_value = {} + item_type = getattr(prop, "item_type") + for keyval in value: + key, val = keyval.split(':', 1) + if Model in item_type.mro(): + val = item_type(id=val) + else: + val = self.decode(item_type, val) + ret_value[key] = val + return ret_value + + def decode_prop(self, prop, value): + if isinstance(prop, ListProperty): + return self.decode_list(prop, value) + elif isinstance(prop, MapProperty): + return self.decode_map(prop, value) + else: + return self.decode(prop.data_type, value) + + def encode_int(self, value): + value = int(value) + value += 2147483648 + return '%010d' % value + + def decode_int(self, value): + value = int(value) + value -= 2147483648 + return int(value) + + def encode_long(self, value): + value = long(value) + value += 9223372036854775808 + return '%020d' % value + + def decode_long(self, value): + value = long(value) + value -= 9223372036854775808 + return value + + def encode_bool(self, value): + if value == True: + return 'true' + else: + return 'false' + + def decode_bool(self, value): + if value.lower() == 'true': + return True + else: + return False + + def encode_float(self, value): + """ + See http://tools.ietf.org/html/draft-wood-ldapext-float-00. + """ + s = '%e' % value + l = s.split('e') + mantissa = l[0].ljust(18, '0') + exponent = l[1] + if value == 0.0: + case = '3' + exponent = '000' + elif mantissa[0] != '-' and exponent[0] == '+': + case = '5' + exponent = exponent[1:].rjust(3, '0') + elif mantissa[0] != '-' and exponent[0] == '-': + case = '4' + exponent = 999 + int(exponent) + exponent = '%03d' % exponent + elif mantissa[0] == '-' and exponent[0] == '-': + case = '2' + mantissa = '%f' % (10 + float(mantissa)) + mantissa = mantissa.ljust(18, '0') + exponent = exponent[1:].rjust(3, '0') + else: + case = '1' + mantissa = '%f' % (10 + float(mantissa)) + mantissa = mantissa.ljust(18, '0') + exponent = 999 - int(exponent) + exponent = '%03d' % exponent + return '%s %s %s' % (case, exponent, mantissa) + + def decode_float(self, value): + case = value[0] + exponent = value[2:5] + mantissa = value[6:] + if case == '3': + return 0.0 + elif case == '5': + pass + elif case == '4': + exponent = '%03d' % (int(exponent) - 999) + elif case == '2': + mantissa = '%f' % (float(mantissa) - 10) + exponent = '-' + exponent + else: + mantissa = '%f' % (float(mantissa) - 10) + exponent = '%03d' % abs((int(exponent) - 999)) + return float(mantissa + 'e' + exponent) + + def encode_datetime(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + return value.strftime(ISO8601) + + def decode_datetime(self, value): + try: + return datetime.strptime(value, ISO8601) + except: + return None + + def encode_reference(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return value.id + + def decode_reference(self, value): + if not value: + return None + return value + + def encode_blob(self, value): + if not value: + return None + + if not value.id: + bucket = self.manager.get_blob_bucket() + key = bucket.new_key(str(uuid.uuid4())) + value.id = "s3://%s/%s" % (key.bucket.name, key.name) + else: + match = re.match("^s3:\/\/([^\/]*)\/(.*)$", value.id) + if match: + s3 = self.manager.get_s3_connection() + bucket = s3.get_bucket(match.group(1), validate=False) + key = bucket.get_key(match.group(2)) + else: + raise SDBPersistenceError("Invalid Blob ID: %s" % value.id) + + if value.value != None: + key.set_contents_from_string(value.value) + return value.id + + + def decode_blob(self, value): + if not value: + return None + match = re.match("^s3:\/\/([^\/]*)\/(.*)$", value) + if match: + s3 = self.manager.get_s3_connection() + bucket = s3.get_bucket(match.group(1), validate=False) + key = bucket.get_key(match.group(2)) + else: + return None + if key: + return Blob(file=key, id="s3://%s/%s" % (key.bucket.name, key.name)) + else: + return None + +class SDBManager(object): + + def __init__(self, cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, ddl_dir, enable_ssl): + self.cls = cls + self.db_name = db_name + self.db_user = db_user + self.db_passwd = db_passwd + self.db_host = db_host + self.db_port = db_port + self.db_table = db_table + self.ddl_dir = ddl_dir + self.enable_ssl = enable_ssl + self.s3 = None + self.bucket = None + self.converter = SDBConverter(self) + self._connect() + + def _connect(self): + self.sdb = boto.connect_sdb(aws_access_key_id=self.db_user, + aws_secret_access_key=self.db_passwd, + is_secure=self.enable_ssl) + # This assumes that the domain has already been created + # It's much more efficient to do it this way rather than + # having this make a roundtrip each time to validate. + # The downside is that if the domain doesn't exist, it breaks + self.domain = self.sdb.lookup(self.db_name, validate=False) + if not self.domain: + self.domain = self.sdb.create_domain(self.db_name) + + def _object_lister(self, cls, query_lister): + for item in query_lister: + obj = self.get_object(cls, item.name, item) + if obj: + yield obj + + def encode_value(self, prop, value): + if value == None: + return None + if not prop: + return str(value) + return self.converter.encode_prop(prop, value) + + def decode_value(self, prop, value): + return self.converter.decode_prop(prop, value) + + def get_s3_connection(self): + if not self.s3: + self.s3 = boto.connect_s3(self.db_user, self.db_passwd) + return self.s3 + + def get_blob_bucket(self, bucket_name=None): + s3 = self.get_s3_connection() + bucket_name = "%s-%s" % (s3.aws_access_key_id, self.domain.name) + bucket_name = bucket_name.lower() + try: + self.bucket = s3.get_bucket(bucket_name) + except: + self.bucket = s3.create_bucket(bucket_name) + return self.bucket + + def load_object(self, obj): + if not obj._loaded: + a = self.domain.get_attributes(obj.id) + if a.has_key('__type__'): + for prop in obj.properties(hidden=False): + if a.has_key(prop.name): + value = self.decode_value(prop, a[prop.name]) + value = prop.make_value_from_datastore(value) + setattr(obj, prop.name, value) + obj._loaded = True + + def get_object(self, cls, id, a=None): + obj = None + if not a: + a = self.domain.get_attributes(id) + if a.has_key('__type__'): + if not cls or a['__type__'] != cls.__name__: + cls = find_class(a['__module__'], a['__type__']) + if cls: + params = {} + for prop in cls.properties(hidden=False): + if a.has_key(prop.name): + value = self.decode_value(prop, a[prop.name]) + value = prop.make_value_from_datastore(value) + params[prop.name] = value + obj = cls(id, **params) + obj._loaded = True + else: + s = '(%s) class %s.%s not found' % (id, a['__module__'], a['__type__']) + boto.log.info('sdbmanager: %s' % s) + return obj + + def get_object_from_id(self, id): + return self.get_object(None, id) + + def query(self, query): + query_str = "select * from `%s` %s" % (self.domain.name, self._build_filter_part(query.model_class, query.filters, query.sort_by)) + if query.limit: + query_str += " limit %s" % query.limit + rs = self.domain.select(query_str, max_items=query.limit, next_token = query.next_token) + query.rs = rs + return self._object_lister(query.model_class, rs) + + def count(self, cls, filters): + """ + Get the number of results that would + be returned in this query + """ + query = "select count(*) from `%s` %s" % (self.domain.name, self._build_filter_part(cls, filters)) + count = int(self.domain.select(query).next()["Count"]) + return count + + def _build_filter_part(self, cls, filters, order_by=None): + """ + Build the filter part + """ + import types + query_parts = [] + order_by_filtered = False + if order_by: + if order_by[0] == "-": + order_by_method = "desc"; + order_by = order_by[1:] + else: + order_by_method = "asc"; + + for filter in filters: + (name, op) = filter[0].strip().split(" ", 1) + value = filter[1] + property = cls.find_property(name) + if name == order_by: + order_by_filtered = True + if types.TypeType(value) == types.ListType: + filter_parts = [] + for val in value: + val = self.encode_value(property, val) + if isinstance(val, list): + for v in val: + filter_parts.append("`%s` %s '%s'" % (name, op, v.replace("'", "''"))) + else: + filter_parts.append("`%s` %s '%s'" % (name, op, val.replace("'", "''"))) + query_parts.append("(%s)" % (" or ".join(filter_parts))) + else: + if op == 'is' and value == None: + query_parts.append("`%s` is null" % name) + elif op == 'is not' and value == None: + query_parts.append("`%s` is not null" % name) + else: + val = self.encode_value(property, value) + if isinstance(val, list): + for v in val: + query_parts.append("`%s` %s '%s'" % (name, op, v.replace("'", "''"))) + else: + query_parts.append("`%s` %s '%s'" % (name, op, val.replace("'", "''"))) + + type_query = "(`__type__` = '%s'" % cls.__name__ + for subclass in cls.__sub_classes__: + type_query += " or `__type__` = '%s'" % subclass.__name__ + type_query +=")" + query_parts.append(type_query) + + order_by_query = "" + if order_by: + if not order_by_filtered: + query_parts.append("`%s` like '%%'" % order_by) + order_by_query = " order by `%s` %s" % (order_by, order_by_method) + + if len(query_parts) > 0: + return "where %s %s" % (" and ".join(query_parts), order_by_query) + else: + return "" + + + def query_gql(self, query_string, *args, **kwds): + raise NotImplementedError, "GQL queries not supported in SimpleDB" + + def save_object(self, obj): + if not obj.id: + obj.id = str(uuid.uuid4()) + + attrs = {'__type__' : obj.__class__.__name__, + '__module__' : obj.__class__.__module__, + '__lineage__' : obj.get_lineage()} + for property in obj.properties(hidden=False): + value = property.get_value_for_datastore(obj) + if value is not None: + value = self.encode_value(property, value) + attrs[property.name] = value + if property.unique: + try: + args = {property.name: value} + obj2 = obj.find(**args).next() + if obj2.id != obj.id: + raise SDBPersistenceError("Error: %s must be unique!" % property.name) + except(StopIteration): + pass + self.domain.put_attributes(obj.id, attrs, replace=True) + + def delete_object(self, obj): + self.domain.delete_attributes(obj.id) + + def set_property(self, prop, obj, name, value): + value = prop.get_value_for_datastore(obj) + value = self.encode_value(prop, value) + if prop.unique: + try: + args = {prop.name: value} + obj2 = obj.find(**args).next() + if obj2.id != obj.id: + raise SDBPersistenceError("Error: %s must be unique!" % prop.name) + except(StopIteration): + pass + self.domain.put_attributes(obj.id, {name : value}, replace=True) + + def get_property(self, prop, obj, name): + a = self.domain.get_attributes(obj.id) + + # try to get the attribute value from SDB + if name in a: + value = self.decode_value(prop, a[name]) + value = prop.make_value_from_datastore(value) + setattr(obj, prop.name, value) + return value + raise AttributeError, '%s not found' % name + + def set_key_value(self, obj, name, value): + self.domain.put_attributes(obj.id, {name : value}, replace=True) + + def delete_key_value(self, obj, name): + self.domain.delete_attributes(obj.id, name) + + def get_key_value(self, obj, name): + a = self.domain.get_attributes(obj.id, name) + if a.has_key(name): + return a[name] + else: + return None + + def get_raw_item(self, obj): + return self.domain.get_item(obj.id) + diff --git a/api/boto/sdb/db/manager/xmlmanager.py b/api/boto/sdb/db/manager/xmlmanager.py new file mode 100644 index 0000000..b12f5df --- /dev/null +++ b/api/boto/sdb/db/manager/xmlmanager.py @@ -0,0 +1,519 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import boto +from boto.utils import find_class, Password +import uuid +from boto.sdb.db.key import Key +from boto.sdb.db.model import Model +from datetime import datetime +from boto.exception import SDBPersistenceError +from xml.dom.minidom import getDOMImplementation, parse, parseString, Node + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +class XMLConverter: + """ + Responsible for converting base Python types to format compatible with underlying + database. For SimpleDB, that means everything needs to be converted to a string + when stored in SimpleDB and from a string when retrieved. + + To convert a value, pass it to the encode or decode method. The encode method + will take a Python native value and convert to DB format. The decode method will + take a DB format value and convert it to Python native format. To find the appropriate + method to call, the generic encode/decode methods will look for the type-specific + method by searching for a method called "encode_" or "decode_". + """ + def __init__(self, manager): + self.manager = manager + self.type_map = { bool : (self.encode_bool, self.decode_bool), + int : (self.encode_int, self.decode_int), + long : (self.encode_long, self.decode_long), + Model : (self.encode_reference, self.decode_reference), + Key : (self.encode_reference, self.decode_reference), + Password : (self.encode_password, self.decode_password), + datetime : (self.encode_datetime, self.decode_datetime)} + + def get_text_value(self, parent_node): + value = '' + for node in parent_node.childNodes: + if node.nodeType == node.TEXT_NODE: + value += node.data + return value + + def encode(self, item_type, value): + if item_type in self.type_map: + encode = self.type_map[item_type][0] + return encode(value) + return value + + def decode(self, item_type, value): + if item_type in self.type_map: + decode = self.type_map[item_type][1] + return decode(value) + else: + value = self.get_text_value(value) + return value + + def encode_prop(self, prop, value): + if isinstance(value, list): + if hasattr(prop, 'item_type'): + new_value = [] + for v in value: + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + item_type = Model + new_value.append(self.encode(item_type, v)) + return new_value + else: + return value + else: + return self.encode(prop.data_type, value) + + def decode_prop(self, prop, value): + if prop.data_type == list: + if hasattr(prop, 'item_type'): + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + item_type = Model + values = [] + for item_node in value.getElementsByTagName('item'): + value = self.decode(item_type, item_node) + values.append(value) + return values + else: + return self.get_text_value(value) + else: + return self.decode(prop.data_type, value) + + def encode_int(self, value): + value = int(value) + return '%d' % value + + def decode_int(self, value): + value = self.get_text_value(value) + if value: + value = int(value) + else: + value = None + return value + + def encode_long(self, value): + value = long(value) + return '%d' % value + + def decode_long(self, value): + value = self.get_text_value(value) + return long(value) + + def encode_bool(self, value): + if value == True: + return 'true' + else: + return 'false' + + def decode_bool(self, value): + value = self.get_text_value(value) + if value.lower() == 'true': + return True + else: + return False + + def encode_datetime(self, value): + return value.strftime(ISO8601) + + def decode_datetime(self, value): + value = self.get_text_value(value) + try: + return datetime.strptime(value, ISO8601) + except: + return None + + def encode_reference(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + val_node = self.manager.doc.createElement("object") + val_node.setAttribute('id', value.id) + val_node.setAttribute('class', '%s.%s' % (value.__class__.__module__, value.__class__.__name__)) + return val_node + + def decode_reference(self, value): + if not value: + return None + try: + value = value.childNodes[0] + class_name = value.getAttribute("class") + id = value.getAttribute("id") + cls = find_class(class_name) + return cls.get_by_ids(id) + except: + return None + + def encode_password(self, value): + if value and len(value) > 0: + return str(value) + else: + return None + + def decode_password(self, value): + value = self.get_text_value(value) + return Password(value) + + +class XMLManager(object): + + def __init__(self, cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, ddl_dir, enable_ssl): + self.cls = cls + if not db_name: + db_name = cls.__name__.lower() + self.db_name = db_name + self.db_user = db_user + self.db_passwd = db_passwd + self.db_host = db_host + self.db_port = db_port + self.db_table = db_table + self.ddl_dir = ddl_dir + self.s3 = None + self.converter = XMLConverter(self) + self.impl = getDOMImplementation() + self.doc = self.impl.createDocument(None, 'objects', None) + + self.connection = None + self.enable_ssl = enable_ssl + self.auth_header = None + if self.db_user: + import base64 + base64string = base64.encodestring('%s:%s' % (self.db_user, self.db_passwd))[:-1] + authheader = "Basic %s" % base64string + self.auth_header = authheader + + def _connect(self): + if self.db_host: + if self.enable_ssl: + from httplib import HTTPSConnection as Connection + else: + from httplib import HTTPConnection as Connection + + self.connection = Connection(self.db_host, self.db_port) + + def _make_request(self, method, url, post_data=None, body=None): + """ + Make a request on this connection + """ + if not self.connection: + self._connect() + try: + self.connection.close() + except: + pass + self.connection.connect() + headers = {} + if self.auth_header: + headers["Authorization"] = self.auth_header + self.connection.request(method, url, body, headers) + resp = self.connection.getresponse() + return resp + + def new_doc(self): + return self.impl.createDocument(None, 'objects', None) + + def _object_lister(self, cls, doc): + for obj_node in doc.getElementsByTagName('object'): + if not cls: + class_name = obj_node.getAttribute('class') + cls = find_class(class_name) + id = obj_node.getAttribute('id') + obj = cls(id) + for prop_node in obj_node.getElementsByTagName('property'): + prop_name = prop_node.getAttribute('name') + prop = obj.find_property(prop_name) + if prop: + if hasattr(prop, 'item_type'): + value = self.get_list(prop_node, prop.item_type) + else: + value = self.decode_value(prop, prop_node) + value = prop.make_value_from_datastore(value) + setattr(obj, prop.name, value) + yield obj + + def reset(self): + self._connect() + + def get_doc(self): + return self.doc + + def encode_value(self, prop, value): + return self.converter.encode_prop(prop, value) + + def decode_value(self, prop, value): + return self.converter.decode_prop(prop, value) + + def get_s3_connection(self): + if not self.s3: + self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key) + return self.s3 + + def get_list(self, prop_node, item_type): + values = [] + try: + items_node = prop_node.getElementsByTagName('items')[0] + except: + return [] + for item_node in items_node.getElementsByTagName('item'): + value = self.converter.decode(item_type, item_node) + values.append(value) + return values + + def get_object_from_doc(self, cls, id, doc): + obj_node = doc.getElementsByTagName('object')[0] + if not cls: + class_name = obj_node.getAttribute('class') + cls = find_class(class_name) + if not id: + id = obj_node.getAttribute('id') + obj = cls(id) + for prop_node in obj_node.getElementsByTagName('property'): + prop_name = prop_node.getAttribute('name') + prop = obj.find_property(prop_name) + value = self.decode_value(prop, prop_node) + value = prop.make_value_from_datastore(value) + if value != None: + try: + setattr(obj, prop.name, value) + except: + pass + return obj + + def get_props_from_doc(self, cls, id, doc): + """ + Pull out the properties from this document + Returns the class, the properties in a hash, and the id if provided as a tuple + :return: (cls, props, id) + """ + obj_node = doc.getElementsByTagName('object')[0] + if not cls: + class_name = obj_node.getAttribute('class') + cls = find_class(class_name) + if not id: + id = obj_node.getAttribute('id') + props = {} + for prop_node in obj_node.getElementsByTagName('property'): + prop_name = prop_node.getAttribute('name') + prop = cls.find_property(prop_name) + value = self.decode_value(prop, prop_node) + value = prop.make_value_from_datastore(value) + if value != None: + props[prop.name] = value + return (cls, props, id) + + + def get_object(self, cls, id): + if not self.connection: + self._connect() + + if not self.connection: + raise NotImplementedError("Can't query without a database connection") + url = "/%s/%s" % (self.db_name, id) + resp = self._make_request('GET', url) + if resp.status == 200: + doc = parse(resp) + else: + raise Exception("Error: %s" % resp.status) + return self.get_object_from_doc(cls, id, doc) + + def query(self, cls, filters, limit=None, order_by=None): + if not self.connection: + self._connect() + + if not self.connection: + raise NotImplementedError("Can't query without a database connection") + + from urllib import urlencode + + query = str(self._build_query(cls, filters, limit, order_by)) + if query: + url = "/%s?%s" % (self.db_name, urlencode({"query": query})) + else: + url = "/%s" % self.db_name + resp = self._make_request('GET', url) + if resp.status == 200: + doc = parse(resp) + else: + raise Exception("Error: %s" % resp.status) + return self._object_lister(cls, doc) + + def _build_query(self, cls, filters, limit, order_by): + import types + if len(filters) > 4: + raise Exception('Too many filters, max is 4') + parts = [] + properties = cls.properties(hidden=False) + for filter, value in filters: + name, op = filter.strip().split() + found = False + for property in properties: + if property.name == name: + found = True + if types.TypeType(value) == types.ListType: + filter_parts = [] + for val in value: + val = self.encode_value(property, val) + filter_parts.append("'%s' %s '%s'" % (name, op, val)) + parts.append("[%s]" % " OR ".join(filter_parts)) + else: + value = self.encode_value(property, value) + parts.append("['%s' %s '%s']" % (name, op, value)) + if not found: + raise Exception('%s is not a valid field' % name) + if order_by: + if order_by.startswith("-"): + key = order_by[1:] + type = "desc" + else: + key = order_by + type = "asc" + parts.append("['%s' starts-with ''] sort '%s' %s" % (key, key, type)) + return ' intersection '.join(parts) + + def query_gql(self, query_string, *args, **kwds): + raise NotImplementedError, "GQL queries not supported in XML" + + def save_list(self, doc, items, prop_node): + items_node = doc.createElement('items') + prop_node.appendChild(items_node) + for item in items: + item_node = doc.createElement('item') + items_node.appendChild(item_node) + if isinstance(item, Node): + item_node.appendChild(item) + else: + text_node = doc.createTextNode(item) + item_node.appendChild(text_node) + + def save_object(self, obj): + """ + Marshal the object and do a PUT + """ + doc = self.marshal_object(obj) + if obj.id: + url = "/%s/%s" % (self.db_name, obj.id) + else: + url = "/%s" % (self.db_name) + resp = self._make_request("PUT", url, body=doc.toxml()) + new_obj = self.get_object_from_doc(obj.__class__, None, parse(resp)) + obj.id = new_obj.id + for prop in obj.properties(): + try: + propname = prop.name + except AttributeError: + propname = None + if propname: + value = getattr(new_obj, prop.name) + if value: + setattr(obj, prop.name, value) + return obj + + + def marshal_object(self, obj, doc=None): + if not doc: + doc = self.new_doc() + if not doc: + doc = self.doc + obj_node = doc.createElement('object') + + if obj.id: + obj_node.setAttribute('id', obj.id) + + obj_node.setAttribute('class', '%s.%s' % (obj.__class__.__module__, + obj.__class__.__name__)) + root = doc.documentElement + root.appendChild(obj_node) + for property in obj.properties(hidden=False): + prop_node = doc.createElement('property') + prop_node.setAttribute('name', property.name) + prop_node.setAttribute('type', property.type_name) + value = property.get_value_for_datastore(obj) + if value is not None: + value = self.encode_value(property, value) + if isinstance(value, list): + self.save_list(doc, value, prop_node) + elif isinstance(value, Node): + prop_node.appendChild(value) + else: + text_node = doc.createTextNode(str(value)) + prop_node.appendChild(text_node) + obj_node.appendChild(prop_node) + + return doc + + def unmarshal_object(self, fp, cls=None, id=None): + if isinstance(fp, str) or isinstance(fp, unicode): + doc = parseString(fp) + else: + doc = parse(fp) + return self.get_object_from_doc(cls, id, doc) + + def unmarshal_props(self, fp, cls=None, id=None): + """ + Same as unmarshalling an object, except it returns + from "get_props_from_doc" + """ + if isinstance(fp, str) or isinstance(fp, unicode): + doc = parseString(fp) + else: + doc = parse(fp) + return self.get_props_from_doc(cls, id, doc) + + def delete_object(self, obj): + url = "/%s/%s" % (self.db_name, obj.id) + return self._make_request("DELETE", url) + + def set_key_value(self, obj, name, value): + self.domain.put_attributes(obj.id, {name : value}, replace=True) + + def delete_key_value(self, obj, name): + self.domain.delete_attributes(obj.id, name) + + def get_key_value(self, obj, name): + a = self.domain.get_attributes(obj.id, name) + if a.has_key(name): + return a[name] + else: + return None + + def get_raw_item(self, obj): + return self.domain.get_item(obj.id) + + def set_property(self, prop, obj, name, value): + pass + + def get_property(self, prop, obj, name): + pass + + def load_object(self, obj): + if not obj._loaded: + obj = obj.get_by_id(obj.id) + obj._loaded = True + return obj + diff --git a/api/boto/sdb/db/model.py b/api/boto/sdb/db/model.py new file mode 100644 index 0000000..dc142e8 --- /dev/null +++ b/api/boto/sdb/db/model.py @@ -0,0 +1,232 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.sdb.db.manager import get_manager +from boto.sdb.db.property import * +from boto.sdb.db.key import Key +from boto.sdb.db.query import Query +import boto + +class ModelMeta(type): + "Metaclass for all Models" + + def __init__(cls, name, bases, dict): + super(ModelMeta, cls).__init__(name, bases, dict) + # Make sure this is a subclass of Model - mainly copied from django ModelBase (thanks!) + cls.__sub_classes__ = [] + try: + if filter(lambda b: issubclass(b, Model), bases): + for base in bases: + base.__sub_classes__.append(cls) + cls._manager = get_manager(cls) + # look for all of the Properties and set their names + for key in dict.keys(): + if isinstance(dict[key], Property): + property = dict[key] + property.__property_config__(cls, key) + prop_names = [] + props = cls.properties() + for prop in props: + if not prop.__class__.__name__.startswith('_'): + prop_names.append(prop.name) + setattr(cls, '_prop_names', prop_names) + except NameError: + # 'Model' isn't defined yet, meaning we're looking at our own + # Model class, defined below. + pass + +class Model(object): + __metaclass__ = ModelMeta + + @classmethod + def get_lineage(cls): + l = [c.__name__ for c in cls.mro()] + l.reverse() + return '.'.join(l) + + @classmethod + def kind(cls): + return cls.__name__ + + @classmethod + def _get_by_id(cls, id, manager=None): + if not manager: + manager = cls._manager + return manager.get_object(cls, id) + + @classmethod + def get_by_id(cls, ids=None, parent=None): + if isinstance(ids, list): + objs = [cls._get_by_id(id) for id in ids] + return objs + else: + return cls._get_by_id(ids) + + get_by_ids = get_by_id + + @classmethod + def get_by_key_name(cls, key_names, parent=None): + raise NotImplementedError, "Key Names are not currently supported" + + @classmethod + def find(cls, limit=None, next_token=None, **params): + q = Query(cls, limit=limit, next_token=next_token) + for key, value in params.items(): + q.filter('%s =' % key, value) + return q + + @classmethod + def lookup(cls, name, value): + return cls._manager.lookup(cls, name, value) + + @classmethod + def all(cls, limit=None, next_token=None): + return cls.find(limit=limit, next_token=next_token) + + @classmethod + def get_or_insert(key_name, **kw): + raise NotImplementedError, "get_or_insert not currently supported" + + @classmethod + def properties(cls, hidden=True): + properties = [] + while cls: + for key in cls.__dict__.keys(): + prop = cls.__dict__[key] + if isinstance(prop, Property): + if hidden or not prop.__class__.__name__.startswith('_'): + properties.append(prop) + if len(cls.__bases__) > 0: + cls = cls.__bases__[0] + else: + cls = None + return properties + + @classmethod + def find_property(cls, prop_name): + property = None + while cls: + for key in cls.__dict__.keys(): + prop = cls.__dict__[key] + if isinstance(prop, Property): + if not prop.__class__.__name__.startswith('_') and prop_name == prop.name: + property = prop + if len(cls.__bases__) > 0: + cls = cls.__bases__[0] + else: + cls = None + return property + + @classmethod + def get_xmlmanager(cls): + if not hasattr(cls, '_xmlmanager'): + from boto.sdb.db.manager.xmlmanager import XMLManager + cls._xmlmanager = XMLManager(cls, None, None, None, + None, None, None, None, False) + return cls._xmlmanager + + @classmethod + def from_xml(cls, fp): + xmlmanager = cls.get_xmlmanager() + return xmlmanager.unmarshal_object(fp) + + def __init__(self, id=None, **kw): + self._loaded = False + # first initialize all properties to their default values + for prop in self.properties(hidden=False): + setattr(self, prop.name, prop.default_value()) + if kw.has_key('manager'): + self._manager = kw['manager'] + self.id = id + for key in kw: + if key != 'manager': + # We don't want any errors populating up when loading an object, + # so if it fails we just revert to it's default value + try: + setattr(self, key, kw[key]) + except Exception, e: + boto.log.exception(e) + + def __repr__(self): + return '%s<%s>' % (self.__class__.__name__, self.id) + + def __str__(self): + return str(self.id) + + def __eq__(self, other): + return other and isinstance(other, Model) and self.id == other.id + + def _get_raw_item(self): + return self._manager.get_raw_item(self) + + def load(self): + if self.id and not self._loaded: + self._manager.load_object(self) + + def put(self): + self._manager.save_object(self) + + save = put + + def delete(self): + self._manager.delete_object(self) + + def key(self): + return Key(obj=self) + + def set_manager(self, manager): + self._manager = manager + + def to_dict(self): + props = {} + for prop in self.properties(hidden=False): + props[prop.name] = getattr(self, prop.name) + obj = {'properties' : props, + 'id' : self.id} + return {self.__class__.__name__ : obj} + + def to_xml(self, doc=None): + xmlmanager = self.get_xmlmanager() + doc = xmlmanager.marshal_object(self, doc) + return doc + +class Expando(Model): + + def __setattr__(self, name, value): + if name in self._prop_names: + object.__setattr__(self, name, value) + elif name.startswith('_'): + object.__setattr__(self, name, value) + elif name == 'id': + object.__setattr__(self, name, value) + else: + self._manager.set_key_value(self, name, value) + object.__setattr__(self, name, value) + + def __getattr__(self, name): + if not name.startswith('_'): + value = self._manager.get_key_value(self, name) + if value: + object.__setattr__(self, name, value) + return value + raise AttributeError + + diff --git a/api/boto/sdb/db/property.py b/api/boto/sdb/db/property.py new file mode 100644 index 0000000..61d424a --- /dev/null +++ b/api/boto/sdb/db/property.py @@ -0,0 +1,556 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import datetime +from key import Key +from boto.utils import Password +from boto.sdb.db.query import Query +from tempfile import TemporaryFile + +import re +import boto +import boto.s3.key +from boto.sdb.db.blob import Blob + +class Property(object): + + data_type = str + type_name = '' + name = '' + verbose_name = '' + + def __init__(self, verbose_name=None, name=None, default=None, required=False, + validator=None, choices=None, unique=False): + self.verbose_name = verbose_name + self.name = name + self.default = default + self.required = required + self.validator = validator + self.choices = choices + self.slot_name = '_' + self.unique = unique + + def __get__(self, obj, objtype): + if obj: + obj.load() + return getattr(obj, self.slot_name) + else: + return None + + def __set__(self, obj, value): + self.validate(value) + + # Fire off any on_set functions + try: + if obj._loaded and hasattr(obj, "on_set_%s" % self.name): + fnc = getattr(obj, "on_set_%s" % self.name) + value = fnc(value) + except Exception, e: + boto.log.exception("Exception running on_set_%s" % self.name) + + setattr(obj, self.slot_name, value) + + def __property_config__(self, model_class, property_name): + self.model_class = model_class + self.name = property_name + self.slot_name = '_' + self.name + + def default_validator(self, value): + if value == self.default_value(): + return + if not isinstance(value, self.data_type): + raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + + def default_value(self): + return self.default + + def validate(self, value): + if self.required and value==None: + raise ValueError, '%s is a required property' % self.name + if self.choices and value and not value in self.choices: + raise ValueError, '%s not a valid choice for %s.%s' % (value, self.model_class.__name__, self.name) + if self.validator: + self.validator(value) + else: + self.default_validator(value) + return value + + def empty(self, value): + return not value + + def get_value_for_datastore(self, model_instance): + return getattr(model_instance, self.name) + + def make_value_from_datastore(self, value): + return value + + def get_choices(self): + if callable(self.choices): + return self.choices() + return self.choices + +def validate_string(value): + if isinstance(value, str) or isinstance(value, unicode): + if len(value) > 1024: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting String, got %s' % type(value) + +class StringProperty(Property): + + type_name = 'String' + + def __init__(self, verbose_name=None, name=None, default='', required=False, + validator=validate_string, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + +class TextProperty(Property): + + type_name = 'Text' + + def __init__(self, verbose_name=None, name=None, default='', required=False, + validator=None, choices=None, unique=False, max_length=None): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.max_length = max_length + + def validate(self, value): + if not isinstance(value, str) and not isinstance(value, unicode): + raise TypeError, 'Expecting Text, got %s' % type(value) + if self.max_length and len(value) > self.max_length: + raise ValueError, 'Length of value greater than maxlength %s' % self.max_length + +class PasswordProperty(StringProperty): + """ + Hashed property who's original value can not be + retrieved, but still can be compaired. + """ + data_type = Password + type_name = 'Password' + + def __init__(self, verbose_name=None, name=None, default='', required=False, + validator=None, choices=None, unique=False): + StringProperty.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def make_value_from_datastore(self, value): + p = Password(value) + return p + + def get_value_for_datastore(self, model_instance): + value = StringProperty.get_value_for_datastore(self, model_instance) + if value and len(value): + return str(value) + else: + return None + + def __set__(self, obj, value): + if not isinstance(value, Password): + p = Password() + p.set(value) + value = p + Property.__set__(self, obj, value) + + def __get__(self, obj, objtype): + return Password(StringProperty.__get__(self, obj, objtype)) + + def validate(self, value): + value = Property.validate(self, value) + if isinstance(value, Password): + if len(value) > 1024: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting Password, got %s' % type(value) + +class BlobProperty(Property): + data_type = Blob + type_name = "blob" + + def __set__(self, obj, value): + if value != self.default_value(): + if not isinstance(value, Blob): + oldb = self.__get__(obj, type(obj)) + id = None + if oldb: + id = oldb.id + b = Blob(value=value, id=id) + value = b + Property.__set__(self, obj, value) + +class S3KeyProperty(Property): + + data_type = boto.s3.key.Key + type_name = 'S3Key' + validate_regex = "^s3:\/\/([^\/]*)\/(.*)$" + + def __init__(self, verbose_name=None, name=None, default=None, + required=False, validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, + validator, choices, unique) + + def validate(self, value): + if value == self.default_value() or value == str(self.default_value()): + return self.default_value() + if isinstance(value, self.data_type): + return + match = re.match(self.validate_regex, value) + if match: + return + raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + + def __get__(self, obj, objtype): + value = Property.__get__(self, obj, objtype) + if value: + if isinstance(value, self.data_type): + return value + match = re.match(self.validate_regex, value) + if match: + s3 = obj._manager.get_s3_connection() + bucket = s3.get_bucket(match.group(1), validate=False) + k = bucket.get_key(match.group(2)) + if not k: + k = bucket.new_key(match.group(2)) + k.set_contents_from_string("") + return k + else: + return value + + def get_value_for_datastore(self, model_instance): + value = Property.get_value_for_datastore(self, model_instance) + if value: + return "s3://%s/%s" % (value.bucket.name, value.name) + else: + return None + +class IntegerProperty(Property): + + data_type = int + type_name = 'Integer' + + def __init__(self, verbose_name=None, name=None, default=0, required=False, + validator=None, choices=None, unique=False, max=2147483647, min=-2147483648): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.max = max + self.min = min + + def validate(self, value): + value = int(value) + value = Property.validate(self, value) + if value > self.max: + raise ValueError, 'Maximum value is %d' % self.max + if value < self.min: + raise ValueError, 'Minimum value is %d' % self.min + return value + + def empty(self, value): + return value is None + +class LongProperty(Property): + + data_type = long + type_name = 'Long' + + def __init__(self, verbose_name=None, name=None, default=0, required=False, + validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def validate(self, value): + value = long(value) + value = Property.validate(self, value) + min = -9223372036854775808 + max = 9223372036854775807 + if value > max: + raise ValueError, 'Maximum value is %d' % max + if value < min: + raise ValueError, 'Minimum value is %d' % min + return value + + def empty(self, value): + return value is None + +class BooleanProperty(Property): + + data_type = bool + type_name = 'Boolean' + + def __init__(self, verbose_name=None, name=None, default=False, required=False, + validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def empty(self, value): + return value is None + +class FloatProperty(Property): + + data_type = float + type_name = 'Float' + + def __init__(self, verbose_name=None, name=None, default=0.0, required=False, + validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def validate(self, value): + value = float(value) + value = Property.validate(self, value) + return value + + def empty(self, value): + return value is None + +class DateTimeProperty(Property): + + data_type = datetime.datetime + type_name = 'DateTime' + + def __init__(self, verbose_name=None, auto_now=False, auto_now_add=False, name=None, + default=None, required=False, validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.auto_now = auto_now + self.auto_now_add = auto_now_add + + def default_value(self): + if self.auto_now or self.auto_now_add: + return self.now() + return Property.default_value(self) + + def validate(self, value): + if value == None: + return + if not isinstance(value, self.data_type): + raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + + def get_value_for_datastore(self, model_instance): + if self.auto_now: + setattr(model_instance, self.name, self.now()) + return Property.get_value_for_datastore(self, model_instance) + + def now(self): + return datetime.datetime.utcnow() + +class ReferenceProperty(Property): + + data_type = Key + type_name = 'Reference' + + def __init__(self, reference_class=None, collection_name=None, + verbose_name=None, name=None, default=None, required=False, validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.reference_class = reference_class + self.collection_name = collection_name + + def __get__(self, obj, objtype): + if obj: + value = getattr(obj, self.slot_name) + if value == self.default_value(): + return value + # If the value is still the UUID for the referenced object, we need to create + # the object now that is the attribute has actually been accessed. This lazy + # instantiation saves unnecessary roundtrips to SimpleDB + if isinstance(value, str) or isinstance(value, unicode): + # This is some minor handling to allow us to use the base "Model" class + # as our reference class. If we do so, we're going to assume we're using + # our own class's manager to fetch objects + if hasattr(self.reference_class, "_manager"): + manager = self.reference_class._manager + else: + manager = obj._manager + value = manager.get_object(self.reference_class, value) + setattr(obj, self.name, value) + return value + + def __property_config__(self, model_class, property_name): + Property.__property_config__(self, model_class, property_name) + if self.collection_name is None: + self.collection_name = '%s_%s_set' % (model_class.__name__.lower(), self.name) + if hasattr(self.reference_class, self.collection_name): + raise ValueError, 'duplicate property: %s' % self.collection_name + setattr(self.reference_class, self.collection_name, + _ReverseReferenceProperty(model_class, property_name, self.collection_name)) + + def check_uuid(self, value): + # This does a bit of hand waving to "type check" the string + t = value.split('-') + if len(t) != 5: + raise ValueError + + def check_instance(self, value): + try: + obj_lineage = value.get_lineage() + cls_lineage = self.reference_class.get_lineage() + if obj_lineage.startswith(cls_lineage): + return + raise TypeError, '%s not instance of %s' % (obj_lineage, cls_lineage) + except: + raise ValueError, '%s is not a Model' % value + + def validate(self, value): + if self.required and value==None: + raise ValueError, '%s is a required property' % self.name + if value == self.default_value(): + return + if not isinstance(value, str) and not isinstance(value, unicode): + self.check_instance(value) + +class _ReverseReferenceProperty(Property): + data_type = Query + type_name = 'query' + + def __init__(self, model, prop, name): + self.__model = model + self.__property = prop + self.name = name + self.item_type = model + + def __get__(self, model_instance, model_class): + """Fetches collection of model instances of this collection property.""" + if model_instance is not None: + query = Query(self.__model) + return query.filter(self.__property + ' =', model_instance) + else: + return self + + def __set__(self, model_instance, value): + """Not possible to set a new collection.""" + raise ValueError, 'Virtual property is read-only' + + +class CalculatedProperty(Property): + + def __init__(self, verbose_name=None, name=None, default=None, + required=False, validator=None, choices=None, + calculated_type=int, unique=False, use_method=False): + Property.__init__(self, verbose_name, name, default, required, + validator, choices, unique) + self.calculated_type = calculated_type + self.use_method = use_method + + def __get__(self, obj, objtype): + value = self.default_value() + if obj: + try: + value = getattr(obj, self.slot_name) + if self.use_method: + value = value() + except AttributeError: + pass + return value + + def __set__(self, obj, value): + """Not possible to set a new AutoID.""" + pass + + def _set_direct(self, obj, value): + if not self.use_method: + setattr(obj, self.slot_name, value) + + def get_value_for_datastore(self, model_instance): + if self.calculated_type in [str, int, bool]: + value = self.__get__(model_instance, model_instance.__class__) + return value + else: + return None + +class ListProperty(Property): + + data_type = list + type_name = 'List' + + def __init__(self, item_type, verbose_name=None, name=None, default=None, **kwds): + if default is None: + default = [] + self.item_type = item_type + Property.__init__(self, verbose_name, name, default=default, required=True, **kwds) + + def validate(self, value): + if value is not None: + if not isinstance(value, list): + value = [value] + + if self.item_type in (int, long): + item_type = (int, long) + elif self.item_type in (str, unicode): + item_type = (str, unicode) + else: + item_type = self.item_type + + for item in value: + if not isinstance(item, item_type): + if item_type == (int, long): + raise ValueError, 'Items in the %s list must all be integers.' % self.name + else: + raise ValueError('Items in the %s list must all be %s instances' % + (self.name, self.item_type.__name__)) + return value + + def empty(self, value): + return value is None + + def default_value(self): + return list(super(ListProperty, self).default_value()) + + def __set__(self, obj, value): + """Override the set method to allow them to set the property to an instance of the item_type instead of requiring a list to be passed in""" + if self.item_type in (int, long): + item_type = (int, long) + elif self.item_type in (str, unicode): + item_type = (str, unicode) + else: + item_type = self.item_type + if isinstance(value, item_type): + value = [value] + return super(ListProperty, self).__set__(obj,value) + + +class MapProperty(Property): + + data_type = dict + type_name = 'Map' + + def __init__(self, item_type=str, verbose_name=None, name=None, default=None, **kwds): + if default is None: + default = {} + self.item_type = item_type + Property.__init__(self, verbose_name, name, default=default, required=True, **kwds) + + def validate(self, value): + if value is not None: + if not isinstance(value, dict): + raise ValueError, 'Value must of type dict' + + if self.item_type in (int, long): + item_type = (int, long) + elif self.item_type in (str, unicode): + item_type = (str, unicode) + else: + item_type = self.item_type + + for key in value: + if not isinstance(value[key], item_type): + if item_type == (int, long): + raise ValueError, 'Values in the %s Map must all be integers.' % self.name + else: + raise ValueError('Values in the %s Map must all be %s instances' % + (self.name, self.item_type.__name__)) + return value + + def empty(self, value): + return value is None + + def default_value(self): + return {} diff --git a/api/boto/sdb/db/query.py b/api/boto/sdb/db/query.py new file mode 100644 index 0000000..034d9d3 --- /dev/null +++ b/api/boto/sdb/db/query.py @@ -0,0 +1,76 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Query(object): + __local_iter__ = None + def __init__(self, model_class, limit=None, next_token=None, manager=None): + self.model_class = model_class + self.limit = limit + if manager: + self.manager = manager + else: + self.manager = self.model_class._manager + self.filters = [] + self.sort_by = None + self.rs = None + self.next_token = next_token + + def __iter__(self): + return iter(self.manager.query(self)) + + def next(self): + if self.__local_iter__ == None: + self.__local_iter__ = self.__iter__() + return self.__local_iter__.next() + + def filter(self, property_operator, value): + self.filters.append((property_operator, value)) + return self + + def fetch(self, limit, offset=0): + raise NotImplementedError, "fetch mode is not currently supported" + + def count(self): + return self.manager.count(self.model_class, self.filters) + + def order(self, key): + self.sort_by = key + return self + + def to_xml(self, doc=None): + if not doc: + xmlmanager = self.model_class.get_xmlmanager() + doc = xmlmanager.new_doc() + for obj in self: + obj.to_xml(doc) + return doc + + def get_next_token(self): + if self.rs: + return self.rs.next_token + if self._next_token: + return self._next_token + return None + + def set_next_token(self, token): + self._next_token = token + + next_token = property(get_next_token, set_next_token) diff --git a/api/boto/sdb/db/test_db.py b/api/boto/sdb/db/test_db.py new file mode 100644 index 0000000..b790b9e --- /dev/null +++ b/api/boto/sdb/db/test_db.py @@ -0,0 +1,224 @@ +from boto.sdb.db.model import Model +from boto.sdb.db.property import * +from boto.sdb.db.manager import get_manager +from datetime import datetime +import time +from boto.exception import SDBPersistenceError + +_objects = {} + +# +# This will eventually be moved to the boto.tests module and become a real unit test +# but for now it will live here. It shows examples of each of the Property types in +# use and tests the basic operations. +# +class TestBasic(Model): + + name = StringProperty() + size = IntegerProperty() + foo = BooleanProperty() + date = DateTimeProperty() + +class TestFloat(Model): + + name = StringProperty() + value = FloatProperty() + +class TestRequired(Model): + + req = StringProperty(required=True, default='foo') + +class TestReference(Model): + + ref = ReferenceProperty(reference_class=TestBasic, collection_name='refs') + +class TestSubClass(TestBasic): + + answer = IntegerProperty() + +class TestPassword(Model): + password = PasswordProperty() + +class TestList(Model): + + name = StringProperty() + nums = ListProperty(int) + +class TestMap(Model): + + name = StringProperty() + map = MapProperty() + +class TestListReference(Model): + + name = StringProperty() + basics = ListProperty(TestBasic) + +class TestAutoNow(Model): + + create_date = DateTimeProperty(auto_now_add=True) + modified_date = DateTimeProperty(auto_now=True) + +class TestUnique(Model): + name = StringProperty(unique=True) + +def test_basic(): + global _objects + t = TestBasic() + t.name = 'simple' + t.size = -42 + t.foo = True + t.date = datetime.now() + print 'saving object' + t.put() + _objects['test_basic_t'] = t + time.sleep(5) + print 'now try retrieving it' + tt = TestBasic.get_by_id(t.id) + _objects['test_basic_tt'] = tt + assert tt.id == t.id + l = TestBasic.get_by_id([t.id]) + assert len(l) == 1 + assert l[0].id == t.id + assert t.size == tt.size + assert t.foo == tt.foo + assert t.name == tt.name + #assert t.date == tt.date + return t + +def test_float(): + global _objects + t = TestFloat() + t.name = 'float object' + t.value = 98.6 + print 'saving object' + t.save() + _objects['test_float_t'] = t + time.sleep(5) + print 'now try retrieving it' + tt = TestFloat.get_by_id(t.id) + _objects['test_float_tt'] = tt + assert tt.id == t.id + assert tt.name == t.name + assert tt.value == t.value + return t + +def test_required(): + global _objects + t = TestRequired() + _objects['test_required_t'] = t + t.put() + return t + +def test_reference(t=None): + global _objects + if not t: + t = test_basic() + tt = TestReference() + tt.ref = t + tt.put() + time.sleep(10) + tt = TestReference.get_by_id(tt.id) + _objects['test_reference_tt'] = tt + assert tt.ref.id == t.id + for o in t.refs: + print o + +def test_subclass(): + global _objects + t = TestSubClass() + _objects['test_subclass_t'] = t + t.name = 'a subclass' + t.size = -489 + t.save() + +def test_password(): + global _objects + t = TestPassword() + _objects['test_password_t'] = t + t.password = "foo" + t.save() + time.sleep(5) + # Make sure it stored ok + tt = TestPassword.get_by_id(t.id) + _objects['test_password_tt'] = tt + #Testing password equality + assert tt.password == "foo" + #Testing password not stored as string + assert str(tt.password) != "foo" + +def test_list(): + global _objects + t = TestList() + _objects['test_list_t'] = t + t.name = 'a list of ints' + t.nums = [1,2,3,4,5] + t.put() + tt = TestList.get_by_id(t.id) + _objects['test_list_tt'] = tt + assert tt.name == t.name + for n in tt.nums: + assert isinstance(n, int) + +def test_list_reference(): + global _objects + t = TestBasic() + t.put() + _objects['test_list_ref_t'] = t + tt = TestListReference() + tt.name = "foo" + tt.basics = [t] + tt.put() + time.sleep(5) + _objects['test_list_ref_tt'] = tt + ttt = TestListReference.get_by_id(tt.id) + assert ttt.basics[0].id == t.id + +def test_unique(): + global _objects + t = TestUnique() + name = 'foo' + str(int(time.time())) + t.name = name + t.put() + _objects['test_unique_t'] = t + time.sleep(10) + tt = TestUnique() + _objects['test_unique_tt'] = tt + tt.name = name + try: + tt.put() + assert False + except(SDBPersistenceError): + pass + +def test_datetime(): + global _objects + t = TestAutoNow() + t.put() + _objects['test_datetime_t'] = t + time.sleep(5) + tt = TestAutoNow.get_by_id(t.id) + assert tt.create_date.timetuple() == t.create_date.timetuple() + +def test(): + print 'test_basic' + t1 = test_basic() + print 'test_required' + test_required() + print 'test_reference' + test_reference(t1) + print 'test_subclass' + test_subclass() + print 'test_password' + test_password() + print 'test_list' + test_list() + print 'test_list_reference' + test_list_reference() + print "test_datetime" + test_datetime() + print 'test_unique' + test_unique() + +if __name__ == "__main__": + test() diff --git a/api/boto/sdb/domain.py b/api/boto/sdb/domain.py new file mode 100644 index 0000000..3c0def6 --- /dev/null +++ b/api/boto/sdb/domain.py @@ -0,0 +1,330 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SDB Domain +""" +from boto.sdb.queryresultset import QueryResultSet, SelectResultSet + +class Domain: + + def __init__(self, connection=None, name=None): + self.connection = connection + self.name = name + self._metadata = None + + def __repr__(self): + return 'Domain:%s' % self.name + + def __iter__(self): + return iter(self.select("SELECT * FROM `%s`" % self.name)) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'DomainName': + self.name = value + else: + setattr(self, name, value) + + def get_metadata(self): + if not self._metadata: + self._metadata = self.connection.domain_metadata(self) + return self._metadata + + def put_attributes(self, item_name, attributes, replace=True): + """ + Store attributes for a given item. + + :type item_name: string + :param item_name: The name of the item whose attributes are being stored. + + :type attribute_names: dict or dict-like object + :param attribute_names: The name/value pairs to store as attributes + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + return self.connection.put_attributes(self, item_name, attributes, replace) + + def batch_put_attributes(self, items, replace=True): + """ + Store attributes for multiple items. + + :type items: dict or dict-like object + :param items: A dictionary-like object. The keys of the dictionary are + the item names and the values are themselves dictionaries + of attribute names/values, exactly the same as the + attribute_names parameter of the scalar put_attributes + call. + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + return self.connection.batch_put_attributes(self, items, replace) + + def get_attributes(self, item_name, attribute_name=None, item=None): + """ + Retrieve attributes for a given item. + + :type item_name: string + :param item_name: The name of the item whose attributes are being retrieved. + + :type attribute_names: string or list of strings + :param attribute_names: An attribute name or list of attribute names. This + parameter is optional. If not supplied, all attributes + will be retrieved for the item. + + :rtype: :class:`boto.sdb.item.Item` + :return: An Item mapping type containing the requested attribute name/values + """ + return self.connection.get_attributes(self, item_name, attribute_name, item) + + def delete_attributes(self, item_name, attributes=None): + """ + Delete attributes from a given item. + + :type item_name: string + :param item_name: The name of the item whose attributes are being deleted. + + :type attributes: dict, list or :class:`boto.sdb.item.Item` + :param attributes: Either a list containing attribute names which will cause + all values associated with that attribute name to be deleted or + a dict or Item containing the attribute names and keys and list + of values to delete as the value. If no value is supplied, + all attribute name/values for the item will be deleted. + + :rtype: bool + :return: True if successful + """ + return self.connection.delete_attributes(self, item_name, attributes) + + def query(self, query='', max_items=None, attr_names=None): + """ + Returns a list of items within domain that match the query. + + :type query: string + :param query: The SimpleDB query to be performed. + + :type max_items: int + :param max_items: The maximum number of items to return. If not + supplied, the default is None which returns all + items matching the query. + + :type attr_names: list + :param attr_names: Either None, meaning return all attributes + or a list of attribute names which means to return + only those attributes. + + :rtype: iter + :return: An iterator containing the results. This is actually a generator + function that will iterate across all search results, not just the + first page. + """ + return iter(QueryResultSet(self, query, max_items, attr_names)) + + def select(self, query='', next_token=None, max_items=None): + """ + Returns a set of Attributes for item names within domain_name that match the query. + The query must be expressed in using the SELECT style syntax rather than the + original SimpleDB query language. + + :type query: string + :param query: The SimpleDB query to be performed. + + :type max_items: int + :param max_items: The maximum number of items to return. + + :rtype: iter + :return: An iterator containing the results. This is actually a generator + function that will iterate across all search results, not just the + first page. + """ + return SelectResultSet(self, query, max_items=max_items, + next_token=next_token) + + def get_item(self, item_name): + item = self.get_attributes(item_name) + if item: + item.domain = self + return item + else: + return None + + def new_item(self, item_name): + return self.connection.item_cls(self, item_name) + + def delete_item(self, item): + self.delete_attributes(item.name) + + def to_xml(self, f=None): + """Get this domain as an XML DOM Document + :param f: Optional File to dump directly to + :type f: File or Stream + + :return: File object where the XML has been dumped to + :rtype: file + """ + if not f: + from tempfile import TemporaryFile + f = TemporaryFile() + print >>f, '' + print >>f, '' % self.name + for item in self: + print >>f, '\t' % item.name + for k in item: + print >>f, '\t\t' % k + values = item[k] + if not isinstance(values, list): + values = [values] + for value in values: + print >>f, '\t\t\t>f, ']]>' + print >>f, '\t\t' + print >>f, '\t' + print >>f, '' + f.flush() + f.seek(0) + return f + + + def from_xml(self, doc): + """Load this domain based on an XML document""" + import xml.sax + handler = DomainDumpParser(self) + xml.sax.parse(doc, handler) + return handler + + +class DomainMetaData: + + def __init__(self, domain=None): + self.domain = domain + self.item_count = None + self.item_names_size = None + self.attr_name_count = None + self.attr_names_size = None + self.attr_value_count = None + self.attr_values_size = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'ItemCount': + self.item_count = int(value) + elif name == 'ItemNamesSizeBytes': + self.item_names_size = int(value) + elif name == 'AttributeNameCount': + self.attr_name_count = int(value) + elif name == 'AttributeNamesSizeBytes': + self.attr_names_size = int(value) + elif name == 'AttributeValueCount': + self.attr_value_count = int(value) + elif name == 'AttributeValuesSizeBytes': + self.attr_values_size = int(value) + elif name == 'Timestamp': + self.timestamp = value + else: + setattr(self, name, value) + +import sys +from xml.sax.handler import ContentHandler +class DomainDumpParser(ContentHandler): + """ + SAX parser for a domain that has been dumped + """ + + def __init__(self, domain): + self.uploader = UploaderThread(domain.name) + self.item_id = None + self.attrs = {} + self.attribute = None + self.value = "" + self.domain = domain + + def startElement(self, name, attrs): + if name == "Item": + self.item_id = attrs['id'] + self.attrs = {} + elif name == "attribute": + self.attribute = attrs['id'] + elif name == "value": + self.value = "" + + def characters(self, ch): + self.value += ch + + def endElement(self, name): + if name == "value": + if self.value and self.attribute: + value = self.value.strip() + attr_name = self.attribute.strip() + if self.attrs.has_key(attr_name): + self.attrs[attr_name].append(value) + else: + self.attrs[attr_name] = [value] + elif name == "Item": + self.uploader.items[self.item_id] = self.attrs + # Every 40 items we spawn off the uploader + if len(self.uploader.items) >= 40: + self.uploader.start() + self.uploader = UploaderThread(self.domain.name) + elif name == "Domain": + # If we're done, spawn off our last Uploader Thread + self.uploader.start() + +from threading import Thread +class UploaderThread(Thread): + """Uploader Thread""" + + def __init__(self, domain_name): + import boto + self.sdb = boto.connect_sdb() + self.db = self.sdb.get_domain(domain_name) + self.items = {} + Thread.__init__(self) + + def run(self): + try: + self.db.batch_put_attributes(self.items) + except: + print "Exception using batch put, trying regular put instead" + for item_name in self.items: + self.db.put_attributes(item_name, self.items[item_name]) + print ".", + sys.stdout.flush() diff --git a/api/boto/sdb/item.py b/api/boto/sdb/item.py new file mode 100644 index 0000000..b81e715 --- /dev/null +++ b/api/boto/sdb/item.py @@ -0,0 +1,97 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SDB Item +""" + +import base64 + +class Item(dict): + + def __init__(self, domain, name='', active=False): + dict.__init__(self) + self.domain = domain + self.name = name + self.active = active + self.request_id = None + self.encoding = None + self.in_attribute = False + self.converter = self.domain.connection.converter + + def startElement(self, name, attrs, connection): + if name == 'Attribute': + self.in_attribute = True + self.encoding = attrs.get('encoding', None) + return None + + def decode_value(self, value): + if self.encoding == 'base64': + self.encoding = None + return base64.decodestring(value) + else: + return value + + def endElement(self, name, value, connection): + if name == 'ItemName': + self.name = self.decode_value(value) + elif name == 'Name': + if self.in_attribute: + self.last_key = self.decode_value(value) + else: + self.name = self.decode_value(value) + elif name == 'Value': + if self.has_key(self.last_key): + if not isinstance(self[self.last_key], list): + self[self.last_key] = [self[self.last_key]] + value = self.decode_value(value) + if self.converter: + value = self.converter.decode(value) + self[self.last_key].append(value) + else: + value = self.decode_value(value) + if self.converter: + value = self.converter.decode(value) + self[self.last_key] = value + elif name == 'BoxUsage': + try: + connection.box_usage += float(value) + except: + pass + elif name == 'RequestId': + self.request_id = value + elif name == 'Attribute': + self.in_attribute = False + else: + setattr(self, name, value) + + def load(self): + self.domain.get_attributes(self.name, item=self) + + def save(self, replace=True): + self.domain.put_attributes(self.name, self, replace) + + def delete(self): + self.domain.delete_item(self) + + + + diff --git a/api/boto/sdb/persist/__init__.py b/api/boto/sdb/persist/__init__.py new file mode 100644 index 0000000..2f2b0c1 --- /dev/null +++ b/api/boto/sdb/persist/__init__.py @@ -0,0 +1,83 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.utils import find_class + +class Manager(object): + + DefaultDomainName = boto.config.get('Persist', 'default_domain', None) + + def __init__(self, domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0): + self.domain_name = domain_name + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + self.domain = None + self.sdb = None + self.s3 = None + if not self.domain_name: + self.domain_name = self.DefaultDomainName + if self.domain_name: + boto.log.info('No SimpleDB domain set, using default_domain: %s' % self.domain_name) + else: + boto.log.warning('No SimpleDB domain set, persistance is disabled') + if self.domain_name: + self.sdb = boto.connect_sdb(aws_access_key_id=self.aws_access_key_id, + aws_secret_access_key=self.aws_secret_access_key, + debug=debug) + self.domain = self.sdb.lookup(self.domain_name) + if not self.domain: + self.domain = self.sdb.create_domain(self.domain_name) + + def get_s3_connection(self): + if not self.s3: + self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key) + return self.s3 + +def get_manager(domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0): + return Manager(domain_name, aws_access_key_id, aws_secret_access_key, debug=debug) + +def set_domain(domain_name): + Manager.DefaultDomainName = domain_name + +def get_domain(): + return Manager.DefaultDomainName + +def revive_object_from_id(id, manager): + if not manager.domain: + return None + attrs = manager.domain.get_attributes(id, ['__module__', '__type__', '__lineage__']) + try: + cls = find_class(attrs['__module__'], attrs['__type__']) + return cls(id, manager=manager) + except ImportError: + return None + +def object_lister(cls, query_lister, manager): + for item in query_lister: + if cls: + yield cls(item.name) + else: + o = revive_object_from_id(item.name, manager) + if o: + yield o + + diff --git a/api/boto/sdb/persist/checker.py b/api/boto/sdb/persist/checker.py new file mode 100644 index 0000000..147ea47 --- /dev/null +++ b/api/boto/sdb/persist/checker.py @@ -0,0 +1,303 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from datetime import datetime +import boto +from boto.s3.key import Key +from boto.s3.bucket import Bucket +from boto.sdb.persist import revive_object_from_id +from boto.exception import SDBPersistenceError +from boto.utils import Password + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +class ValueChecker: + + def check(self, value): + """ + Checks a value to see if it is of the right type. + + Should raise a TypeError exception if an in appropriate value is passed in. + """ + raise TypeError + + def from_string(self, str_value, obj): + """ + Takes a string as input and returns the type-specific value represented by that string. + + Should raise a ValueError if the value cannot be converted to the appropriate type. + """ + raise ValueError + + def to_string(self, value): + """ + Convert a value to it's string representation. + + Should raise a ValueError if the value cannot be converted to a string representation. + """ + raise ValueError + +class StringChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('maxlength'): + self.maxlength = params['maxlength'] + else: + self.maxlength = 1024 + if params.has_key('default'): + self.check(params['default']) + self.default = params['default'] + else: + self.default = '' + + def check(self, value): + if isinstance(value, str) or isinstance(value, unicode): + if len(value) > self.maxlength: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting String, got %s' % type(value) + + def from_string(self, str_value, obj): + return str_value + + def to_string(self, value): + self.check(value) + return value + +class PasswordChecker(StringChecker): + def check(self, value): + if isinstance(value, str) or isinstance(value, unicode) or isinstance(value, Password): + if len(value) > self.maxlength: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting String, got %s' % type(value) + +class IntegerChecker(ValueChecker): + + __sizes__ = { 'small' : (65535, 32767, -32768, 5), + 'medium' : (4294967295, 2147483647, -2147483648, 10), + 'large' : (18446744073709551615, 9223372036854775807, -9223372036854775808, 20)} + + def __init__(self, **params): + self.size = params.get('size', 'medium') + if self.size not in self.__sizes__.keys(): + raise ValueError, 'size must be one of %s' % self.__sizes__.keys() + self.signed = params.get('signed', True) + self.default = params.get('default', 0) + self.format_string = '%%0%dd' % self.__sizes__[self.size][-1] + + def check(self, value): + if not isinstance(value, int) and not isinstance(value, long): + raise TypeError, 'Expecting int or long, got %s' % type(value) + if self.signed: + min = self.__sizes__[self.size][2] + max = self.__sizes__[self.size][1] + else: + min = 0 + max = self.__sizes__[self.size][0] + if value > max: + raise ValueError, 'Maximum value is %d' % max + if value < min: + raise ValueError, 'Minimum value is %d' % min + + def from_string(self, str_value, obj): + val = int(str_value) + if self.signed: + val = val + self.__sizes__[self.size][2] + return val + + def to_string(self, value): + self.check(value) + if self.signed: + value += -self.__sizes__[self.size][2] + return self.format_string % value + +class BooleanChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('default'): + self.default = params['default'] + else: + self.default = False + + def check(self, value): + if not isinstance(value, bool): + raise TypeError, 'Expecting bool, got %s' % type(value) + + def from_string(self, str_value, obj): + if str_value.lower() == 'true': + return True + else: + return False + + def to_string(self, value): + self.check(value) + if value == True: + return 'true' + else: + return 'false' + +class DateTimeChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('maxlength'): + self.maxlength = params['maxlength'] + else: + self.maxlength = 1024 + if params.has_key('default'): + self.default = params['default'] + else: + self.default = datetime.now() + + def check(self, value): + if not isinstance(value, datetime): + raise TypeError, 'Expecting datetime, got %s' % type(value) + + def from_string(self, str_value, obj): + try: + return datetime.strptime(str_value, ISO8601) + except: + raise ValueError, 'Unable to convert %s to DateTime' % str_value + + def to_string(self, value): + self.check(value) + return value.strftime(ISO8601) + +class ObjectChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + self.ref_class = params.get('ref_class', None) + if self.ref_class == None: + raise SDBPersistenceError('ref_class parameter is required') + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + # ugly little hack - sometimes I want to just stick a UUID string + # in here rather than instantiate an object. + # This does a bit of hand waving to "type check" the string + t = value.split('-') + if len(t) != 5: + raise ValueError + else: + try: + obj_lineage = value.get_lineage() + cls_lineage = self.ref_class.get_lineage() + if obj_lineage.startswith(cls_lineage): + return + raise TypeError, '%s not instance of %s' % (obj_lineage, cls_lineage) + except: + raise ValueError, '%s is not an SDBObject' % value + + def from_string(self, str_value, obj): + if not str_value: + return None + try: + return revive_object_from_id(str_value, obj._manager) + except: + raise ValueError, 'Unable to convert %s to Object' % str_value + + def to_string(self, value): + self.check(value) + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return value.id + +class S3KeyChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + try: + bucket_name, key_name = value.split('/', 1) + except: + raise ValueError + elif not isinstance(value, Key): + raise TypeError, 'Expecting Key, got %s' % type(value) + + def from_string(self, str_value, obj): + if not str_value: + return None + if str_value == 'None': + return None + try: + bucket_name, key_name = str_value.split('/', 1) + if obj: + s3 = obj._manager.get_s3_connection() + bucket = s3.get_bucket(bucket_name) + key = bucket.get_key(key_name) + if not key: + key = bucket.new_key(key_name) + return key + except: + raise ValueError, 'Unable to convert %s to S3Key' % str_value + + def to_string(self, value): + self.check(value) + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return '%s/%s' % (value.bucket.name, value.name) + +class S3BucketChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + return + elif not isinstance(value, Bucket): + raise TypeError, 'Expecting Bucket, got %s' % type(value) + + def from_string(self, str_value, obj): + if not str_value: + return None + if str_value == 'None': + return None + try: + if obj: + s3 = obj._manager.get_s3_connection() + bucket = s3.get_bucket(str_value) + return bucket + except: + raise ValueError, 'Unable to convert %s to S3Bucket' % str_value + + def to_string(self, value): + self.check(value) + if value == None: + return '' + else: + return '%s' % value.name + diff --git a/api/boto/sdb/persist/object.py b/api/boto/sdb/persist/object.py new file mode 100644 index 0000000..3646d43 --- /dev/null +++ b/api/boto/sdb/persist/object.py @@ -0,0 +1,207 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager, object_lister +from boto.sdb.persist.property import * +import uuid + +class SDBBase(type): + "Metaclass for all SDBObjects" + def __init__(cls, name, bases, dict): + super(SDBBase, cls).__init__(name, bases, dict) + # Make sure this is a subclass of SDBObject - mainly copied from django ModelBase (thanks!) + try: + if filter(lambda b: issubclass(b, SDBObject), bases): + # look for all of the Properties and set their names + for key in dict.keys(): + if isinstance(dict[key], Property): + property = dict[key] + property.set_name(key) + prop_names = [] + props = cls.properties() + for prop in props: + prop_names.append(prop.name) + setattr(cls, '_prop_names', prop_names) + except NameError: + # 'SDBObject' isn't defined yet, meaning we're looking at our own + # SDBObject class, defined below. + pass + +class SDBObject(object): + __metaclass__ = SDBBase + + _manager = get_manager() + + @classmethod + def get_lineage(cls): + l = [c.__name__ for c in cls.mro()] + l.reverse() + return '.'.join(l) + + @classmethod + def get(cls, id=None, **params): + if params.has_key('manager'): + manager = params['manager'] + else: + manager = cls._manager + if manager.domain and id: + a = cls._manager.domain.get_attributes(id, '__type__') + if a.has_key('__type__'): + return cls(id, manager) + else: + raise SDBPersistenceError('%s object with id=%s does not exist' % (cls.__name__, id)) + else: + rs = cls.find(**params) + try: + obj = rs.next() + except StopIteration: + raise SDBPersistenceError('%s object matching query does not exist' % cls.__name__) + try: + rs.next() + except StopIteration: + return obj + raise SDBPersistenceError('Query matched more than 1 item') + + @classmethod + def find(cls, **params): + if params.has_key('manager'): + manager = params['manager'] + del params['manager'] + else: + manager = cls._manager + keys = params.keys() + if len(keys) > 4: + raise SDBPersistenceError('Too many fields, max is 4') + parts = ["['__type__'='%s'] union ['__lineage__'starts-with'%s']" % (cls.__name__, cls.get_lineage())] + properties = cls.properties() + for key in keys: + found = False + for property in properties: + if property.name == key: + found = True + if isinstance(property, ScalarProperty): + checker = property.checker + parts.append("['%s' = '%s']" % (key, checker.to_string(params[key]))) + else: + raise SDBPersistenceError('%s is not a searchable field' % key) + if not found: + raise SDBPersistenceError('%s is not a valid field' % key) + query = ' intersection '.join(parts) + if manager.domain: + rs = manager.domain.query(query) + else: + rs = [] + return object_lister(None, rs, manager) + + @classmethod + def list(cls, max_items=None, manager=None): + if not manager: + manager = cls._manager + if manager.domain: + rs = manager.domain.query("['__type__' = '%s']" % cls.__name__, max_items=max_items) + else: + rs = [] + return object_lister(cls, rs, manager) + + @classmethod + def properties(cls): + properties = [] + while cls: + for key in cls.__dict__.keys(): + if isinstance(cls.__dict__[key], Property): + properties.append(cls.__dict__[key]) + if len(cls.__bases__) > 0: + cls = cls.__bases__[0] + else: + cls = None + return properties + + # for backwards compatibility + find_properties = properties + + def __init__(self, id=None, manager=None): + if manager: + self._manager = manager + self.id = id + if self.id: + self._auto_update = True + if self._manager.domain: + attrs = self._manager.domain.get_attributes(self.id, '__type__') + if len(attrs.keys()) == 0: + raise SDBPersistenceError('Object %s: not found' % self.id) + else: + self.id = str(uuid.uuid4()) + self._auto_update = False + + def __setattr__(self, name, value): + if name in self._prop_names: + object.__setattr__(self, name, value) + elif name.startswith('_'): + object.__setattr__(self, name, value) + elif name == 'id': + object.__setattr__(self, name, value) + else: + self._persist_attribute(name, value) + object.__setattr__(self, name, value) + + def __getattr__(self, name): + if not name.startswith('_'): + a = self._manager.domain.get_attributes(self.id, name) + if a.has_key(name): + object.__setattr__(self, name, a[name]) + return a[name] + raise AttributeError + + def __repr__(self): + return '%s<%s>' % (self.__class__.__name__, self.id) + + def _persist_attribute(self, name, value): + if self.id: + self._manager.domain.put_attributes(self.id, {name : value}, replace=True) + + def _get_sdb_item(self): + return self._manager.domain.get_item(self.id) + + def save(self): + attrs = {'__type__' : self.__class__.__name__, + '__module__' : self.__class__.__module__, + '__lineage__' : self.get_lineage()} + for property in self.properties(): + attrs[property.name] = property.to_string(self) + if self._manager.domain: + self._manager.domain.put_attributes(self.id, attrs, replace=True) + self._auto_update = True + + def delete(self): + if self._manager.domain: + self._manager.domain.delete_attributes(self.id) + + def get_related_objects(self, ref_name, ref_cls=None): + if self._manager.domain: + query = "['%s' = '%s']" % (ref_name, self.id) + if ref_cls: + query += " intersection ['__type__'='%s']" % ref_cls.__name__ + rs = self._manager.domain.query(query) + else: + rs = [] + return object_lister(ref_cls, rs, self._manager) + diff --git a/api/boto/sdb/persist/property.py b/api/boto/sdb/persist/property.py new file mode 100644 index 0000000..6eea765 --- /dev/null +++ b/api/boto/sdb/persist/property.py @@ -0,0 +1,370 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import SDBPersistenceError +from boto.sdb.persist.checker import * +from boto.utils import Password + +class Property(object): + + def __init__(self, checker_class, **params): + self.name = '' + self.checker = checker_class(**params) + self.slot_name = '__' + + def set_name(self, name): + self.name = name + self.slot_name = '__' + self.name + +class ScalarProperty(Property): + + def save(self, obj): + domain = obj._manager.domain + domain.put_attributes(obj.id, {self.name : self.to_string(obj)}, replace=True) + + def to_string(self, obj): + return self.checker.to_string(getattr(obj, self.name)) + + def load(self, obj): + domain = obj._manager.domain + a = domain.get_attributes(obj.id, self.name) + # try to get the attribute value from SDB + if self.name in a: + value = self.checker.from_string(a[self.name], obj) + setattr(obj, self.slot_name, value) + # if it's not there, set the value to the default value + else: + self.__set__(obj, self.checker.default) + + def __get__(self, obj, objtype): + if obj: + try: + value = getattr(obj, self.slot_name) + except AttributeError: + if obj._auto_update: + self.load(obj) + value = getattr(obj, self.slot_name) + else: + value = self.checker.default + setattr(obj, self.slot_name, self.checker.default) + return value + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class StringProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, StringChecker, **params) + +class PasswordProperty(ScalarProperty): + """ + Hashed password + """ + + def __init__(self, **params): + ScalarProperty.__init__(self, PasswordChecker, **params) + + def __set__(self, obj, value): + p = Password() + p.set(value) + ScalarProperty.__set__(self, obj, p) + + def __get__(self, obj, objtype): + return Password(ScalarProperty.__get__(self, obj, objtype)) + +class SmallPositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class SmallIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class PositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class IntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class LargePositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class LargeIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class BooleanProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, BooleanChecker, **params) + +class DateTimeProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, DateTimeChecker, **params) + +class ObjectProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, ObjectChecker, **params) + +class S3KeyProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, S3KeyChecker, **params) + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + if isinstance(value, str): + value = self.checker.from_string(value, obj) + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class S3BucketProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, S3BucketChecker, **params) + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + if isinstance(value, str): + value = self.checker.from_string(value, obj) + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class MultiValueProperty(Property): + + def __init__(self, checker_class, **params): + Property.__init__(self, checker_class, **params) + + def __get__(self, obj, objtype): + if obj: + try: + value = getattr(obj, self.slot_name) + except AttributeError: + if obj._auto_update: + self.load(obj) + value = getattr(obj, self.slot_name) + else: + value = MultiValue(self, obj, []) + setattr(obj, self.slot_name, value) + return value + + def load(self, obj): + if obj != None: + _list = [] + domain = obj._manager.domain + a = domain.get_attributes(obj.id, self.name) + if self.name in a: + lst = a[self.name] + if not isinstance(lst, list): + lst = [lst] + for value in lst: + value = self.checker.from_string(value, obj) + _list.append(value) + setattr(obj, self.slot_name, MultiValue(self, obj, _list)) + + def __set__(self, obj, value): + if not isinstance(value, list): + raise SDBPersistenceError('Value must be a list') + setattr(obj, self.slot_name, MultiValue(self, obj, value)) + str_list = self.to_string(obj) + domain = obj._manager.domain + if obj._auto_update: + if len(str_list) == 1: + domain.put_attributes(obj.id, {self.name : str_list[0]}, replace=True) + else: + try: + self.__delete__(obj) + except: + pass + domain.put_attributes(obj.id, {self.name : str_list}, replace=True) + setattr(obj, self.slot_name, MultiValue(self, obj, value)) + + def __delete__(self, obj): + if obj._auto_update: + domain = obj._manager.domain + domain.delete_attributes(obj.id, [self.name]) + setattr(obj, self.slot_name, MultiValue(self, obj, [])) + + def to_string(self, obj): + str_list = [] + for value in self.__get__(obj, type(obj)): + str_list.append(self.checker.to_string(value)) + return str_list + +class StringListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, StringChecker, **params) + +class SmallIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class SmallPositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class IntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class PositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class LargeIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class LargePositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class BooleanListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, BooleanChecker, **params) + +class ObjectListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, ObjectChecker, **params) + +class HasManyProperty(Property): + + def set_name(self, name): + self.name = name + self.slot_name = '__' + self.name + + def __get__(self, obj, objtype): + return self + + +class MultiValue: + """ + Special Multi Value for boto persistence layer to allow us to do + obj.list.append(foo) + """ + def __init__(self, property, obj, _list): + self.checker = property.checker + self.name = property.name + self.object = obj + self._list = _list + + def __repr__(self): + return repr(self._list) + + def __getitem__(self, key): + return self._list.__getitem__(key) + + def __delitem__(self, key): + item = self[key] + self._list.__delitem__(key) + domain = self.object._manager.domain + domain.delete_attributes(self.object.id, {self.name: [self.checker.to_string(item)]}) + + def __len__(self): + return len(self._list) + + def append(self, value): + self.checker.check(value) + self._list.append(value) + domain = self.object._manager.domain + domain.put_attributes(self.object.id, {self.name: self.checker.to_string(value)}, replace=False) + + def index(self, value): + for x in self._list: + if x.id == value.id: + return self._list.index(x) + + def remove(self, value): + del(self[self.index(value)]) diff --git a/api/boto/sdb/persist/test_persist.py b/api/boto/sdb/persist/test_persist.py new file mode 100644 index 0000000..3207e58 --- /dev/null +++ b/api/boto/sdb/persist/test_persist.py @@ -0,0 +1,138 @@ +from boto.sdb.persist.object import SDBObject +from boto.sdb.persist.property import * +from boto.sdb.persist import Manager +from datetime import datetime +import time + +# +# This will eventually be moved to the boto.tests module and become a real unit test +# but for now it will live here. It shows examples of each of the Property types in +# use and tests the basic operations. +# +class TestScalar(SDBObject): + + name = StringProperty() + description = StringProperty() + size = PositiveIntegerProperty() + offset = IntegerProperty() + foo = BooleanProperty() + date = DateTimeProperty() + file = S3KeyProperty() + +class TestRef(SDBObject): + + name = StringProperty() + ref = ObjectProperty(ref_class=TestScalar) + +class TestSubClass1(TestRef): + + answer = PositiveIntegerProperty() + +class TestSubClass2(TestScalar): + + flag = BooleanProperty() + +class TestList(SDBObject): + + names = StringListProperty() + numbers = PositiveIntegerListProperty() + bools = BooleanListProperty() + objects = ObjectListProperty(ref_class=TestScalar) + +def test1(): + s = TestScalar() + s.name = 'foo' + s.description = 'This is foo' + s.size = 42 + s.offset = -100 + s.foo = True + s.date = datetime.now() + s.save() + return s + +def test2(ref_name): + s = TestRef() + s.name = 'testref' + rs = TestScalar.find(name=ref_name) + s.ref = rs.next() + s.save() + return s + +def test3(): + s = TestScalar() + s.name = 'bar' + s.description = 'This is bar' + s.size = 24 + s.foo = False + s.date = datetime.now() + s.save() + return s + +def test4(ref1, ref2): + s = TestList() + s.names.append(ref1.name) + s.names.append(ref2.name) + s.numbers.append(ref1.size) + s.numbers.append(ref2.size) + s.bools.append(ref1.foo) + s.bools.append(ref2.foo) + s.objects.append(ref1) + s.objects.append(ref2) + s.save() + return s + +def test5(ref): + s = TestSubClass1() + s.answer = 42 + s.ref = ref + s.save() + # test out free form attribute + s.fiddlefaddle = 'this is fiddlefaddle' + s._fiddlefaddle = 'this is not fiddlefaddle' + return s + +def test6(): + s = TestSubClass2() + s.name = 'fie' + s.description = 'This is fie' + s.size = 4200 + s.offset = -820 + s.foo = False + s.date = datetime.now() + s.flag = True + s.save() + return s + +def test(domain_name): + print 'Initialize the Persistance system' + Manager.DefaultDomainName = domain_name + print 'Call test1' + s1 = test1() + # now create a new instance and read the saved data from SDB + print 'Now sleep to wait for things to converge' + time.sleep(5) + print 'Now lookup the object and compare the fields' + s2 = TestScalar(s1.id) + assert s1.name == s2.name + assert s1.description == s2.description + assert s1.size == s2.size + assert s1.offset == s2.offset + assert s1.foo == s2.foo + #assert s1.date == s2.date + print 'Call test2' + s2 = test2(s1.name) + print 'Call test3' + s3 = test3() + print 'Call test4' + s4 = test4(s1, s3) + print 'Call test5' + s6 = test6() + s5 = test5(s6) + domain = s5._manager.domain + item1 = domain.get_item(s1.id) + item2 = domain.get_item(s2.id) + item3 = domain.get_item(s3.id) + item4 = domain.get_item(s4.id) + item5 = domain.get_item(s5.id) + item6 = domain.get_item(s6.id) + return [(s1, item1), (s2, item2), (s3, item3), (s4, item4), (s5, item5), (s6, item6)] diff --git a/api/boto/sdb/queryresultset.py b/api/boto/sdb/queryresultset.py new file mode 100644 index 0000000..a9430f4 --- /dev/null +++ b/api/boto/sdb/queryresultset.py @@ -0,0 +1,92 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.sdb.item import Item + +def query_lister(domain, query='', max_items=None, attr_names=None): + more_results = True + num_results = 0 + next_token = None + while more_results: + rs = domain.connection.query_with_attributes(domain, query, attr_names, + next_token=next_token) + for item in rs: + if max_items: + if num_results == max_items: + raise StopIteration + yield item + num_results += 1 + next_token = rs.next_token + more_results = next_token != None + +class QueryResultSet: + + def __init__(self, domain=None, query='', max_items=None, attr_names=None): + self.max_items = max_items + self.domain = domain + self.query = query + self.attr_names = attr_names + + def __iter__(self): + return query_lister(self.domain, self.query, self.max_items, self.attr_names) + +def select_lister(domain, query='', max_items=None): + more_results = True + num_results = 0 + next_token = None + while more_results: + rs = domain.connection.select(domain, query, next_token=next_token) + for item in rs: + if max_items: + if num_results == max_items: + raise StopIteration + yield item + num_results += 1 + next_token = rs.next_token + more_results = next_token != None + +class SelectResultSet(object): + + def __init__(self, domain=None, query='', max_items=None, + next_token=None): + self.domain = domain + self.query = query + self.max_items = max_items + self.next_token = next_token + + def __iter__(self): + more_results = True + num_results = 0 + while more_results: + rs = self.domain.connection.select(self.domain, self.query, + next_token=self.next_token) + for item in rs: + if self.max_items and num_results >= self.max_items: + raise StopIteration + yield item + num_results += 1 + self.next_token = rs.next_token + if self.max_items and num_results >= self.max_items: + raise StopIteration + more_results = self.next_token != None + + def next(self): + return self.__iter__().next() diff --git a/api/boto/sdb/regioninfo.py b/api/boto/sdb/regioninfo.py new file mode 100644 index 0000000..bff9dea --- /dev/null +++ b/api/boto/sdb/regioninfo.py @@ -0,0 +1,40 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from boto.ec2.regioninfo import RegionInfo + +class SDBRegionInfo(RegionInfo): + + def connect(self, **kw_params): + """ + Connect to this Region's endpoint. Returns an SDBConnection + object pointing to the endpoint associated with this region. + You may pass any of the arguments accepted by the SDBConnection + object's constructor as keyword arguments and they will be + passed along to the SDBConnection object. + + :rtype: :class:`boto.sdb.connection.SDBConnection` + :return: The connection to this regions endpoint + """ + from boto.sdb.connection import SDBConnection + return SDBConnection(region=self, **kw_params) + diff --git a/api/boto/services/__init__.py b/api/boto/services/__init__.py new file mode 100644 index 0000000..449bd16 --- /dev/null +++ b/api/boto/services/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/api/boto/services/bs.py b/api/boto/services/bs.py new file mode 100755 index 0000000..aafe867 --- /dev/null +++ b/api/boto/services/bs.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +from optparse import OptionParser +from boto.services.servicedef import ServiceDef +from boto.services.message import ServiceMessage +from boto.services.submit import Submitter +from boto.services.result import ResultProcessor +import boto +import sys, os, StringIO + +class BS(object): + + Usage = "usage: %prog [options] config_file command" + + Commands = {'reset' : 'Clear input queue and output bucket', + 'submit' : 'Submit local files to the service', + 'start' : 'Start the service', + 'status' : 'Report on the status of the service buckets and queues', + 'retrieve' : 'Retrieve output generated by a batch', + 'batches' : 'List all batches stored in current output_domain'} + + def __init__(self): + self.service_name = None + self.parser = OptionParser(usage=self.Usage) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option("-a", "--access-key", action="store", type="string", + help="your AWS Access Key") + self.parser.add_option("-s", "--secret-key", action="store", type="string", + help="your AWS Secret Access Key") + self.parser.add_option("-p", "--path", action="store", type="string", dest="path", + help="the path to local directory for submit and retrieve") + self.parser.add_option("-k", "--keypair", action="store", type="string", dest="keypair", + help="the SSH keypair used with launched instance(s)") + self.parser.add_option("-l", "--leave", action="store_true", dest="leave", + help="leave the files (don't retrieve) files during retrieve command") + self.parser.set_defaults(leave=False) + self.parser.add_option("-n", "--num-instances", action="store", type="string", dest="num_instances", + help="the number of launched instance(s)") + self.parser.set_defaults(num_instances=1) + self.parser.add_option("-i", "--ignore-dirs", action="append", type="string", dest="ignore", + help="directories that should be ignored by submit command") + self.parser.add_option("-b", "--batch-id", action="store", type="string", dest="batch", + help="batch identifier required by the retrieve command") + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_reset(self): + iq = self.sd.get_obj('input_queue') + if iq: + print 'clearing out input queue' + i = 0 + m = iq.read() + while m: + i += 1 + iq.delete_message(m) + m = iq.read() + print 'deleted %d messages' % i + ob = self.sd.get_obj('output_bucket') + ib = self.sd.get_obj('input_bucket') + if ob: + if ib and ob.name == ib.name: + return + print 'delete generated files in output bucket' + i = 0 + for k in ob: + i += 1 + k.delete() + print 'deleted %d keys' % i + + def do_submit(self): + if not self.options.path: + self.parser.error('No path provided') + if not os.path.exists(self.options.path): + self.parser.error('Invalid path (%s)' % self.options.path) + s = Submitter(self.sd) + t = s.submit_path(self.options.path, None, self.options.ignore, None, + None, True, self.options.path) + print 'A total of %d files were submitted' % t[1] + print 'Batch Identifier: %s' % t[0] + + def do_start(self): + ami_id = self.sd.get('ami_id') + instance_type = self.sd.get('instance_type', 'm1.small') + security_group = self.sd.get('security_group', 'default') + if not ami_id: + self.parser.error('ami_id option is required when starting the service') + ec2 = boto.connect_ec2() + if not self.sd.has_section('Credentials'): + self.sd.add_section('Credentials') + self.sd.set('Credentials', 'aws_access_key_id', ec2.aws_access_key_id) + self.sd.set('Credentials', 'aws_secret_access_key', ec2.aws_secret_access_key) + s = StringIO.StringIO() + self.sd.write(s) + rs = ec2.get_all_images([ami_id]) + img = rs[0] + r = img.run(user_data=s.getvalue(), key_name=self.options.keypair, + max_count=self.options.num_instances, + instance_type=instance_type, + security_groups=[security_group]) + print 'Starting AMI: %s' % ami_id + print 'Reservation %s contains the following instances:' % r.id + for i in r.instances: + print '\t%s' % i.id + + def do_status(self): + iq = self.sd.get_obj('input_queue') + if iq: + print 'The input_queue (%s) contains approximately %s messages' % (iq.id, iq.count()) + ob = self.sd.get_obj('output_bucket') + ib = self.sd.get_obj('input_bucket') + if ob: + if ib and ob.name == ib.name: + return + total = 0 + for k in ob: + total += 1 + print 'The output_bucket (%s) contains %d keys' % (ob.name, total) + + def do_retrieve(self): + if not self.options.path: + self.parser.error('No path provided') + if not os.path.exists(self.options.path): + self.parser.error('Invalid path (%s)' % self.options.path) + if not self.options.batch: + self.parser.error('batch identifier is required for retrieve command') + s = ResultProcessor(self.options.batch, self.sd) + s.get_results(self.options.path, get_file=(not self.options.leave)) + + def do_batches(self): + d = self.sd.get_obj('output_domain') + if d: + print 'Available Batches:' + rs = d.query("['type'='Batch']") + for item in rs: + print ' %s' % item.name + else: + self.parser.error('No output_domain specified for service') + + def main(self): + self.options, self.args = self.parser.parse_args() + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) != 2: + self.parser.error("config_file and command are required") + self.config_file = self.args[0] + self.sd = ServiceDef(self.config_file) + self.command = self.args[1] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + +if __name__ == "__main__": + bs = BS() + bs.main() diff --git a/api/boto/services/message.py b/api/boto/services/message.py new file mode 100644 index 0000000..6bb2e58 --- /dev/null +++ b/api/boto/services/message.py @@ -0,0 +1,59 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.sqs.message import MHMessage +from boto.utils import get_ts +from socket import gethostname +import os, mimetypes, time + +class ServiceMessage(MHMessage): + + def for_key(self, key, params=None, bucket_name=None): + if params: + self.update(params) + if key.path: + t = os.path.split(key.path) + self['OriginalLocation'] = t[0] + self['OriginalFileName'] = t[1] + mime_type = mimetypes.guess_type(t[1])[0] + if mime_type == None: + mime_type = 'application/octet-stream' + self['Content-Type'] = mime_type + s = os.stat(key.path) + t = time.gmtime(s[7]) + self['FileAccessedDate'] = get_ts(t) + t = time.gmtime(s[8]) + self['FileModifiedDate'] = get_ts(t) + t = time.gmtime(s[9]) + self['FileCreateDate'] = get_ts(t) + else: + self['OriginalFileName'] = key.name + self['OriginalLocation'] = key.bucket.name + self['ContentType'] = key.content_type + self['Host'] = gethostname() + if bucket_name: + self['Bucket'] = bucket_name + else: + self['Bucket'] = key.bucket.name + self['InputKey'] = key.name + self['Size'] = key.size + diff --git a/api/boto/services/result.py b/api/boto/services/result.py new file mode 100644 index 0000000..240085b --- /dev/null +++ b/api/boto/services/result.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import getopt, sys, os, time, mimetypes +from datetime import datetime, timedelta +from boto.services.servicedef import ServiceDef +from boto.utils import parse_ts +import boto + +class ResultProcessor: + + LogFileName = 'log.csv' + + def __init__(self, batch_name, sd, mimetype_files=None): + self.sd = sd + self.batch = batch_name + self.log_fp = None + self.num_files = 0 + self.total_time = 0 + self.min_time = timedelta.max + self.max_time = timedelta.min + self.earliest_time = datetime.max + self.latest_time = datetime.min + self.queue = self.sd.get_obj('output_queue') + self.domain = self.sd.get_obj('output_domain') + + def calculate_stats(self, msg): + start_time = parse_ts(msg['Service-Read']) + end_time = parse_ts(msg['Service-Write']) + elapsed_time = end_time - start_time + if elapsed_time > self.max_time: + self.max_time = elapsed_time + if elapsed_time < self.min_time: + self.min_time = elapsed_time + self.total_time += elapsed_time.seconds + if start_time < self.earliest_time: + self.earliest_time = start_time + if end_time > self.latest_time: + self.latest_time = end_time + + def log_message(self, msg, path): + keys = msg.keys() + keys.sort() + if not self.log_fp: + self.log_fp = open(os.path.join(path, self.LogFileName), 'w') + line = ','.join(keys) + self.log_fp.write(line+'\n') + values = [] + for key in keys: + value = msg[key] + if value.find(',') > 0: + value = '"%s"' % value + values.append(value) + line = ','.join(values) + self.log_fp.write(line+'\n') + + def process_record(self, record, path, get_file=True): + self.log_message(record, path) + self.calculate_stats(record) + outputs = record['OutputKey'].split(',') + if record.has_key('OutputBucket'): + bucket = boto.lookup('s3', record['OutputBucket']) + else: + bucket = boto.lookup('s3', record['Bucket']) + for output in outputs: + if get_file: + key_name, type = output.split(';') + if type: + mimetype = type.split('=')[1] + key = bucket.lookup(key_name) + file_name = os.path.join(path, key_name) + print 'retrieving file: %s to %s' % (key_name, file_name) + key.get_contents_to_filename(file_name) + self.num_files += 1 + + def get_results_from_queue(self, path, get_file=True, delete_msg=True): + m = self.queue.read() + while m: + if m.has_key('Batch') and m['Batch'] == self.batch: + self.process_record(m, path, get_file) + if delete_msg: + self.queue.delete_message(m) + m = self.queue.read() + + def get_results_from_domain(self, path, get_file=True): + rs = self.domain.query("['Batch'='%s']" % self.batch) + for item in rs: + self.process_record(item, path, get_file) + + def get_results_from_bucket(self, path): + bucket = self.sd.get_obj('output_bucket') + if bucket: + print 'No output queue or domain, just retrieving files from output_bucket' + for key in bucket: + file_name = os.path.join(path, key_name) + print 'retrieving file: %s to %s' % (key_name, file_name) + key.get_contents_to_filename(file_name) + self.num_files + 1 + + def get_results(self, path, get_file=True, delete_msg=True): + if not os.path.isdir(path): + os.mkdir(path) + if self.queue: + self.get_results_from_queue(path, get_file) + elif self.domain: + self.get_results_from_domain(path, get_file) + else: + self.get_results_from_bucket(path) + if self.log_fp: + self.log_fp.close() + print '%d results successfully retrieved.' % self.num_files + if self.num_files > 0: + self.avg_time = float(self.total_time)/self.num_files + print 'Minimum Processing Time: %d' % self.min_time.seconds + print 'Maximum Processing Time: %d' % self.max_time.seconds + print 'Average Processing Time: %f' % self.avg_time + self.elapsed_time = self.latest_time-self.earliest_time + print 'Elapsed Time: %d' % self.elapsed_time.seconds + tput = 1.0 / ((self.elapsed_time.seconds/60.0) / self.num_files) + print 'Throughput: %f transactions / minute' % tput + diff --git a/api/boto/services/service.py b/api/boto/services/service.py new file mode 100644 index 0000000..942c47f --- /dev/null +++ b/api/boto/services/service.py @@ -0,0 +1,163 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.services.message import ServiceMessage +from boto.services.servicedef import ServiceDef +from boto.pyami.scriptbase import ScriptBase +from boto.exception import S3ResponseError +from boto.utils import get_ts +import StringIO +import time +import os +import sys, traceback +import mimetypes + +class Service(ScriptBase): + + # Time required to process a transaction + ProcessingTime = 60 + + def __init__(self, config_file=None, mimetype_files=None): + ScriptBase.__init__(self, config_file) + self.name = self.__class__.__name__ + self.working_dir = boto.config.get('Pyami', 'working_dir') + self.sd = ServiceDef(config_file) + self.retry_count = self.sd.getint('retry_count', 5) + self.loop_delay = self.sd.getint('loop_delay', 30) + self.processing_time = self.sd.getint('processing_time', 60) + self.input_queue = self.sd.get_obj('input_queue') + self.output_queue = self.sd.get_obj('output_queue') + self.output_domain = self.sd.get_obj('output_domain') + if mimetype_files: + mimetypes.init(mimetype_files) + + def split_key(key): + if key.find(';') < 0: + t = (key, '') + else: + key, type = key.split(';') + label, mtype = type.split('=') + t = (key, mtype) + return t + + def read_message(self): + boto.log.info('read_message') + message = self.input_queue.read(self.processing_time) + if message: + boto.log.info(message.get_body()) + key = 'Service-Read' + message[key] = get_ts() + return message + + # retrieve the source file from S3 + def get_file(self, message): + bucket_name = message['Bucket'] + key_name = message['InputKey'] + file_name = os.path.join(self.working_dir, message.get('OriginalFileName', 'in_file')) + boto.log.info('get_file: %s/%s to %s' % (bucket_name, key_name, file_name)) + bucket = boto.lookup('s3', bucket_name) + key = bucket.new_key(key_name) + key.get_contents_to_filename(os.path.join(self.working_dir, file_name)) + return file_name + + # process source file, return list of output files + def process_file(self, in_file_name, msg): + return [] + + # store result file in S3 + def put_file(self, bucket_name, file_path, key_name=None): + boto.log.info('putting file %s as %s.%s' % (file_path, bucket_name, key_name)) + bucket = boto.lookup('s3', bucket_name) + key = bucket.new_key(key_name) + key.set_contents_from_filename(file_path) + return key + + def save_results(self, results, input_message, output_message): + output_keys = [] + for file, type in results: + if input_message.has_key('OutputBucket'): + output_bucket = input_message['OutputBucket'] + else: + output_bucket = input_message['Bucket'] + key_name = os.path.split(file)[1] + key = self.put_file(output_bucket, file, key_name) + output_keys.append('%s;type=%s' % (key.name, type)) + output_message['OutputKey'] = ','.join(output_keys) + + # write message to each output queue + def write_message(self, message): + message['Service-Write'] = get_ts() + message['Server'] = self.name + if os.environ.has_key('HOSTNAME'): + message['Host'] = os.environ['HOSTNAME'] + else: + message['Host'] = 'unknown' + message['Instance-ID'] = self.instance_id + if self.output_queue: + boto.log.info('Writing message to SQS queue: %s' % self.output_queue.id) + self.output_queue.write(message) + if self.output_domain: + boto.log.info('Writing message to SDB domain: %s' % self.output_domain.name) + item_name = '/'.join([message['Service-Write'], message['Bucket'], message['InputKey']]) + self.output_domain.put_attributes(item_name, message) + + # delete message from input queue + def delete_message(self, message): + boto.log.info('deleting message from %s' % self.input_queue.id) + self.input_queue.delete_message(message) + + # to clean up any files, etc. after each iteration + def cleanup(self): + pass + + def shutdown(self): + on_completion = self.sd.get('on_completion', 'shutdown') + if on_completion == 'shutdown': + if self.instance_id: + time.sleep(60) + c = boto.connect_ec2() + c.terminate_instances([self.instance_id]) + + def main(self, notify=False): + self.notify('Service: %s Starting' % self.name) + empty_reads = 0 + while self.retry_count < 0 or empty_reads < self.retry_count: + try: + input_message = self.read_message() + if input_message: + empty_reads = 0 + output_message = ServiceMessage(None, input_message.get_body()) + input_file = self.get_file(input_message) + results = self.process_file(input_file, output_message) + self.save_results(results, input_message, output_message) + self.write_message(output_message) + self.delete_message(input_message) + self.cleanup() + else: + empty_reads += 1 + time.sleep(self.loop_delay) + except Exception, e: + boto.log.exception('Service Failed') + empty_reads += 1 + self.notify('Service: %s Shutting Down' % self.name) + self.shutdown() + diff --git a/api/boto/services/servicedef.py b/api/boto/services/servicedef.py new file mode 100644 index 0000000..1cb01aa --- /dev/null +++ b/api/boto/services/servicedef.py @@ -0,0 +1,91 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.pyami.config import Config +from boto.services.message import ServiceMessage +import boto + +class ServiceDef(Config): + + def __init__(self, config_file, aws_access_key_id=None, aws_secret_access_key=None): + Config.__init__(self, config_file) + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + script = Config.get(self, 'Pyami', 'scripts') + if script: + self.name = script.split('.')[-1] + else: + self.name = None + + + def get(self, name, default=None): + return Config.get(self, self.name, name, default) + + def has_option(self, option): + return Config.has_option(self, self.name, option) + + def getint(self, option, default=0): + try: + val = Config.get(self, self.name, option) + val = int(val) + except: + val = int(default) + return val + + def getbool(self, option, default=False): + try: + val = Config.get(self, self.name, option) + if val.lower() == 'true': + val = True + else: + val = False + except: + val = default + return val + + def get_obj(self, name): + """ + Returns the AWS object associated with a given option. + + The heuristics used are a bit lame. If the option name contains + the word 'bucket' it is assumed to be an S3 bucket, if the name + contains the word 'queue' it is assumed to be an SQS queue and + if it contains the word 'domain' it is assumed to be a SimpleDB + domain. If the option name specified does not exist in the + config file or if the AWS object cannot be retrieved this + returns None. + """ + val = self.get(name) + if not val: + return None + if name.find('queue') >= 0: + obj = boto.lookup('sqs', val) + if obj: + obj.set_message_class(ServiceMessage) + elif name.find('bucket') >= 0: + obj = boto.lookup('s3', val) + elif name.find('domain') >= 0: + obj = boto.lookup('sdb', val) + else: + obj = None + return obj + + diff --git a/api/boto/services/sonofmmm.cfg b/api/boto/services/sonofmmm.cfg new file mode 100644 index 0000000..d70d379 --- /dev/null +++ b/api/boto/services/sonofmmm.cfg @@ -0,0 +1,43 @@ +# +# Your AWS Credentials +# You only need to supply these in this file if you are not using +# the boto tools to start your service +# +#[Credentials] +#aws_access_key_id = +#aws_secret_access_key = + +# +# Fill out this section if you want emails from the service +# when it starts and stops +# +#[Notification] +#smtp_host = +#smtp_user = +#smtp_pass = +#smtp_from = +#smtp_to = + +[Pyami] +scripts = boto.services.sonofmmm.SonOfMMM + +[SonOfMMM] +# id of the AMI to be launched +ami_id = ami-dc799cb5 +# number of times service will read an empty queue before exiting +# a negative value will cause the service to run forever +retry_count = 5 +# seconds to wait after empty queue read before reading again +loop_delay = 10 +# average time it takes to process a transaction +# controls invisibility timeout of messages +processing_time = 60 +ffmpeg_args = -y -i %%s -f mov -r 29.97 -b 1200kb -mbd 2 -flags +4mv+trell -aic 2 -cmp 2 -subcmp 2 -ar 48000 -ab 19200 -s 320x240 -vcodec mpeg4 -acodec libfaac %%s +output_mimetype = video/quicktime +output_ext = .mov +input_bucket = +output_bucket = +output_domain = +output_queue = +input_queue = + diff --git a/api/boto/services/sonofmmm.py b/api/boto/services/sonofmmm.py new file mode 100644 index 0000000..5b94f90 --- /dev/null +++ b/api/boto/services/sonofmmm.py @@ -0,0 +1,80 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.services.service import Service +from boto.services.message import ServiceMessage +import os, time, mimetypes + +class SonOfMMM(Service): + + def __init__(self, config_file=None): + Service.__init__(self, config_file) + self.log_file = '%s.log' % self.instance_id + self.log_path = os.path.join(self.working_dir, self.log_file) + boto.set_file_logger(self.name, self.log_path) + if self.sd.has_option('ffmpeg_args'): + self.command = '/usr/local/bin/ffmpeg ' + self.sd.get('ffmpeg_args') + else: + self.command = '/usr/local/bin/ffmpeg -y -i %s %s' + self.output_mimetype = self.sd.get('output_mimetype') + if self.sd.has_option('output_ext'): + self.output_ext = self.sd.get('output_ext') + else: + self.output_ext = mimetypes.guess_extension(self.output_mimetype) + self.output_bucket = self.sd.get_obj('output_bucket') + self.input_bucket = self.sd.get_obj('input_bucket') + # check to see if there are any messages queue + # if not, create messages for all files in input_bucket + m = self.input_queue.read(1) + if not m: + self.queue_files() + + def queue_files(self): + boto.log.info('Queueing files from %s' % self.input_bucket.name) + for key in self.input_bucket: + boto.log.info('Queueing %s' % key.name) + m = ServiceMessage() + if self.output_bucket: + d = {'OutputBucket' : self.output_bucket.name} + else: + d = None + m.for_key(key, d) + self.input_queue.write(m) + + def process_file(self, in_file_name, msg): + base, ext = os.path.splitext(in_file_name) + out_file_name = os.path.join(self.working_dir, + base+self.output_ext) + command = self.command % (in_file_name, out_file_name) + boto.log.info('running:\n%s' % command) + status = self.run(command) + if status == 0: + return [(out_file_name, self.output_mimetype)] + else: + return [] + + def shutdown(self): + if os.path.isfile(self.log_path): + if self.output_bucket: + key = self.output_bucket.new_key(self.log_file) + key.set_contents_from_filename(self.log_path) + Service.shutdown(self) diff --git a/api/boto/services/submit.py b/api/boto/services/submit.py new file mode 100644 index 0000000..dfa71f2 --- /dev/null +++ b/api/boto/services/submit.py @@ -0,0 +1,87 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +import time, os + +class Submitter: + + def __init__(self, sd): + self.sd = sd + self.input_bucket = self.sd.get_obj('input_bucket') + self.output_bucket = self.sd.get_obj('output_bucket') + self.output_domain = self.sd.get_obj('output_domain') + self.queue = self.sd.get_obj('input_queue') + + def get_key_name(self, fullpath, prefix): + key_name = fullpath[len(prefix):] + l = key_name.split(os.sep) + return '/'.join(l) + + def write_message(self, key, metadata): + if self.queue: + m = self.queue.new_message() + m.for_key(key, metadata) + if self.output_bucket: + m['OutputBucket'] = self.output_bucket.name + self.queue.write(m) + + def submit_file(self, path, metadata=None, cb=None, num_cb=0, prefix='/'): + if not metadata: + metadata = {} + key_name = self.get_key_name(path, prefix) + k = self.input_bucket.new_key(key_name) + k.update_metadata(metadata) + k.set_contents_from_filename(path, replace=False, cb=cb, num_cb=num_cb) + self.write_message(k, metadata) + + def submit_path(self, path, tags=None, ignore_dirs=None, cb=None, num_cb=0, status=False, prefix='/'): + path = os.path.expanduser(path) + path = os.path.expandvars(path) + path = os.path.abspath(path) + total = 0 + metadata = {} + if tags: + metadata['Tags'] = tags + l = [] + for t in time.gmtime(): + l.append(str(t)) + metadata['Batch'] = '_'.join(l) + if self.output_domain: + self.output_domain.put_attributes(metadata['Batch'], {'type' : 'Batch'}) + if os.path.isdir(path): + for root, dirs, files in os.walk(path): + if ignore_dirs: + for ignore in ignore_dirs: + if ignore in dirs: + dirs.remove(ignore) + for file in files: + fullpath = os.path.join(root, file) + if status: + print 'Submitting %s' % fullpath + self.submit_file(fullpath, metadata, cb, num_cb, prefix) + total += 1 + elif os.path.isfile(path): + self.submit_file(path, metadata, cb, num_cb) + total += 1 + else: + print 'problem with %s' % path + return (metadata['Batch'], total) diff --git a/api/boto/sqs/20070501/__init__.py b/api/boto/sqs/20070501/__init__.py new file mode 100644 index 0000000..561f9cf --- /dev/null +++ b/api/boto/sqs/20070501/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from connection import SQSConnection as Connection +from queue import Queue +from message import Message, MHMessage + +__all__ = ['Connection', 'Queue', 'Message', 'MHMessage'] diff --git a/api/boto/sqs/20070501/attributes.py b/api/boto/sqs/20070501/attributes.py new file mode 100644 index 0000000..b13370d --- /dev/null +++ b/api/boto/sqs/20070501/attributes.py @@ -0,0 +1,45 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Attribute Name/Value set +""" + +class Attributes(dict): + + def __init__(self): + self.current_key = None + self.current_value = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'AttributedValue': + self[self.current_key] = self.current_value + elif name == 'Attribute': + self.current_key = value + elif name == 'Value': + self.current_value = value + else: + setattr(self, name, value) + + diff --git a/api/boto/sqs/20070501/connection.py b/api/boto/sqs/20070501/connection.py new file mode 100644 index 0000000..7890aa2 --- /dev/null +++ b/api/boto/sqs/20070501/connection.py @@ -0,0 +1,389 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.connection import AWSAuthConnection, AWSQueryConnection +import xml.sax +from boto.sqs.queue import Queue +from boto.sqs.message import Message +from boto.sqs.attributes import Attributes +from boto import handler +from boto.resultset import ResultSet +from boto.exception import SQSError + +PERM_ReceiveMessage = 'ReceiveMessage' +PERM_SendMessage = 'SendMessage' +PERM_FullControl = 'FullControl' + +AllPermissions = [PERM_ReceiveMessage, PERM_SendMessage, PERM_FullControl] + +class SQSQueryConnection(AWSQueryConnection): + + """ + This class uses the Query API (boo!) to SQS to access some of the + new features which have not yet been added to the REST api (yeah!). + """ + + DefaultHost = 'queue.amazonaws.com' + APIVersion = '2007-05-01' + SignatureVersion = '1' + DefaultContentType = 'text/plain' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0, https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + host, debug, https_connection_factory) + + def get_queue_attributes(self, queue_url, attribute='All'): + params = {'Attribute' : attribute} + response = self.make_request('GetQueueAttributes', params, queue_url) + body = response.read() + if response.status == 200: + attrs = Attributes() + h = handler.XmlHandler(attrs, self) + xml.sax.parseString(body, h) + return attrs + else: + raise SQSError(response.status, response.reason, body) + + def set_queue_attribute(self, queue_url, attribute, value): + params = {'Attribute' : attribute, 'Value' : value} + response = self.make_request('SetQueueAttributes', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def change_message_visibility(self, queue_url, message_id, vtimeout): + params = {'MessageId' : message_id, + 'VisibilityTimeout' : vtimeout} + response = self.make_request('ChangeMessageVisibility', params, + queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def add_grant(self, queue_url, permission, email_address=None, user_id=None): + params = {'Permission' : permission} + if user_id: + params['Grantee.ID'] = user_id + if email_address: + params['Grantee.EmailAddress'] = email_address + response = self.make_request('AddGrant', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def remove_grant(self, queue_url, permission, email_address=None, user_id=None): + params = {'Permission' : permission} + if user_id: + params['Grantee.ID'] = user_id + if email_address: + params['Grantee.EmailAddress'] = email_address + response = self.make_request('RemoveGrant', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def list_grants(self, queue_url, permission=None, email_address=None, user_id=None): + params = {} + if user_id: + params['Grantee.ID'] = user_id + if email_address: + params['Grantee.EmailAddress'] = email_address + if permission: + params['Permission'] = permission + response = self.make_request('ListGrants', params, queue_url) + body = response.read() + if response.status == 200: + return body + else: + raise SQSError(response.status, response.reason, body) + + def receive_message(self, queue_url, number_messages=1, + visibility_timeout=None, message_class=Message): + """ + This provides the same functionality as the read and get_messages methods + of the queue object. The only reason this is included here is that there is + currently a bug in SQS that makes it impossible to read a message from a queue + owned by someone else (even if you have been granted appropriate permissions) + via the REST interface. As it turns out, I need to be able to do this so until + the REST interface gets fixed this is the workaround. + """ + params = {'NumberOfMessages' : number_messages} + if visibility_timeout: + params['VisibilityTimeout'] = visibility_timeout + response = self.make_request('ReceiveMessage', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet([('Message', message_class)]) + h = handler.XmlHandler(rs, queue_url) + xml.sax.parseString(body, h) + if len(rs) == 1: + return rs[0] + else: + return rs + else: + raise SQSError(response.status, response.reason, body) + + def delete_message(self, queue_url, message_id): + """ + Because we have to use the Query interface to read messages from queues that + we don't own, we also have to provide a way to delete those messages via Query. + """ + params = {'MessageId' : message_id} + response = self.make_request('DeleteMessage', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + +class SQSConnection(AWSAuthConnection): + + DefaultHost = 'queue.amazonaws.com' + APIVersion = '2007-05-01' + DefaultContentType = 'text/plain' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0, https_connection_factory=None): + AWSAuthConnection.__init__(self, host, + aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, debug, + https_connection_factory) + self.query_conn = None + + def make_request(self, method, path, headers=None, data=''): + # add auth header + if headers == None: + headers = {} + + if not headers.has_key('AWS-Version'): + headers['AWS-Version'] = self.APIVersion + + if not headers.has_key('Content-Type'): + headers['Content-Type'] = self.DefaultContentType + + return AWSAuthConnection.make_request(self, method, path, + headers, data) + + def get_query_connection(self): + if not self.query_conn: + self.query_conn = SQSQueryConnection(self.aws_access_key_id, + self.aws_secret_access_key, + self.is_secure, self.port, + self.proxy, self.proxy_port, + self.server, self.debug, + self.https_connection_factory) + return self.query_conn + + def get_all_queues(self, prefix=''): + if prefix: + path = '/?QueueNamePrefix=%s' % prefix + else: + path = '/' + response = self.make_request('GET', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet([('QueueUrl', Queue)]) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def get_queue(self, queue_name): + i = 0 + rs = self.get_all_queues(queue_name) + for q in rs: + i += 1 + if i != 1: + return None + else: + return q + + def get_queue_attributes(self, queue_url, attribute='All'): + """ + Performs a GetQueueAttributes request and returns an Attributes + instance (subclass of a Dictionary) holding the requested + attribute name/value pairs. + Inputs: + queue_url - the URL of the desired SQS queue + attribute - All|ApproximateNumberOfMessages|VisibilityTimeout + Default value is "All" + Returns: + An Attribute object which is a mapping type holding the + requested name/value pairs + """ + qc = self.get_query_connection() + return qc.get_queue_attributes(queue_url, attribute) + + def set_queue_attribute(self, queue_url, attribute, value): + """ + Performs a SetQueueAttributes request. + Inputs: + queue_url - The URL of the desired SQS queue + attribute - The name of the attribute you want to set. The + only valid value at this time is: VisibilityTimeout + value - The new value for the attribute. + For VisibilityTimeout the value must be an + integer number of seconds from 0 to 86400. + Returns: + Boolean True if successful, otherwise False. + """ + qc = self.get_query_connection() + return qc.set_queue_attribute(queue_url, attribute, value) + + def change_message_visibility(self, queue_url, message_id, vtimeout): + """ + Change the VisibilityTimeout for an individual message. + Inputs: + queue_url - The URL of the desired SQS queue + message_id - The ID of the message whose timeout will be changed + vtimeout - The new VisibilityTimeout value, in seconds + Returns: + Boolean True if successful, otherwise False + Note: This functionality is also available as a method of the + Message object. + """ + qc = self.get_query_connection() + return qc.change_message_visibility(queue_url, message_id, vtimeout) + + def add_grant(self, queue_url, permission, email_address=None, user_id=None): + """ + Add a grant to a queue. + Inputs: + queue_url - The URL of the desired SQS queue + permission - The permission being granted. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + qc = self.get_query_connection() + return qc.add_grant(queue_url, permission, email_address, user_id) + + def remove_grant(self, queue_url, permission, email_address=None, user_id=None): + """ + Remove a grant from a queue. + Inputs: + queue_url - The URL of the desired SQS queue + permission - The permission being removed. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + qc = self.get_query_connection() + return qc.remove_grant(queue_url, permission, email_address, user_id) + + def list_grants(self, queue_url, permission=None, email_address=None, user_id=None): + """ + List the grants to a queue. + Inputs: + queue_url - The URL of the desired SQS queue + permission - The permission granted. One of "ReceiveMessage", "SendMessage" or "FullControl". + If supplied, only grants that allow this permission will be returned. + email_address - the email address of the grantee. If supplied, only grants related to this email + address will be returned + user_id - The ID of the grantee. If supplied, only grants related to his user_id will be returned. + Returns: + A string containing the XML Response elements describing the grants. + """ + qc = self.get_query_connection() + return qc.list_grants(queue_url, permission, email_address, user_id) + + def create_queue(self, queue_name, visibility_timeout=None): + """ + Create a new queue. + Inputs: + queue_name - The name of the new queue + visibility_timeout - (Optional) The default visibility + timeout for the new queue. + Returns: + A new Queue object representing the newly created queue. + """ + path = '/?QueueName=%s' % queue_name + if visibility_timeout: + path = path + '&DefaultVisibilityTimeout=%d' % visibility_timeout + response = self.make_request('POST', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + q = Queue(self) + h = handler.XmlHandler(q, self) + xml.sax.parseString(body, h) + return q + + def delete_queue(self, queue, force_deletion=False): + """ + Delete an SQS Queue. + Inputs: + queue - a Queue object representing the SQS queue to be deleted. + force_deletion - (Optional) Normally, SQS will not delete a + queue that contains messages. However, if + the force_deletion argument is True, the + queue will be deleted regardless of whether + there are messages in the queue or not. + USE WITH CAUTION. This will delete all + messages in the queue as well. + Returns: + An empty ResultSet object. Not sure why, actually. It + should probably return a Boolean indicating success or + failure. + """ + method = 'DELETE' + path = queue.id + if force_deletion: + path = path + '?ForceDeletion=true' + response = self.make_request(method, path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + diff --git a/api/boto/sqs/20070501/message.py b/api/boto/sqs/20070501/message.py new file mode 100644 index 0000000..0c45b31 --- /dev/null +++ b/api/boto/sqs/20070501/message.py @@ -0,0 +1,180 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Message +""" + +import base64 +import StringIO + +class RawMessage: + """ + Base class for SQS messages. RawMessage does not encode the message + in any way. Whatever you store in the body of the message is what + will be written to SQS and whatever is returned from SQS is stored + directly into the body of the message. + """ + + def __init__(self, queue=None, body=''): + self.queue = queue + self._body = '' + self.set_body(body) + self.id = None + + def __len__(self): + return len(self._body) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'MessageBody': + self.set_body(value) + elif name == 'MessageId': + self.id = value + else: + setattr(self, name, value) + + def set_body(self, body): + """ + Set the body of the message. You should always call this method + rather than setting the attribute directly. + """ + self._body = body + + def get_body(self): + """ + Retrieve the body of the message. + """ + return self._body + + def get_body_encoded(self): + """ + This method is really a semi-private method used by the Queue.write + method when writing the contents of the message to SQS. The + RawMessage class does not encode the message in any way so this + just calls get_body(). You probably shouldn't need to call this + method in the normal course of events. + """ + return self.get_body() + + def change_visibility(self, vtimeout): + """ + Convenience function to allow you to directly change the + invisibility timeout for an individual message that has been + read from an SQS queue. This won't affect the default visibility + timeout of the queue. + """ + return self.queue.connection.change_message_visibility(self.queue.id, + self.id, + vtimeout) +class Message(RawMessage): + """ + The default Message class used for SQS queues. This class automatically + encodes/decodes the message body using Base64 encoding to avoid any + illegal characters in the message body. See: + + http://developer.amazonwebservices.com/connect/thread.jspa?messageID=49680%EC%88%90 + + for details on why this is a good idea. The encode/decode is meant to + be transparent to the end-user. + """ + + def endElement(self, name, value, connection): + if name == 'MessageBody': + # Decode the message body returned from SQS using base64 + self.set_body(base64.b64decode(value)) + elif name == 'MessageId': + self.id = value + else: + setattr(self, name, value) + + def get_body_encoded(self): + """ + Because the Message class encodes the message body in base64 + this private method used by queue.write needs to perform the + encoding. + """ + return base64.b64encode(self.get_body()) + +class MHMessage(Message): + """ + The MHMessage class provides a message that provides RFC821-like + headers like this: + + HeaderName: HeaderValue + + The encoding/decoding of this is handled automatically and after + the message body has been read, the message instance can be treated + like a mapping object, i.e. m['HeaderName'] would return 'HeaderValue'. + """ + + def __init__(self, queue=None, body='', xml_attrs=None): + self._dict = {} + Message.__init__(self, queue, body) + + def set_body(self, body): + fp = StringIO.StringIO(body) + line = fp.readline() + while line: + delim = line.find(':') + key = line[0:delim] + value = line[delim+1:].strip() + self._dict[key.strip()] = value.strip() + line = fp.readline() + + def get_body(self): + s = '' + for key,value in self._dict.items(): + s = s + '%s: %s\n' % (key, value) + return s + + def __len__(self): + return len(self.get_body()) + + def __getitem__(self, key): + if self._dict.has_key(key): + return self._dict[key] + else: + raise KeyError(key) + + def __setitem__(self, key, value): + self._dict[key] = value + + def keys(self): + return self._dict.keys() + + def values(self): + return self._dict.values() + + def items(self): + return self._dict.items() + + def has_key(self, key): + return self._dict.has_key(key) + + def update(self, d): + return self._dict.update(d) + + def get(self, key, default=None): + return self._dict.get(key, default) + diff --git a/api/boto/sqs/20070501/queue.py b/api/boto/sqs/20070501/queue.py new file mode 100644 index 0000000..64289ef --- /dev/null +++ b/api/boto/sqs/20070501/queue.py @@ -0,0 +1,343 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Queue +""" + +import xml.sax +import urlparse +from boto.exception import SQSError +from boto.handler import XmlHandler +from boto.sqs.message import Message +from boto.resultset import ResultSet + +class Queue: + + def __init__(self, connection=None, url=None, message_class=Message): + self.connection = connection + self.url = url + self.message_class = message_class + self.visibility_timeout = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'QueueUrl': + self.url = value + if value: + self.id = urlparse.urlparse(value)[2] + elif name == 'VisibilityTimeout': + self.visibility_timeout = int(value) + else: + setattr(self, name, value) + + def set_message_class(self, message_class): + """ + Set the message class that should be used when instantiating messages read + from the queue. By default, the class boto.sqs.message.Message is used but + this can be overriden with any class that behaves like a message. + Inputs: + message_class - The new message class + Returns: + Nothing + """ + self.message_class = message_class + + def get_attributes(self, attributes='All'): + """ + Retrieves attributes about this queue object and returns + them in an Attribute instance (subclass of a Dictionary). + Inputs: + attributes - A string containing + All|ApproximateNumberOfMessages|VisibilityTimeout + Default value is "All" + Returns: + An Attribute object which is a mapping type holding the + requested name/value pairs + """ + return self.connection.get_queue_attributes(self.id, attributes) + + def set_attribute(self, attribute, value): + """ + Set a new value for an attribute of the Queue. + Inputs: + attribute - The name of the attribute you want to set. The + only valid value at this time is: VisibilityTimeout + value - The new value for the attribute. + For VisibilityTimeout the value must be an + integer number of seconds from 0 to 86400. + Returns: + Boolean True if successful, otherwise False. + """ + return self.connection.set_queue_attribute(self.id, attribute, value) + + def add_grant(self, permission, email_address=None, user_id=None): + """ + Add a grant to this queue. + Inputs: + permission - The permission being granted. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + return self.connection.add_grant(self.id, permission, email_address, user_id) + + def remove_grant(self, permission, email_address=None, user_id=None): + """ + Remove a grant from this queue. + Inputs: + permission - The permission being removed. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + return self.connection.remove_grant(self.id, permission, email_address, user_id) + + def list_grants(self, permission=None, email_address=None, user_id=None): + """ + List the grants to this queue. + Inputs: + permission - The permission granted. One of "ReceiveMessage", "SendMessage" or "FullControl". + If supplied, only grants that allow this permission will be returned. + email_address - the email address of the grantee. If supplied, only grants related to this email + address will be returned + user_id - The ID of the grantee. If supplied, only grants related to his user_id will be returned. + Returns: + A string containing the XML Response elements describing the grants. + """ + return self.connection.list_grants(self.id, permission, email_address, user_id) + + def get_timeout(self): + """ + Get the visibility timeout for the queue. + Inputs: + None + Returns: + The number of seconds as an integer. + """ + a = self.get_attributes('VisibilityTimeout') + return int(a['VisibilityTimeout']) + + def set_timeout(self, visibility_timeout): + """ + Set the visibility timeout for the queue. + Inputs: + visibility_timeout - The desired timeout in seconds + Returns: + Nothing + """ + retval = self.set_attribute('VisibilityTimeout', visibility_timeout) + if retval: + self.visibility_timeout = visibility_timeout + return retval + + def read(self, visibility_timeout=None): + """ + Read a single message from the queue. + Inputs: + visibility_timeout - The timeout for this message in seconds + Returns: + A single message or None if queue is empty + """ + rs = self.get_messages(1, visibility_timeout) + if len(rs) == 1: + return rs[0] + else: + return None + + def write(self, message): + """ + Add a single message to the queue. + Inputs: + message - The message to be written to the queue + Returns: + None + """ + path = '%s/back' % self.id + message.queue = self + response = self.connection.make_request('PUT', path, None, + message.get_body_encoded()) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + handler = XmlHandler(message, self.connection) + xml.sax.parseString(body, handler) + return None + + def new_message(self, body=''): + return self.message_class(self, body) + + # get a variable number of messages, returns a list of messages + def get_messages(self, num_messages=1, visibility_timeout=None): + path = '%s/front?NumberOfMessages=%d' % (self.id, num_messages) + if visibility_timeout: + path = '%s&VisibilityTimeout=%d' % (path, visibility_timeout) + response = self.connection.make_request('GET', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet([('Message', self.message_class)]) + h = XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def delete_message(self, message): + path = '%s/%s' % (self.id, message.id) + response = self.connection.make_request('DELETE', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet() + h = XmlHandler(rs, self.connection) + xml.sax.parseString(body, h) + return rs + + def clear(self, page_size=100, vtimeout=10): + """Utility function to remove all messages from a queue""" + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + self.delete_message(m) + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def count(self, page_size=100, vtimeout=10): + """ + Utility function to count the number of messages in a queue. + Note: This function now calls GetQueueAttributes to obtain + an 'approximate' count of the number of messages in a queue. + """ + a = self.get_attributes('ApproximateNumberOfMessages') + return int(a['ApproximateNumberOfMessages']) + + def count_slow(self, page_size=100, vtimeout=10): + """ + Deprecated. This is the old 'count' method that actually counts + the messages by reading them all. This gives an accurate count but + is very slow for queues with non-trivial number of messasges. + Instead, use get_attribute('ApproximateNumberOfMessages') to take + advantage of the new SQS capability. This is retained only for + the unit tests. + """ + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def dump(self, file_name, page_size=100, vtimeout=10, sep='\n'): + """Utility function to dump the messages in a queue to a file""" + fp = open(file_name, 'wb') + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + fp.write(m.get_body()) + if sep: + fp.write(sep) + n += 1 + l = self.get_messages(page_size, vtimeout) + fp.close() + return n + + def save(self, file_name, sep='\n'): + """ + Read all messages from the queue and persist them to local file. + Messages are written to the file and the 'sep' string is written + in between messages. Messages are deleted from the queue after + being written to the file. + Returns the number of messages saved. + """ + fp = open(file_name, 'wb') + n = 0 + m = self.read() + while m: + n += 1 + fp.write(m.get_body()) + if sep: + fp.write(sep) + self.delete_message(m) + m = self.read() + fp.close() + return n + + def save_to_s3(self, bucket): + """ + Read all messages from the queue and persist them to S3. + Messages are stored in the S3 bucket using a naming scheme of: + / + Messages are deleted from the queue after being saved to S3. + Returns the number of messages saved. + """ + n = 0 + m = self.read() + while m: + n += 1 + key = bucket.new_key('%s/%s' % (self.id, m.id)) + key.set_contents_from_string(m.get_body()) + self.delete_message(m) + m = self.read() + return n + + def load_from_s3(self, bucket, prefix=None): + """ + Load messages previously saved to S3. + """ + n = 0 + if prefix: + prefix = '%s/' % prefix + else: + prefix = '%s/' % self.id + rs = bucket.list(prefix=prefix) + for key in rs: + n += 1 + m = self.new_message(key.get_contents_as_string()) + self.write(m) + return n + + def load(self, file_name, sep='\n'): + """Utility function to load messages from a file to a queue""" + fp = open(file_name, 'rb') + n = 0 + body = '' + l = fp.readline() + while l: + if l == sep: + m = Message(self, body) + self.write(m) + n += 1 + print 'writing message %d' % n + body = '' + else: + body = body + l + l = fp.readline() + fp.close() + return n + + diff --git a/api/boto/sqs/20070501/readme.txt b/api/boto/sqs/20070501/readme.txt new file mode 100644 index 0000000..7132579 --- /dev/null +++ b/api/boto/sqs/20070501/readme.txt @@ -0,0 +1,6 @@ +The main SQS implementation now uses the 2008-01-01 API verson. To use the older API version +(2007-05-01) you need to edit your /etc/boto.cfg or ~/.boto file to add the following line: + +boto.sqs_extend = 20070501 + +This will allow the code in the boto.sqs.20070501 module to override the code in boto.sqs. diff --git a/api/boto/sqs/__init__.py b/api/boto/sqs/__init__.py new file mode 100644 index 0000000..0b3924c --- /dev/null +++ b/api/boto/sqs/__init__.py @@ -0,0 +1,46 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto + +boto.check_extensions(__name__, __path__) + +from queue import Queue +from message import Message, MHMessage, EncodedMHMessage +from regioninfo import SQSRegionInfo + +def regions(): + """ + Get all available regions for the SQS service. + + :rtype: list + :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` + """ + return [SQSRegionInfo(name='us-east-1', endpoint='queue.amazonaws.com'), + SQSRegionInfo(name='eu-west-1', endpoint='eu-west-1.queue.amazonaws.com'), + SQSRegionInfo(name='us-west-1', endpoint='us-west-1.queue.amazonaws.com')] + +def connect_to_region(region_name): + for region in regions(): + if region.name == region_name: + return region.connect() + return None diff --git a/api/boto/sqs/attributes.py b/api/boto/sqs/attributes.py new file mode 100644 index 0000000..26c7204 --- /dev/null +++ b/api/boto/sqs/attributes.py @@ -0,0 +1,46 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Attribute Name/Value set +""" + +class Attributes(dict): + + def __init__(self, parent): + self.parent = parent + self.current_key = None + self.current_value = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'Attribute': + self[self.current_key] = self.current_value + elif name == 'Name': + self.current_key = value + elif name == 'Value': + self.current_value = value + else: + setattr(self, name, value) + + diff --git a/api/boto/sqs/connection.py b/api/boto/sqs/connection.py new file mode 100644 index 0000000..fd13d2a --- /dev/null +++ b/api/boto/sqs/connection.py @@ -0,0 +1,257 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.connection import AWSQueryConnection +import xml.sax +from boto.sqs.regioninfo import SQSRegionInfo +from boto.sqs.queue import Queue +from boto.sqs.message import Message +from boto.sqs.attributes import Attributes +from boto import handler +from boto.resultset import ResultSet +from boto.exception import SQSError + +class SQSConnection(AWSQueryConnection): + """ + A Connection to the SQS Service. + """ + DefaultRegionName = 'us-east-1' + DefaultRegionEndpoint = 'queue.amazonaws.com' + APIVersion = '2009-02-01' + SignatureVersion = '2' + DefaultContentType = 'text/plain' + ResponseError = SQSError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/'): + if not region: + region = SQSRegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + self.region.endpoint, debug, https_connection_factory, path) + + def create_queue(self, queue_name, visibility_timeout=None): + """ + Create an SQS Queue. + + :type queue_name: str or unicode + :param queue_name: The name of the new queue. Names are scoped to an account and need to + be unique within that account. Calling this method on an existing + queue name will not return an error from SQS unless the value for + visibility_timeout is different than the value of the existing queue + of that name. This is still an expensive operation, though, and not + the preferred way to check for the existence of a queue. See the + :func:`boto.sqs.connection.SQSConnection.lookup` method. + + :type visibility_timeout: int + :param visibility_timeout: The default visibility timeout for all messages written in the + queue. This can be overridden on a per-message. + + :rtype: :class:`boto.sqs.queue.Queue` + :return: The newly created queue. + + """ + params = {'QueueName': queue_name} + if visibility_timeout: + params['DefaultVisibilityTimeout'] = '%d' % (visibility_timeout,) + return self.get_object('CreateQueue', params, Queue) + + def delete_queue(self, queue, force_deletion=False): + """ + Delete an SQS Queue. + + :type queue: A Queue object + :param queue: The SQS queue to be deleted + + :type force_deletion: Boolean + :param force_deletion: Normally, SQS will not delete a queue that contains messages. + However, if the force_deletion argument is True, the + queue will be deleted regardless of whether there are messages in + the queue or not. USE WITH CAUTION. This will delete all + messages in the queue as well. + + :rtype: bool + :return: True if the command succeeded, False otherwise + """ + return self.get_status('DeleteQueue', None, queue.id) + + def get_queue_attributes(self, queue, attribute='All'): + """ + Gets one or all attributes of a Queue + + :type queue: A Queue object + :param queue: The SQS queue to be deleted + + :type attribute: str + :type attribute: The specific attribute requested. If not supplied, the default + is to return all attributes. Valid attributes are: + ApproximateNumberOfMessages, + ApproximateNumberOfMessagesNotVisible, + VisibilityTimeout, + CreatedTimestamp, + LastModifiedTimestamp, + Policy + + :rtype: :class:`boto.sqs.attributes.Attributes` + :return: An Attributes object containing request value(s). + """ + params = {'AttributeName' : attribute} + return self.get_object('GetQueueAttributes', params, Attributes, queue.id) + + def set_queue_attribute(self, queue, attribute, value): + params = {'Attribute.Name' : attribute, 'Attribute.Value' : value} + return self.get_status('SetQueueAttributes', params, queue.id) + + def receive_message(self, queue, number_messages=1, visibility_timeout=None, + attributes=None): + """ + Read messages from an SQS Queue. + + :type queue: A Queue object + :param queue: The Queue from which messages are read. + + :type number_messages: int + :param number_messages: The maximum number of messages to read (default=1) + + :type visibility_timeout: int + :param visibility_timeout: The number of seconds the message should remain invisible + to other queue readers (default=None which uses the Queues default) + + :type attributes: list of strings + :param attributes: A list of additional attributes that will be returned + with the response. Valid values: + All + SenderId + SentTimestamp + ApproximateReceiveCount + ApproximateFirstReceiveTimestamp + + """ + params = {'MaxNumberOfMessages' : number_messages} + if visibility_timeout: + params['VisibilityTimeout'] = visibility_timeout + if attributes: + self.build_list_params(self, params, attributes, 'AttributeName') + return self.get_list('ReceiveMessage', params, [('Message', queue.message_class)], + queue.id, queue) + + def delete_message(self, queue, message): + params = {'ReceiptHandle' : message.receipt_handle} + return self.get_status('DeleteMessage', params, queue.id) + + def send_message(self, queue, message_content): + params = {'MessageBody' : message_content} + return self.get_object('SendMessage', params, Message, queue.id, verb='POST') + + def change_message_visibility(self, queue, receipt_handle, visibility_timeout): + """ + Extends the read lock timeout for the specified message from the specified queue + to the specified value. + + :type queue: A :class:`boto.sqs.queue.Queue` object + :param queue: The Queue from which messages are read. + + :type receipt_handle: str + :param queue: The receipt handle associated with the message whose + visibility timeout will be changed. + + :type visibility_timeout: int + :param visibility_timeout: The new value of the message's visibility timeout + in seconds. + """ + params = {'ReceiptHandle' : receipt_handle, + 'VisibilityTimeout' : visibility_timeout} + return self.get_status('ChangeMessageVisibility', params, queue.id) + + def get_all_queues(self, prefix=''): + params = {} + if prefix: + params['QueueNamePrefix'] = prefix + return self.get_list('ListQueues', params, [('QueueUrl', Queue)]) + + def get_queue(self, queue_name): + rs = self.get_all_queues(queue_name) + for q in rs: + if q.url.endswith(queue_name): + return q + return None + + lookup = get_queue + + # + # Permissions methods + # + + def add_permission(self, queue, label, aws_account_id, action_name): + """ + Add a permission to a queue. + + :type queue: :class:`boto.sqs.queue.Queue` + :param queue: The queue object + + :type label: str or unicode + :param label: A unique identification of the permission you are setting. + Maximum of 80 characters ``[0-9a-zA-Z_-]`` + Example, AliceSendMessage + + :type aws_account_id: str or unicode + :param principal_id: The AWS account number of the principal who will be given + permission. The principal must have an AWS account, but + does not need to be signed up for Amazon SQS. For information + about locating the AWS account identification. + + :type action_name: str or unicode + :param action_name: The action. Valid choices are: + \*|SendMessage|ReceiveMessage|DeleteMessage| + ChangeMessageVisibility|GetQueueAttributes + + :rtype: bool + :return: True if successful, False otherwise. + + """ + params = {'Label': label, + 'AWSAccountId' : aws_account_id, + 'ActionName' : action_name} + return self.get_status('AddPermission', params, queue.id) + + def remove_permission(self, queue, label): + """ + Remove a permission from a queue. + + :type queue: :class:`boto.sqs.queue.Queue` + :param queue: The queue object + + :type label: str or unicode + :param label: The unique label associated with the permission being removed. + + :rtype: bool + :return: True if successful, False otherwise. + """ + params = {'Label': label} + return self.get_status('RemovePermission', params, queue.id) + + + + + diff --git a/api/boto/sqs/jsonmessage.py b/api/boto/sqs/jsonmessage.py new file mode 100644 index 0000000..ab05a60 --- /dev/null +++ b/api/boto/sqs/jsonmessage.py @@ -0,0 +1,42 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.sqs.message import MHMessage +from boto.exception import SQSDecodeError +import base64 +import simplejson + +class JSONMessage(MHMessage): + """ + Acts like a dictionary but encodes it's data as a Base64 encoded JSON payload. + """ + + def decode(self, value): + try: + value = base64.b64decode(value) + value = simplejson.loads(value) + except: + raise SQSDecodeError('Unable to decode message', self) + return value + + def encode(self, value): + value = simplejson.dumps(value) + return base64.b64encode(value) diff --git a/api/boto/sqs/message.py b/api/boto/sqs/message.py new file mode 100644 index 0000000..da1ba68 --- /dev/null +++ b/api/boto/sqs/message.py @@ -0,0 +1,251 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +SQS Message + +A Message represents the data stored in an SQS queue. The rules for what is allowed within an SQS +Message are here: + + http://docs.amazonwebservices.com/AWSSimpleQueueService/2008-01-01/SQSDeveloperGuide/Query_QuerySendMessage.html + +So, at it's simplest level a Message just needs to allow a developer to store bytes in it and get the bytes +back out. However, to allow messages to have richer semantics, the Message class must support the +following interfaces: + +The constructor for the Message class must accept a keyword parameter "queue" which is an instance of a +boto Queue object and represents the queue that the message will be stored in. The default value for +this parameter is None. + +The constructor for the Message class must accept a keyword parameter "body" which represents the +content or body of the message. The format of this parameter will depend on the behavior of the +particular Message subclass. For example, if the Message subclass provides dictionary-like behavior to the +user the body passed to the constructor should be a dict-like object that can be used to populate +the initial state of the message. + +The Message class must provide an encode method that accepts a value of the same type as the body +parameter of the constructor and returns a string of characters that are able to be stored in an +SQS message body (see rules above). + +The Message class must provide a decode method that accepts a string of characters that can be +stored (and probably were stored!) in an SQS message and return an object of a type that is consistent +with the "body" parameter accepted on the class constructor. + +The Message class must provide a __len__ method that will return the size of the encoded message +that would be stored in SQS based on the current state of the Message object. + +The Message class must provide a get_body method that will return the body of the message in the +same format accepted in the constructor of the class. + +The Message class must provide a set_body method that accepts a message body in the same format +accepted by the constructor of the class. This method should alter to the internal state of the +Message object to reflect the state represented in the message body parameter. + +The Message class must provide a get_body_encoded method that returns the current body of the message +in the format in which it would be stored in SQS. +""" + +import base64 +import StringIO +from boto.sqs.attributes import Attributes +from boto.exception import SQSDecodeError + +class RawMessage: + """ + Base class for SQS messages. RawMessage does not encode the message + in any way. Whatever you store in the body of the message is what + will be written to SQS and whatever is returned from SQS is stored + directly into the body of the message. + """ + + def __init__(self, queue=None, body=''): + self.queue = queue + self.set_body(body) + self.id = None + self.receipt_handle = None + self.md5 = None + self.attributes = Attributes(self) + + def __len__(self): + return len(self.encode(self._body)) + + def startElement(self, name, attrs, connection): + if name == 'Attribute': + return self.attributes + return None + + def endElement(self, name, value, connection): + if name == 'Body': + self.set_body(self.decode(value)) + elif name == 'MessageId': + self.id = value + elif name == 'ReceiptHandle': + self.receipt_handle = value + elif name == 'MD5OfMessageBody': + self.md5 = value + else: + setattr(self, name, value) + + def encode(self, value): + """Transform body object into serialized byte array format.""" + return value + + def decode(self, value): + """Transform seralized byte array into any object.""" + return value + + def set_body(self, body): + """Override the current body for this object, using decoded format.""" + self._body = body + + def get_body(self): + return self._body + + def get_body_encoded(self): + """ + This method is really a semi-private method used by the Queue.write + method when writing the contents of the message to SQS. + You probably shouldn't need to call this method in the normal course of events. + """ + return self.encode(self.get_body()) + + def delete(self): + if self.queue: + return self.queue.delete_message(self) + + def change_visibility(self, visibility_timeout): + if self.queue: + self.queue.connection.change_message_visibility(self.queue, + self.receipt_handle, + visibility_timeout) + +class Message(RawMessage): + """ + The default Message class used for SQS queues. This class automatically + encodes/decodes the message body using Base64 encoding to avoid any + illegal characters in the message body. See: + + http://developer.amazonwebservices.com/connect/thread.jspa?messageID=49680%EC%88%90 + + for details on why this is a good idea. The encode/decode is meant to + be transparent to the end-user. + """ + + def encode(self, value): + return base64.b64encode(value) + + def decode(self, value): + try: + value = base64.b64decode(value) + except: + raise SQSDecodeError('Unable to decode message', self) + return value + +class MHMessage(Message): + """ + The MHMessage class provides a message that provides RFC821-like + headers like this: + + HeaderName: HeaderValue + + The encoding/decoding of this is handled automatically and after + the message body has been read, the message instance can be treated + like a mapping object, i.e. m['HeaderName'] would return 'HeaderValue'. + """ + + def __init__(self, queue=None, body=None, xml_attrs=None): + if body == None or body == '': + body = {} + Message.__init__(self, queue, body) + + def decode(self, value): + try: + msg = {} + fp = StringIO.StringIO(value) + line = fp.readline() + while line: + delim = line.find(':') + key = line[0:delim] + value = line[delim+1:].strip() + msg[key.strip()] = value.strip() + line = fp.readline() + except: + raise SQSDecodeError('Unable to decode message', self) + return msg + + def encode(self, value): + s = '' + for item in value.items(): + s = s + '%s: %s\n' % (item[0], item[1]) + return s + + def __getitem__(self, key): + if self._body.has_key(key): + return self._body[key] + else: + raise KeyError(key) + + def __setitem__(self, key, value): + self._body[key] = value + self.set_body(self._body) + + def keys(self): + return self._body.keys() + + def values(self): + return self._body.values() + + def items(self): + return self._body.items() + + def has_key(self, key): + return self._body.has_key(key) + + def update(self, d): + self._body.update(d) + self.set_body(self._body) + + def get(self, key, default=None): + return self._body.get(key, default) + +class EncodedMHMessage(MHMessage): + """ + The EncodedMHMessage class provides a message that provides RFC821-like + headers like this: + + HeaderName: HeaderValue + + This variation encodes/decodes the body of the message in base64 automatically. + The message instance can be treated like a mapping object, + i.e. m['HeaderName'] would return 'HeaderValue'. + """ + + def decode(self, value): + try: + value = base64.b64decode(value) + except: + raise SQSDecodeError('Unable to decode message', self) + return MHMessage.decode(self, value) + + def encode(self, value): + value = MHMessage.encode(value) + return base64.b64encode(self, value) + diff --git a/api/boto/sqs/queue.py b/api/boto/sqs/queue.py new file mode 100644 index 0000000..48b6115 --- /dev/null +++ b/api/boto/sqs/queue.py @@ -0,0 +1,415 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Queue +""" + +import xml.sax +import urlparse +from boto.exception import SQSError +from boto.handler import XmlHandler +from boto.sqs.message import Message +from boto.resultset import ResultSet + +class Queue: + + def __init__(self, connection=None, url=None, message_class=Message): + self.connection = connection + self.url = url + self.message_class = message_class + self.visibility_timeout = None + + def _id(self): + if self.url: + val = urlparse.urlparse(self.url)[2] + else: + val = self.url + return val + id = property(_id) + + def _name(self): + if self.url: + val = urlparse.urlparse(self.url)[2].split('/')[2] + else: + val = self.url + return val + name = property(_name) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'QueueUrl': + self.url = value + elif name == 'VisibilityTimeout': + self.visibility_timeout = int(value) + else: + setattr(self, name, value) + + def set_message_class(self, message_class): + """ + Set the message class that should be used when instantiating messages read + from the queue. By default, the class boto.sqs.message.Message is used but + this can be overriden with any class that behaves like a message. + + :type message_class: Message-like class + :param message_class: The new Message class + """ + self.message_class = message_class + + def get_attributes(self, attributes='All'): + """ + Retrieves attributes about this queue object and returns + them in an Attribute instance (subclass of a Dictionary). + + :type attributes: string + :param attributes: String containing one of: + ApproximateNumberOfMessages, + ApproximateNumberOfMessagesNotVisible, + VisibilityTimeout, + CreatedTimestamp, + LastModifiedTimestamp, + Policy + :rtype: Attribute object + :return: An Attribute object which is a mapping type holding the + requested name/value pairs + """ + return self.connection.get_queue_attributes(self, attributes) + + def set_attribute(self, attribute, value): + """ + Set a new value for an attribute of the Queue. + + :type attribute: String + :param attribute: The name of the attribute you want to set. The + only valid value at this time is: VisibilityTimeout + :type value: int + :param value: The new value for the attribute. + For VisibilityTimeout the value must be an + integer number of seconds from 0 to 86400. + + :rtype: bool + :return: True if successful, otherwise False. + """ + return self.connection.set_queue_attribute(self, attribute, value) + + def get_timeout(self): + """ + Get the visibility timeout for the queue. + + :rtype: int + :return: The number of seconds as an integer. + """ + a = self.get_attributes('VisibilityTimeout') + return int(a['VisibilityTimeout']) + + def set_timeout(self, visibility_timeout): + """ + Set the visibility timeout for the queue. + + :type visibility_timeout: int + :param visibility_timeout: The desired timeout in seconds + """ + retval = self.set_attribute('VisibilityTimeout', visibility_timeout) + if retval: + self.visibility_timeout = visibility_timeout + return retval + + def add_permission(self, label, aws_account_id, action_name): + """ + Add a permission to a queue. + + :type label: str or unicode + :param label: A unique identification of the permission you are setting. + Maximum of 80 characters ``[0-9a-zA-Z_-]`` + Example, AliceSendMessage + + :type aws_account_id: str or unicode + :param principal_id: The AWS account number of the principal who will be given + permission. The principal must have an AWS account, but + does not need to be signed up for Amazon SQS. For information + about locating the AWS account identification. + + :type action_name: str or unicode + :param action_name: The action. Valid choices are: + \*|SendMessage|ReceiveMessage|DeleteMessage| + ChangeMessageVisibility|GetQueueAttributes + + :rtype: bool + :return: True if successful, False otherwise. + + """ + return self.connection.add_permission(self, label, aws_account_id, action_name) + + def remove_permission(self, label): + """ + Remove a permission from a queue. + + :type label: str or unicode + :param label: The unique label associated with the permission being removed. + + :rtype: bool + :return: True if successful, False otherwise. + """ + return self.connection.remove_permission(self, label) + + def read(self, visibility_timeout=None): + """ + Read a single message from the queue. + + :type visibility_timeout: int + :param visibility_timeout: The timeout for this message in seconds + + :rtype: :class:`boto.sqs.message.Message` + :return: A single message or None if queue is empty + """ + rs = self.get_messages(1, visibility_timeout) + if len(rs) == 1: + return rs[0] + else: + return None + + def write(self, message): + """ + Add a single message to the queue. + + :type message: Message + :param message: The message to be written to the queue + + :rtype: :class:`boto.sqs.message.Message` + :return: The :class:`boto.sqs.message.Message` object that was written. + """ + new_msg = self.connection.send_message(self, message.get_body_encoded()) + message.id = new_msg.id + message.md5 = new_msg.md5 + return message + + def new_message(self, body=''): + """ + Create new message of appropriate class. + + :type body: message body + :param body: The body of the newly created message (optional). + + :rtype: :class:`boto.sqs.message.Message` + :return: A new Message object + """ + m = self.message_class(self, body) + m.queue = self + return m + + # get a variable number of messages, returns a list of messages + def get_messages(self, num_messages=1, visibility_timeout=None, + attributes=None): + """ + Get a variable number of messages. + + :type num_messages: int + :param num_messages: The maximum number of messages to read from the queue. + + :type visibility_timeout: int + :param visibility_timeout: The VisibilityTimeout for the messages read. + + :type attributes: list of strings + :param attributes: A list of additional attributes that will be returned + with the response. Valid values: + All + SenderId + SentTimestamp + ApproximateReceiveCount + ApproximateFirstReceiveTimestamp + :rtype: list + :return: A list of :class:`boto.sqs.message.Message` objects. + """ + return self.connection.receive_message(self, number_messages=num_messages, + visibility_timeout=visibility_timeout, + attributes=attributes) + + def delete_message(self, message): + """ + Delete a message from the queue. + + :type message: :class:`boto.sqs.message.Message` + :param message: The :class:`boto.sqs.message.Message` object to delete. + + :rtype: bool + :return: True if successful, False otherwise + """ + return self.connection.delete_message(self, message) + + def delete(self): + """ + Delete the queue. + """ + return self.connection.delete_queue(self) + + def clear(self, page_size=10, vtimeout=10): + """Utility function to remove all messages from a queue""" + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + self.delete_message(m) + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def count(self, page_size=10, vtimeout=10): + """ + Utility function to count the number of messages in a queue. + Note: This function now calls GetQueueAttributes to obtain + an 'approximate' count of the number of messages in a queue. + """ + a = self.get_attributes('ApproximateNumberOfMessages') + return int(a['ApproximateNumberOfMessages']) + + def count_slow(self, page_size=10, vtimeout=10): + """ + Deprecated. This is the old 'count' method that actually counts + the messages by reading them all. This gives an accurate count but + is very slow for queues with non-trivial number of messasges. + Instead, use get_attribute('ApproximateNumberOfMessages') to take + advantage of the new SQS capability. This is retained only for + the unit tests. + """ + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def dump_(self, file_name, page_size=10, vtimeout=10, sep='\n'): + """Utility function to dump the messages in a queue to a file + NOTE: Page size must be < 10 else SQS errors""" + fp = open(file_name, 'wb') + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + fp.write(m.get_body()) + if sep: + fp.write(sep) + n += 1 + l = self.get_messages(page_size, vtimeout) + fp.close() + return n + + def save_to_file(self, fp, sep='\n'): + """ + Read all messages from the queue and persist them to file-like object. + Messages are written to the file and the 'sep' string is written + in between messages. Messages are deleted from the queue after + being written to the file. + Returns the number of messages saved. + """ + n = 0 + m = self.read() + while m: + n += 1 + fp.write(m.get_body()) + if sep: + fp.write(sep) + self.delete_message(m) + m = self.read() + return n + + def save_to_filename(self, file_name, sep='\n'): + """ + Read all messages from the queue and persist them to local file. + Messages are written to the file and the 'sep' string is written + in between messages. Messages are deleted from the queue after + being written to the file. + Returns the number of messages saved. + """ + fp = open(file_name, 'wb') + n = self.save_to_file(fp, sep) + fp.close() + return n + + # for backwards compatibility + save = save_to_filename + + def save_to_s3(self, bucket): + """ + Read all messages from the queue and persist them to S3. + Messages are stored in the S3 bucket using a naming scheme of:: + + / + + Messages are deleted from the queue after being saved to S3. + Returns the number of messages saved. + """ + n = 0 + m = self.read() + while m: + n += 1 + key = bucket.new_key('%s/%s' % (self.id, m.id)) + key.set_contents_from_string(m.get_body()) + self.delete_message(m) + m = self.read() + return n + + def load_from_s3(self, bucket, prefix=None): + """ + Load messages previously saved to S3. + """ + n = 0 + if prefix: + prefix = '%s/' % prefix + else: + prefix = '%s/' % self.id[1:] + rs = bucket.list(prefix=prefix) + for key in rs: + n += 1 + m = self.new_message(key.get_contents_as_string()) + self.write(m) + return n + + def load_from_file(self, fp, sep='\n'): + """Utility function to load messages from a file-like object to a queue""" + n = 0 + body = '' + l = fp.readline() + while l: + if l == sep: + m = Message(self, body) + self.write(m) + n += 1 + print 'writing message %d' % n + body = '' + else: + body = body + l + l = fp.readline() + return n + + def load_from_filename(self, file_name, sep='\n'): + """Utility function to load messages from a local filename to a queue""" + fp = open(file_name, 'rb') + n = self.load_file_file(fp, sep) + fp.close() + return n + + # for backward compatibility + load = load_from_filename + diff --git a/api/boto/sqs/regioninfo.py b/api/boto/sqs/regioninfo.py new file mode 100644 index 0000000..1d13a40 --- /dev/null +++ b/api/boto/sqs/regioninfo.py @@ -0,0 +1,40 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from boto.ec2.regioninfo import RegionInfo + +class SQSRegionInfo(RegionInfo): + + def connect(self, **kw_params): + """ + Connect to this Region's endpoint. Returns an SQSConnection + object pointing to the endpoint associated with this region. + You may pass any of the arguments accepted by the SQSConnection + object's constructor as keyword arguments and they will be + passed along to the SQSConnection object. + + :rtype: :class:`boto.sqs.connection.SQSConnection` + :return: The connection to this regions endpoint + """ + from boto.sqs.connection import SQSConnection + return SQSConnection(region=self, **kw_params) + diff --git a/api/boto/tests/__init__.py b/api/boto/tests/__init__.py new file mode 100644 index 0000000..449bd16 --- /dev/null +++ b/api/boto/tests/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/api/boto/tests/devpay_s3.py b/api/boto/tests/devpay_s3.py new file mode 100644 index 0000000..bb91125 --- /dev/null +++ b/api/boto/tests/devpay_s3.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the S3Connection +""" + +import time +import os +import urllib + +from boto.s3.connection import S3Connection +from boto.exception import S3PermissionsError + +# this test requires a devpay product and user token to run: + +AMAZON_USER_TOKEN = '{UserToken}...your token here...' +DEVPAY_HEADERS = { 'x-amz-security-token': AMAZON_USER_TOKEN } + +print '--- running S3Connection tests (DevPay) ---' +c = S3Connection() +# create a new, empty bucket +bucket_name = 'test-%d' % int(time.time()) +bucket = c.create_bucket(bucket_name, headers=DEVPAY_HEADERS) +# now try a get_bucket call and see if it's really there +bucket = c.get_bucket(bucket_name, headers=DEVPAY_HEADERS) +# test logging +logging_bucket = c.create_bucket(bucket_name + '-log', headers=DEVPAY_HEADERS) +logging_bucket.set_as_logging_target(headers=DEVPAY_HEADERS) +bucket.enable_logging(target_bucket=logging_bucket, target_prefix=bucket.name, headers=DEVPAY_HEADERS) +bucket.disable_logging(headers=DEVPAY_HEADERS) +c.delete_bucket(logging_bucket, headers=DEVPAY_HEADERS) +# create a new key and store it's content from a string +k = bucket.new_key() +k.name = 'foobar' +s1 = 'This is a test of file upload and download' +s2 = 'This is a second string to test file upload and download' +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +fp = open('foobar', 'wb') +# now get the contents from s3 to a local file +k.get_contents_to_file(fp, headers=DEVPAY_HEADERS) +fp.close() +fp = open('foobar') +# check to make sure content read from s3 is identical to original +assert s1 == fp.read(), 'corrupted file' +fp.close() +# test generated URLs +url = k.generate_url(3600, headers=DEVPAY_HEADERS) +file = urllib.urlopen(url) +assert s1 == file.read(), 'invalid URL %s' % url +url = k.generate_url(3600, force_http=True, headers=DEVPAY_HEADERS) +file = urllib.urlopen(url) +assert s1 == file.read(), 'invalid URL %s' % url +bucket.delete_key(k, headers=DEVPAY_HEADERS) +# test a few variations on get_all_keys - first load some data +# for the first one, let's override the content type +phony_mimetype = 'application/x-boto-test' +headers = {'Content-Type': phony_mimetype} +headers.update(DEVPAY_HEADERS) +k.name = 'foo/bar' +k.set_contents_from_string(s1, headers) +k.name = 'foo/bas' +k.set_contents_from_filename('foobar', headers=DEVPAY_HEADERS) +k.name = 'foo/bat' +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +k.name = 'fie/bar' +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +k.name = 'fie/bas' +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +k.name = 'fie/bat' +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +# try resetting the contents to another value +md5 = k.md5 +k.set_contents_from_string(s2, headers=DEVPAY_HEADERS) +assert k.md5 != md5 +os.unlink('foobar') +all = bucket.get_all_keys(headers=DEVPAY_HEADERS) +assert len(all) == 6 +rs = bucket.get_all_keys(prefix='foo', headers=DEVPAY_HEADERS) +assert len(rs) == 3 +rs = bucket.get_all_keys(prefix='', delimiter='/', headers=DEVPAY_HEADERS) +assert len(rs) == 2 +rs = bucket.get_all_keys(maxkeys=5, headers=DEVPAY_HEADERS) +assert len(rs) == 5 +# test the lookup method +k = bucket.lookup('foo/bar', headers=DEVPAY_HEADERS) +assert isinstance(k, bucket.key_class) +assert k.content_type == phony_mimetype +k = bucket.lookup('notthere', headers=DEVPAY_HEADERS) +assert k == None +# try some metadata stuff +k = bucket.new_key() +k.name = 'has_metadata' +mdkey1 = 'meta1' +mdval1 = 'This is the first metadata value' +k.set_metadata(mdkey1, mdval1) +mdkey2 = 'meta2' +mdval2 = 'This is the second metadata value' +k.set_metadata(mdkey2, mdval2) +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +k = bucket.lookup('has_metadata', headers=DEVPAY_HEADERS) +assert k.get_metadata(mdkey1) == mdval1 +assert k.get_metadata(mdkey2) == mdval2 +k = bucket.new_key() +k.name = 'has_metadata' +k.get_contents_as_string(headers=DEVPAY_HEADERS) +assert k.get_metadata(mdkey1) == mdval1 +assert k.get_metadata(mdkey2) == mdval2 +bucket.delete_key(k, headers=DEVPAY_HEADERS) +# test list and iterator +rs1 = bucket.list(headers=DEVPAY_HEADERS) +num_iter = 0 +for r in rs1: + num_iter = num_iter + 1 +rs = bucket.get_all_keys(headers=DEVPAY_HEADERS) +num_keys = len(rs) +assert num_iter == num_keys +# try a key with a funny character +k = bucket.new_key() +k.name = 'testnewline\n' +k.set_contents_from_string('This is a test', headers=DEVPAY_HEADERS) +rs = bucket.get_all_keys(headers=DEVPAY_HEADERS) +assert len(rs) == num_keys + 1 +bucket.delete_key(k, headers=DEVPAY_HEADERS) +rs = bucket.get_all_keys(headers=DEVPAY_HEADERS) +assert len(rs) == num_keys +# try some acl stuff +bucket.set_acl('public-read', headers=DEVPAY_HEADERS) +policy = bucket.get_acl(headers=DEVPAY_HEADERS) +assert len(policy.acl.grants) == 2 +bucket.set_acl('private', headers=DEVPAY_HEADERS) +policy = bucket.get_acl(headers=DEVPAY_HEADERS) +assert len(policy.acl.grants) == 1 +k = bucket.lookup('foo/bar', headers=DEVPAY_HEADERS) +k.set_acl('public-read', headers=DEVPAY_HEADERS) +policy = k.get_acl(headers=DEVPAY_HEADERS) +assert len(policy.acl.grants) == 2 +k.set_acl('private', headers=DEVPAY_HEADERS) +policy = k.get_acl(headers=DEVPAY_HEADERS) +assert len(policy.acl.grants) == 1 +# try the convenience methods for grants +# this doesn't work with devpay +#bucket.add_user_grant('FULL_CONTROL', +# 'c1e724fbfa0979a4448393c59a8c055011f739b6d102fb37a65f26414653cd67', +# headers=DEVPAY_HEADERS) +try: + bucket.add_email_grant('foobar', 'foo@bar.com', headers=DEVPAY_HEADERS) +except S3PermissionsError: + pass +# now delete all keys in bucket +for k in all: + bucket.delete_key(k, headers=DEVPAY_HEADERS) +# now delete bucket + +c.delete_bucket(bucket, headers=DEVPAY_HEADERS) + +print '--- tests completed ---' diff --git a/api/boto/tests/test.py b/api/boto/tests/test.py new file mode 100755 index 0000000..c6175ca --- /dev/null +++ b/api/boto/tests/test.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +do the unit tests! +""" + +import sys, os, unittest +import getopt, sys +import boto + +from boto.tests.test_sqsconnection import SQSConnectionTest +from boto.tests.test_s3connection import S3ConnectionTest +from boto.tests.test_ec2connection import EC2ConnectionTest +from boto.tests.test_sdbconnection import SDBConnectionTest + +def usage(): + print 'test.py [-t testsuite] [-v verbosity]' + print ' -t run specific testsuite (s3|sqs|ec2|sdb|all)' + print ' -v verbosity (0|1|2)' + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'ht:v:', + ['help', 'testsuite', 'verbosity']) + except: + usage() + sys.exit(2) + testsuite = 'all' + verbosity = 1 + for o, a in opts: + if o in ('-h', '--help'): + usage() + sys.exit() + if o in ('-t', '--testsuite'): + testsuite = a + if o in ('-v', '--verbosity'): + verbosity = int(a) + if len(args) != 0: + usage() + sys.exit() + suite = unittest.TestSuite() + if testsuite == 'all': + suite.addTest(unittest.makeSuite(SQSConnectionTest)) + suite.addTest(unittest.makeSuite(S3ConnectionTest)) + suite.addTest(unittest.makeSuite(EC2ConnectionTest)) + suite.addTest(unittest.makeSuite(SDBConnectionTest)) + elif testsuite == 's3': + suite.addTest(unittest.makeSuite(S3ConnectionTest)) + elif testsuite == 'sqs': + suite.addTest(unittest.makeSuite(SQSConnectionTest)) + elif testsuite == 'ec2': + suite.addTest(unittest.makeSuite(EC2ConnectionTest)) + elif testsuite == 'sdb': + suite.addTest(unittest.makeSuite(SDBConnectionTest)) + else: + usage() + sys.exit() + unittest.TextTestRunner(verbosity=verbosity).run(suite) + +if __name__ == "__main__": + main() diff --git a/api/boto/tests/test_ec2connection.py b/api/boto/tests/test_ec2connection.py new file mode 100644 index 0000000..8f1fb59 --- /dev/null +++ b/api/boto/tests/test_ec2connection.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the EC2Connection +""" + +import unittest +import time +import os +from boto.ec2.connection import EC2Connection +import telnetlib +import socket + +class EC2ConnectionTest (unittest.TestCase): + + def test_1_basic(self): + # this is my user_id, if you want to run these tests you should + # replace this with yours or they won't work + user_id = '084307701560' + print '--- running EC2Connection tests ---' + c = EC2Connection() + # get list of private AMI's + rs = c.get_all_images(owners=[user_id]) + assert len(rs) > 0 + # now pick the first one + image = rs[0] + # temporarily make this image runnable by everyone + status = image.set_launch_permissions(group_names=['all']) + assert status + d = image.get_launch_permissions() + assert d.has_key('groups') + assert len(d['groups']) > 0 + # now remove that permission + status = image.remove_launch_permissions(group_names=['all']) + assert status + d = image.get_launch_permissions() + assert not d.has_key('groups') + + # create a new security group + group_name = 'test-%d' % int(time.time()) + group_desc = 'This is a security group created during unit testing' + group = c.create_security_group(group_name, group_desc) + # now get a listing of all security groups and look for our new one + rs = c.get_all_security_groups() + found = False + for g in rs: + if g.name == group_name: + found = True + assert found + # now pass arg to filter results to only our new group + rs = c.get_all_security_groups([group_name]) + assert len(rs) == 1 + group = rs[0] + # + # now delete the security group + status = c.delete_security_group(group_name) + # now make sure it's really gone + rs = c.get_all_security_groups() + found = False + for g in rs: + if g.name == group_name: + found = True + assert not found + # now create it again for use with the instance test + group = c.create_security_group(group_name, group_desc) + + # now try to launch apache image with our new security group + rs = c.get_all_images() + img_loc = 'ec2-public-images/fedora-core4-apache.manifest.xml' + for image in rs: + if image.location == img_loc: + break + reservation = image.run(security_groups=[group.name]) + instance = reservation.instances[0] + while instance.state != 'running': + print '\tinstance is %s' % instance.state + time.sleep(30) + instance.update() + # instance in now running, try to telnet to port 80 + t = telnetlib.Telnet() + try: + t.open(instance.dns_name, 80) + except socket.error: + pass + # now open up port 80 and try again, it should work + group.authorize('tcp', 80, 80, '0.0.0.0/0') + t.open(instance.dns_name, 80) + t.close() + # now revoke authorization and try again + group.revoke('tcp', 80, 80, '0.0.0.0/0') + try: + t.open(instance.dns_name, 80) + except socket.error: + pass + # now kill the instance and delete the security group + instance.stop() + # unfortunately, I can't delete the sg within this script + #sg.delete() + + # create a new key pair + key_name = 'test-%d' % int(time.time()) + status = c.create_key_pair(key_name) + assert status + # now get a listing of all key pairs and look for our new one + rs = c.get_all_key_pairs() + found = False + for k in rs: + if k.name == key_name: + found = True + assert found + # now pass arg to filter results to only our new key pair + rs = c.get_all_key_pairs([key_name]) + assert len(rs) == 1 + key_pair = rs[0] + # now delete the key pair + status = c.delete_key_pair(key_name) + # now make sure it's really gone + rs = c.get_all_key_pairs() + found = False + for k in rs: + if k.name == key_name: + found = True + assert not found + + # short test around Paid AMI capability + demo_paid_ami_id = 'ami-bd9d78d4' + demo_paid_ami_product_code = 'A79EC0DB' + l = c.get_all_images([demo_paid_ami_id]) + assert len(l) == 1 + assert len(l[0].product_codes) == 1 + assert l[0].product_codes[0] == demo_paid_ami_product_code + + print '--- tests completed ---' diff --git a/api/boto/tests/test_s3connection.py b/api/boto/tests/test_s3connection.py new file mode 100644 index 0000000..7afc8d2 --- /dev/null +++ b/api/boto/tests/test_s3connection.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the S3Connection +""" + +import unittest +import time +import os +import urllib +from boto.s3.connection import S3Connection +from boto.exception import S3PermissionsError + +class S3ConnectionTest (unittest.TestCase): + + def test_1_basic(self): + print '--- running S3Connection tests ---' + c = S3Connection() + # create a new, empty bucket + bucket_name = 'test-%d' % int(time.time()) + bucket = c.create_bucket(bucket_name) + # now try a get_bucket call and see if it's really there + bucket = c.get_bucket(bucket_name) + # test logging + logging_bucket = c.create_bucket(bucket_name + '-log') + logging_bucket.set_as_logging_target() + bucket.enable_logging(target_bucket=logging_bucket, target_prefix=bucket.name) + bucket.disable_logging() + c.delete_bucket(logging_bucket) + k = bucket.new_key() + k.name = 'foobar' + s1 = 'This is a test of file upload and download' + s2 = 'This is a second string to test file upload and download' + k.set_contents_from_string(s1) + fp = open('foobar', 'wb') + # now get the contents from s3 to a local file + k.get_contents_to_file(fp) + fp.close() + fp = open('foobar') + # check to make sure content read from s3 is identical to original + assert s1 == fp.read(), 'corrupted file' + fp.close() + # test generated URLs + url = k.generate_url(3600) + file = urllib.urlopen(url) + assert s1 == file.read(), 'invalid URL %s' % url + url = k.generate_url(3600, force_http=True) + file = urllib.urlopen(url) + assert s1 == file.read(), 'invalid URL %s' % url + bucket.delete_key(k) + # test a few variations on get_all_keys - first load some data + # for the first one, let's override the content type + phony_mimetype = 'application/x-boto-test' + headers = {'Content-Type': phony_mimetype} + k.name = 'foo/bar' + k.set_contents_from_string(s1, headers) + k.name = 'foo/bas' + k.set_contents_from_filename('foobar') + k.name = 'foo/bat' + k.set_contents_from_string(s1) + k.name = 'fie/bar' + k.set_contents_from_string(s1) + k.name = 'fie/bas' + k.set_contents_from_string(s1) + k.name = 'fie/bat' + k.set_contents_from_string(s1) + # try resetting the contents to another value + md5 = k.md5 + k.set_contents_from_string(s2) + assert k.md5 != md5 + os.unlink('foobar') + all = bucket.get_all_keys() + assert len(all) == 6 + rs = bucket.get_all_keys(prefix='foo') + assert len(rs) == 3 + rs = bucket.get_all_keys(prefix='', delimiter='/') + assert len(rs) == 2 + rs = bucket.get_all_keys(maxkeys=5) + assert len(rs) == 5 + # test the lookup method + k = bucket.lookup('foo/bar') + assert isinstance(k, bucket.key_class) + assert k.content_type == phony_mimetype + k = bucket.lookup('notthere') + assert k == None + # try some metadata stuff + k = bucket.new_key() + k.name = 'has_metadata' + mdkey1 = 'meta1' + mdval1 = 'This is the first metadata value' + k.set_metadata(mdkey1, mdval1) + mdkey2 = 'meta2' + mdval2 = 'This is the second metadata value' + k.set_metadata(mdkey2, mdval2) + k.set_contents_from_string(s1) + k = bucket.lookup('has_metadata') + assert k.get_metadata(mdkey1) == mdval1 + assert k.get_metadata(mdkey2) == mdval2 + k = bucket.new_key() + k.name = 'has_metadata' + k.get_contents_as_string() + assert k.get_metadata(mdkey1) == mdval1 + assert k.get_metadata(mdkey2) == mdval2 + bucket.delete_key(k) + # test list and iterator + rs1 = bucket.list() + num_iter = 0 + for r in rs1: + num_iter = num_iter + 1 + rs = bucket.get_all_keys() + num_keys = len(rs) + assert num_iter == num_keys + # try a key with a funny character + k = bucket.new_key() + k.name = 'testnewline\n' + k.set_contents_from_string('This is a test') + rs = bucket.get_all_keys() + assert len(rs) == num_keys + 1 + bucket.delete_key(k) + rs = bucket.get_all_keys() + assert len(rs) == num_keys + # try some acl stuff + bucket.set_acl('public-read') + policy = bucket.get_acl() + assert len(policy.acl.grants) == 2 + bucket.set_acl('private') + policy = bucket.get_acl() + assert len(policy.acl.grants) == 1 + k = bucket.lookup('foo/bar') + k.set_acl('public-read') + policy = k.get_acl() + assert len(policy.acl.grants) == 2 + k.set_acl('private') + policy = k.get_acl() + assert len(policy.acl.grants) == 1 + # try the convenience methods for grants + bucket.add_user_grant('FULL_CONTROL', + 'c1e724fbfa0979a4448393c59a8c055011f739b6d102fb37a65f26414653cd67') + try: + bucket.add_email_grant('foobar', 'foo@bar.com') + except S3PermissionsError: + pass + # now delete all keys in bucket + for k in all: + bucket.delete_key(k) + # now delete bucket + c.delete_bucket(bucket) + print '--- tests completed ---' diff --git a/api/boto/tests/test_sdbconnection.py b/api/boto/tests/test_sdbconnection.py new file mode 100644 index 0000000..c2bb74e --- /dev/null +++ b/api/boto/tests/test_sdbconnection.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the SDBConnection +""" + +import unittest +import time +from boto.sdb.connection import SDBConnection +from boto.exception import SDBResponseError + +class SDBConnectionTest (unittest.TestCase): + + def test_1_basic(self): + print '--- running SDBConnection tests ---' + c = SDBConnection() + rs = c.get_all_domains() + num_domains = len(rs) + + # try illegal name + try: + domain = c.create_domain('bad:domain:name') + except SDBResponseError: + pass + + # now create one that should work and should be unique (i.e. a new one) + domain_name = 'test%d' % int(time.time()) + domain = c.create_domain(domain_name) + rs = c.get_all_domains() + assert len(rs) == num_domains+1 + + # now let's a couple of items and attributes + item_1 = 'item1' + same_value = 'same_value' + attrs_1 = {'name1' : same_value, 'name2' : 'diff_value_1'} + domain.put_attributes(item_1, attrs_1) + item_2 = 'item2' + attrs_2 = {'name1' : same_value, 'name2' : 'diff_value_2'} + domain.put_attributes(item_2, attrs_2) + time.sleep(10) + + # try to get the attributes and see if they match + item = domain.get_attributes(item_1) + assert len(item.keys()) == len(attrs_1.keys()) + assert item['name1'] == attrs_1['name1'] + assert item['name2'] == attrs_1['name2'] + + # try a search or two + rs = domain.query("['name1'='%s']" % same_value) + n = 0 + for item in rs: + n += 1 + assert n == 2 + rs = domain.query("['name2'='diff_value_2']") + n = 0 + for item in rs: + n += 1 + assert n == 1 + + # delete all attributes associated with item_1 + stat = domain.delete_attributes(item_1) + assert stat + + # now try a batch put operation on the domain + item3 = {'name3_1' : 'value3_1', + 'name3_2' : 'value3_2', + 'name3_3' : ['value3_3_1', 'value3_3_2']} + + item4 = {'name4_1' : 'value4_1', + 'name4_2' : ['value4_2_1', 'value4_2_2'], + 'name4_3' : 'value4_3'} + items = {'item3' : item3, 'item4' : item4} + domain.batch_put_attributes(items) + time.sleep(10) + item = domain.get_attributes('item3') + assert item['name3_2'] == 'value3_2' + + # now delete the domain + stat = c.delete_domain(domain) + assert stat + + print '--- tests completed ---' + diff --git a/api/boto/tests/test_sqsconnection.py b/api/boto/tests/test_sqsconnection.py new file mode 100644 index 0000000..f24ad32 --- /dev/null +++ b/api/boto/tests/test_sqsconnection.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the SQSConnection +""" + +import unittest +import time +from boto.sqs.connection import SQSConnection +from boto.sqs.message import MHMessage +from boto.exception import SQSError + +class SQSConnectionTest (unittest.TestCase): + + def test_1_basic(self): + print '--- running SQSConnection tests ---' + c = SQSConnection() + rs = c.get_all_queues() + num_queues = 0 + for q in rs: + num_queues += 1 + + # try illegal name + try: + queue = c.create_queue('bad_queue_name') + except SQSError: + pass + + # now create one that should work and should be unique (i.e. a new one) + queue_name = 'test%d' % int(time.time()) + timeout = 60 + queue = c.create_queue(queue_name, timeout) + time.sleep(60) + rs = c.get_all_queues() + i = 0 + for q in rs: + i += 1 + assert i == num_queues+1 + assert queue.count_slow() == 0 + + # check the visibility timeout + t = queue.get_timeout() + assert t == timeout, '%d != %d' % (t, timeout) + + # now try to get queue attributes + a = q.get_attributes() + assert a.has_key('ApproximateNumberOfMessages') + assert a.has_key('VisibilityTimeout') + a = q.get_attributes('ApproximateNumberOfMessages') + assert a.has_key('ApproximateNumberOfMessages') + assert not a.has_key('VisibilityTimeout') + a = q.get_attributes('VisibilityTimeout') + assert not a.has_key('ApproximateNumberOfMessages') + assert a.has_key('VisibilityTimeout') + + # now change the visibility timeout + timeout = 45 + queue.set_timeout(timeout) + time.sleep(60) + t = queue.get_timeout() + assert t == timeout, '%d != %d' % (t, timeout) + + # now add a message + message_body = 'This is a test\n' + message = queue.new_message(message_body) + queue.write(message) + time.sleep(30) + assert queue.count_slow() == 1 + time.sleep(30) + + # now read the message from the queue with a 10 second timeout + message = queue.read(visibility_timeout=10) + assert message + assert message.get_body() == message_body + + # now immediately try another read, shouldn't find anything + message = queue.read() + assert message == None + + # now wait 10 seconds and try again + time.sleep(10) + message = queue.read() + assert message + + if c.APIVersion == '2007-05-01': + # now terminate the visibility timeout for this message + message.change_visibility(0) + # now see if we can read it in the queue + message = queue.read() + assert message + + # now delete the message + queue.delete_message(message) + time.sleep(30) + assert queue.count_slow() == 0 + + # create another queue so we can test force deletion + # we will also test MHMessage with this queue + queue_name = 'test%d' % int(time.time()) + timeout = 60 + queue = c.create_queue(queue_name, timeout) + queue.set_message_class(MHMessage) + time.sleep(30) + + # now add a couple of messages + message = queue.new_message() + message['foo'] = 'bar' + queue.write(message) + message_body = {'fie' : 'baz', 'foo' : 'bar'} + message = queue.new_message(body=message_body) + queue.write(message) + time.sleep(30) + + m = queue.read() + assert m['foo'] == 'bar' + + # now delete that queue and messages + c.delete_queue(queue, True) + + print '--- tests completed ---' + diff --git a/api/boto/utils.py b/api/boto/utils.py new file mode 100644 index 0000000..db16d30 --- /dev/null +++ b/api/boto/utils.py @@ -0,0 +1,560 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# +# Parts of this code were copied or derived from sample code supplied by AWS. +# The following notice applies to that code. +# +# This software code is made available "AS IS" without warranties of any +# kind. You may copy, display, modify and redistribute the software +# code either by itself or as incorporated into your code; provided that +# you do not remove any proprietary notices. Your use of this software +# code is at your own risk and you waive any claim against Amazon +# Digital Services, Inc. or its affiliates with respect to your use of +# this software code. (c) 2006 Amazon Digital Services, Inc. or its +# affiliates. + +""" +Some handy utility functions used by several classes. +""" + +import base64 +import hmac +import re +import urllib, urllib2 +import imp +import subprocess, os, StringIO +import time, datetime +import logging.handlers +import boto +import tempfile +import smtplib +import datetime +from email.MIMEMultipart import MIMEMultipart +from email.MIMEBase import MIMEBase +from email.MIMEText import MIMEText +from email.Utils import formatdate +from email import Encoders + +try: + import hashlib + _hashfn = hashlib.sha512 +except ImportError: + import md5 + _hashfn = md5.md5 + +METADATA_PREFIX = 'x-amz-meta-' +AMAZON_HEADER_PREFIX = 'x-amz-' + +# generates the aws canonical string for the given parameters +def canonical_string(method, path, headers, expires=None): + interesting_headers = {} + for key in headers: + lk = key.lower() + if lk in ['content-md5', 'content-type', 'date'] or lk.startswith(AMAZON_HEADER_PREFIX): + interesting_headers[lk] = headers[key].strip() + + # these keys get empty strings if they don't exist + if not interesting_headers.has_key('content-type'): + interesting_headers['content-type'] = '' + if not interesting_headers.has_key('content-md5'): + interesting_headers['content-md5'] = '' + + # just in case someone used this. it's not necessary in this lib. + if interesting_headers.has_key('x-amz-date'): + interesting_headers['date'] = '' + + # if you're using expires for query string auth, then it trumps date + # (and x-amz-date) + if expires: + interesting_headers['date'] = str(expires) + + sorted_header_keys = interesting_headers.keys() + sorted_header_keys.sort() + + buf = "%s\n" % method + for key in sorted_header_keys: + if key.startswith(AMAZON_HEADER_PREFIX): + buf += "%s:%s\n" % (key, interesting_headers[key]) + else: + buf += "%s\n" % interesting_headers[key] + + # don't include anything after the first ? in the resource... + buf += "%s" % path.split('?')[0] + + # ...unless there is an acl or torrent parameter + if re.search("[&?]acl($|=|&)", path): + buf += "?acl" + elif re.search("[&?]logging($|=|&)", path): + buf += "?logging" + elif re.search("[&?]torrent($|=|&)", path): + buf += "?torrent" + elif re.search("[&?]location($|=|&)", path): + buf += "?location" + elif re.search("[&?]requestPayment($|=|&)", path): + buf += "?requestPayment" + + return buf + +def merge_meta(headers, metadata): + final_headers = headers.copy() + for k in metadata.keys(): + if k.lower() in ['cache-control', 'content-md5', 'content-type', + 'content-encoding', 'content-disposition', + 'date', 'expires']: + final_headers[k] = metadata[k] + else: + final_headers[METADATA_PREFIX + k] = metadata[k] + + return final_headers + +def get_aws_metadata(headers): + metadata = {} + for hkey in headers.keys(): + if hkey.lower().startswith(METADATA_PREFIX): + metadata[hkey[len(METADATA_PREFIX):]] = headers[hkey] + del headers[hkey] + return metadata + +def retry_url(url, retry_on_404=True): + for i in range(0, 10): + try: + req = urllib2.Request(url) + resp = urllib2.urlopen(req) + return resp.read() + except urllib2.HTTPError, e: + # in 2.6 you use getcode(), in 2.5 and earlier you use code + if hasattr(e, 'getcode'): + code = e.getcode() + else: + code = e.code + if code == 404 and not retry_on_404: + return '' + except: + pass + boto.log.exception('Caught exception reading instance data') + time.sleep(2**i) + boto.log.error('Unable to read instance data, giving up') + return '' + +def _get_instance_metadata(url): + d = {} + data = retry_url(url) + if data: + fields = data.split('\n') + for field in fields: + if field.endswith('/'): + d[field[0:-1]] = _get_instance_metadata(url + field) + else: + p = field.find('=') + if p > 0: + key = field[p+1:] + resource = field[0:p] + '/openssh-key' + else: + key = resource = field + val = retry_url(url + resource) + p = val.find('\n') + if p > 0: + val = val.split('\n') + d[key] = val + return d + +def get_instance_metadata(version='latest'): + """ + Returns the instance metadata as a nested Python dictionary. + Simple values (e.g. local_hostname, hostname, etc.) will be + stored as string values. Values such as ancestor-ami-ids will + be stored in the dict as a list of string values. More complex + fields such as public-keys and will be stored as nested dicts. + """ + url = 'http://169.254.169.254/%s/meta-data/' % version + return _get_instance_metadata(url) + +def get_instance_userdata(version='latest', sep=None): + url = 'http://169.254.169.254/%s/user-data' % version + user_data = retry_url(url, retry_on_404=False) + if user_data: + if sep: + l = user_data.split(sep) + user_data = {} + for nvpair in l: + t = nvpair.split('=') + user_data[t[0].strip()] = t[1].strip() + return user_data + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +def get_ts(ts=None): + if not ts: + ts = time.gmtime() + return time.strftime(ISO8601, ts) + +def parse_ts(ts): + return datetime.datetime.strptime(ts, ISO8601) + +def find_class(module_name, class_name=None): + if class_name: + module_name = "%s.%s" % (module_name, class_name) + modules = module_name.split('.') + path = None + c = None + + try: + for m in modules[1:]: + if c: + c = getattr(c, m) + else: + c = getattr(__import__(".".join(modules[0:-1])), m) + return c + except: + return None + +def update_dme(username, password, dme_id, ip_address): + """ + Update your Dynamic DNS record with DNSMadeEasy.com + """ + dme_url = 'https://www.dnsmadeeasy.com/servlet/updateip' + dme_url += '?username=%s&password=%s&id=%s&ip=%s' + s = urllib2.urlopen(dme_url % (username, password, dme_id, ip_address)) + return s.read() + +def fetch_file(uri, file=None, username=None, password=None): + """ + Fetch a file based on the URI provided. If you do not pass in a file pointer + a tempfile.NamedTemporaryFile, or None if the file could not be + retrieved is returned. + The URI can be either an HTTP url, or "s3://bucket_name/key_name" + """ + boto.log.info('Fetching %s' % uri) + if file == None: + file = tempfile.NamedTemporaryFile() + try: + working_dir = boto.config.get("General", "working_dir") + if uri.startswith('s3://'): + bucket_name, key_name = uri[len('s3://'):].split('/', 1) + c = boto.connect_s3() + bucket = c.get_bucket(bucket_name) + key = bucket.get_key(key_name) + key.get_contents_to_file(file) + else: + if username and password: + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + passman.add_password(None, uri, username, password) + authhandler = urllib2.HTTPBasicAuthHandler(passman) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) + s = urllib2.urlopen(uri) + file.write(s.read()) + file.seek(0) + except: + raise + boto.log.exception('Problem Retrieving file: %s' % uri) + file = None + return file + +class ShellCommand(object): + + def __init__(self, command, wait=True): + self.exit_code = 0 + self.command = command + self.log_fp = StringIO.StringIO() + self.wait = wait + self.run() + + def run(self): + boto.log.info('running:%s' % self.command) + self.process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if(self.wait): + while self.process.poll() == None: + time.sleep(1) + t = self.process.communicate() + self.log_fp.write(t[0]) + self.log_fp.write(t[1]) + boto.log.info(self.log_fp.getvalue()) + self.exit_code = self.process.returncode + return self.exit_code + + def setReadOnly(self, value): + raise AttributeError + + def getStatus(self): + return self.exit_code + + status = property(getStatus, setReadOnly, None, 'The exit code for the command') + + def getOutput(self): + return self.log_fp.getvalue() + + output = property(getOutput, setReadOnly, None, 'The STDIN and STDERR output of the command') + +class AuthSMTPHandler(logging.handlers.SMTPHandler): + """ + This class extends the SMTPHandler in the standard Python logging module + to accept a username and password on the constructor and to then use those + credentials to authenticate with the SMTP server. To use this, you could + add something like this in your boto config file: + + [handler_hand07] + class=boto.utils.AuthSMTPHandler + level=WARN + formatter=form07 + args=('localhost', 'username', 'password', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject') + """ + + def __init__(self, mailhost, username, password, fromaddr, toaddrs, subject): + """ + Initialize the handler. + + We have extended the constructor to accept a username/password + for SMTP authentication. + """ + logging.handlers.SMTPHandler.__init__(self, mailhost, fromaddr, toaddrs, subject) + self.username = username + self.password = password + + def emit(self, record): + """ + Emit a record. + + Format the record and send it to the specified addressees. + It would be really nice if I could add authorization to this class + without having to resort to cut and paste inheritance but, no. + """ + try: + import smtplib + try: + from email.Utils import formatdate + except: + formatdate = self.date_time + port = self.mailport + if not port: + port = smtplib.SMTP_PORT + smtp = smtplib.SMTP(self.mailhost, port) + smtp.login(self.username, self.password) + msg = self.format(record) + msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % ( + self.fromaddr, + string.join(self.toaddrs, ","), + self.getSubject(record), + formatdate(), msg) + smtp.sendmail(self.fromaddr, self.toaddrs, msg) + smtp.quit() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handleError(record) + +class LRUCache(dict): + """A dictionary-like object that stores only a certain number of items, and + discards its least recently used item when full. + + >>> cache = LRUCache(3) + >>> cache['A'] = 0 + >>> cache['B'] = 1 + >>> cache['C'] = 2 + >>> len(cache) + 3 + + >>> cache['A'] + 0 + + Adding new items to the cache does not increase its size. Instead, the least + recently used item is dropped: + + >>> cache['D'] = 3 + >>> len(cache) + 3 + >>> 'B' in cache + False + + Iterating over the cache returns the keys, starting with the most recently + used: + + >>> for key in cache: + ... print key + D + A + C + + This code is based on the LRUCache class from Genshi which is based on + Mighty's LRUCache from ``myghtyutils.util``, written + by Mike Bayer and released under the MIT license (Genshi uses the + BSD License). See: + + http://svn.myghty.org/myghtyutils/trunk/lib/myghtyutils/util.py + """ + + class _Item(object): + def __init__(self, key, value): + self.previous = self.next = None + self.key = key + self.value = value + def __repr__(self): + return repr(self.value) + + def __init__(self, capacity): + self._dict = dict() + self.capacity = capacity + self.head = None + self.tail = None + + def __contains__(self, key): + return key in self._dict + + def __iter__(self): + cur = self.head + while cur: + yield cur.key + cur = cur.next + + def __len__(self): + return len(self._dict) + + def __getitem__(self, key): + item = self._dict[key] + self._update_item(item) + return item.value + + def __setitem__(self, key, value): + item = self._dict.get(key) + if item is None: + item = self._Item(key, value) + self._dict[key] = item + self._insert_item(item) + else: + item.value = value + self._update_item(item) + self._manage_size() + + def __repr__(self): + return repr(self._dict) + + def _insert_item(self, item): + item.previous = None + item.next = self.head + if self.head is not None: + self.head.previous = item + else: + self.tail = item + self.head = item + self._manage_size() + + def _manage_size(self): + while len(self._dict) > self.capacity: + olditem = self._dict[self.tail.key] + del self._dict[self.tail.key] + if self.tail != self.head: + self.tail = self.tail.previous + self.tail.next = None + else: + self.head = self.tail = None + + def _update_item(self, item): + if self.head == item: + return + + previous = item.previous + previous.next = item.next + if item.next is not None: + item.next.previous = previous + else: + self.tail = previous + + item.previous = None + item.next = self.head + self.head.previous = self.head = item + +class Password(object): + """ + Password object that stores itself as SHA512 hashed. + """ + def __init__(self, str=None): + """ + Load the string from an initial value, this should be the raw SHA512 hashed password + """ + self.str = str + + def set(self, value): + self.str = _hashfn(value).hexdigest() + + def __str__(self): + return str(self.str) + + def __eq__(self, other): + if other == None: + return False + return str(_hashfn(other).hexdigest()) == str(self.str) + + def __len__(self): + if self.str: + return len(self.str) + else: + return 0 + +def notify(subject, body=None, html_body=None, to_string=None, attachments=[], append_instance_id=True): + if append_instance_id: + subject = "[%s] %s" % (boto.config.get_value("Instance", "instance-id"), subject) + if not to_string: + to_string = boto.config.get_value('Notification', 'smtp_to', None) + if to_string: + try: + from_string = boto.config.get_value('Notification', 'smtp_from', 'boto') + msg = MIMEMultipart() + msg['From'] = from_string + msg['To'] = to_string + msg['Date'] = formatdate(localtime=True) + msg['Subject'] = subject + + if body: + msg.attach(MIMEText(body)) + + if html_body: + part = MIMEBase('text', 'html') + part.set_payload(html_body) + Encoders.encode_base64(part) + msg.attach(part) + + for part in attachments: + msg.attach(part) + + smtp_host = boto.config.get_value('Notification', 'smtp_host', 'localhost') + + # Alternate port support + if boto.config.get_value("Notification", "smtp_port"): + server = smtplib.SMTP(smtp_host, int(boto.config.get_value("Notification", "smtp_port"))) + else: + server = smtplib.SMTP(smtp_host) + + # TLS support + if boto.config.getbool("Notification", "smtp_tls"): + server.ehlo() + server.starttls() + server.ehlo() + smtp_user = boto.config.get_value('Notification', 'smtp_user', '') + smtp_pass = boto.config.get_value('Notification', 'smtp_pass', '') + if smtp_user: + server.login(smtp_user, smtp_pass) + server.sendmail(from_string, to_string, msg.as_string()) + server.quit() + except: + boto.log.exception('notify failed') + diff --git a/api/boto/vpc/__init__.py b/api/boto/vpc/__init__.py new file mode 100644 index 0000000..80b0073 --- /dev/null +++ b/api/boto/vpc/__init__.py @@ -0,0 +1,478 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a connection to the EC2 service. +""" + +import urllib +import base64 +import boto +from boto import config +from boto.ec2.connection import EC2Connection +from boto.resultset import ResultSet +from boto.vpc.vpc import VPC +from boto.vpc.customergateway import CustomerGateway +from boto.vpc.vpngateway import VpnGateway, Attachment +from boto.vpc.dhcpoptions import DhcpOptions +from boto.vpc.subnet import Subnet +from boto.vpc.vpnconnection import VpnConnection + +class VPCConnection(EC2Connection): + + # VPC methods + + def get_all_vpcs(self, vpc_ids=None, filters=None): + """ + Retrieve information about your VPCs. You can filter results to + return information only about those VPCs that match your search + parameters. Otherwise, all VPCs associated with your account + are returned. + + :type vpc_ids: list + :param vpc_ids: A list of strings with the desired VPC ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the VPC (pending or available) + - *cidrBlock*, CIDR block of the VPC + - *dhcpOptionsId*, the ID of a set of DHCP options + + :rtype: list + :return: A list of :class:`boto.vpc.vpc.VPC` + """ + params = {} + if vpc_ids: + self.build_list_params(params, vpc_ids, 'VpcId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeVpcs', params, [('item', VPC)]) + + def create_vpc(self, cidr_block): + """ + Create a new Virtual Private Cloud. + + :type cidr_block: str + :param cidr_block: A valid CIDR block + + :rtype: The newly created VPC + :return: A :class:`boto.vpc.vpc.VPC` object + """ + params = {'CidrBlock' : cidr_block} + return self.get_object('CreateVpc', params, VPC) + + def delete_vpc(self, vpc_id): + """ + Delete a Virtual Private Cloud. + + :type vpc_id: str + :param vpc_id: The ID of the vpc to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'VpcId': vpc_id} + return self.get_status('DeleteVpc', params) + + # Customer Gateways + + def get_all_customer_gateways(self, customer_gateway_ids=None, filters=None): + """ + Retrieve information about your CustomerGateways. You can filter results to + return information only about those CustomerGateways that match your search + parameters. Otherwise, all CustomerGateways associated with your account + are returned. + + :type customer_gateway_ids: list + :param customer_gateway_ids: A list of strings with the desired CustomerGateway ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the CustomerGateway + (pending,available,deleting,deleted) + - *type*, the type of customer gateway (ipsec.1) + - *ipAddress* the IP address of customer gateway's + internet-routable external inteface + + :rtype: list + :return: A list of :class:`boto.vpc.customergateway.CustomerGateway` + """ + params = {} + if customer_gateway_ids: + self.build_list_params(params, customer_gateway_ids, 'CustomerGatewayId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeCustomerGateways', params, [('item', CustomerGateway)]) + + def create_customer_gateway(self, type, ip_address, bgp_asn): + """ + Create a new Customer Gateway + + :type type: str + :param type: Type of VPN Connection. Only valid valid currently is 'ipsec.1' + + :type ip_address: str + :param ip_address: Internet-routable IP address for customer's gateway. + Must be a static address. + + :type bgp_asn: str + :param bgp_asn: Customer gateway's Border Gateway Protocol (BGP) + Autonomous System Number (ASN) + + :rtype: The newly created CustomerGateway + :return: A :class:`boto.vpc.customergateway.CustomerGateway` object + """ + params = {'Type' : type, + 'IpAddress' : ip_address, + 'BgpAsn' : bgp_asn} + return self.get_object('CreateCustomerGateway', params, CustomerGateway) + + def delete_customer_gateway(self, customer_gateway_id): + """ + Delete a Customer Gateway. + + :type customer_gateway_id: str + :param customer_gateway_id: The ID of the customer_gateway to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'CustomerGatewayId': customer_gateway_id} + return self.get_status('DeleteCustomerGateway', params) + + # VPN Gateways + + def get_all_vpn_gateways(self, vpn_gateway_ids=None, filters=None): + """ + Retrieve information about your VpnGateways. You can filter results to + return information only about those VpnGateways that match your search + parameters. Otherwise, all VpnGateways associated with your account + are returned. + + :type vpn_gateway_ids: list + :param vpn_gateway_ids: A list of strings with the desired VpnGateway ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the VpnGateway + (pending,available,deleting,deleted) + - *type*, the type of customer gateway (ipsec.1) + - *availabilityZone*, the Availability zone the + VPN gateway is in. + + :rtype: list + :return: A list of :class:`boto.vpc.customergateway.VpnGateway` + """ + params = {} + if vpn_gateway_ids: + self.build_list_params(params, vpn_gateway_ids, 'VpnGatewayId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeVpnGateways', params, [('item', VpnGateway)]) + + def create_vpn_gateway(self, type, availability_zone=None): + """ + Create a new Vpn Gateway + + :type type: str + :param type: Type of VPN Connection. Only valid valid currently is 'ipsec.1' + + :type availability_zone: str + :param availability_zone: The Availability Zone where you want the VPN gateway. + + :rtype: The newly created VpnGateway + :return: A :class:`boto.vpc.vpngateway.VpnGateway` object + """ + params = {'Type' : type} + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('CreateVpnGateway', params, VpnGateway) + + def delete_vpn_gateway(self, vpn_gateway_id): + """ + Delete a Vpn Gateway. + + :type vpn_gateway_id: str + :param vpn_gateway_id: The ID of the vpn_gateway to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'VpnGatewayId': vpn_gateway_id} + return self.get_status('DeleteVpnGateway', params) + + def attach_vpn_gateway(self, vpn_gateway_id, vpc_id): + """ + Attaches a VPN gateway to a VPC. + + :type vpn_gateway_id: str + :param vpn_gateway_id: The ID of the vpn_gateway to attach + + :type vpc_id: str + :param vpc_id: The ID of the VPC you want to attach the gateway to. + + :rtype: An attachment + :return: a :class:`boto.vpc.vpngateway.Attachment` + """ + params = {'VpnGatewayId': vpn_gateway_id, + 'VpcId' : vpc_id} + return self.get_object('AttachVpnGateway', params, Attachment) + + # Subnets + + def get_all_subnets(self, subnet_ids=None, filters=None): + """ + Retrieve information about your Subnets. You can filter results to + return information only about those Subnets that match your search + parameters. Otherwise, all Subnets associated with your account + are returned. + + :type subnet_ids: list + :param subnet_ids: A list of strings with the desired Subnet ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the Subnet + (pending,available) + - *vpdId*, the ID of teh VPC the subnet is in. + - *cidrBlock*, CIDR block of the subnet + - *availabilityZone*, the Availability Zone + the subnet is in. + + + :rtype: list + :return: A list of :class:`boto.vpc.subnet.Subnet` + """ + params = {} + if subnet_ids: + self.build_list_params(params, subnet_ids, 'SubnetId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeSubnets', params, [('item', Subnet)]) + + def create_subnet(self, vpc_id, cidr_block, availability_zone=None): + """ + Create a new Subnet + + :type vpc_id: str + :param vpc_id: The ID of the VPC where you want to create the subnet. + + :type cidr_block: str + :param cidr_block: The CIDR block you want the subnet to cover. + + :type availability_zone: str + :param availability_zone: The AZ you want the subnet in + + :rtype: The newly created Subnet + :return: A :class:`boto.vpc.customergateway.Subnet` object + """ + params = {'VpcId' : vpc_id, + 'CidrBlock' : cidr_block} + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('CreateSubnet', params, Subnet) + + def delete_subnet(self, subnet_id): + """ + Delete a subnet. + + :type subnet_id: str + :param subnet_id: The ID of the subnet to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'SubnetId': subnet_id} + return self.get_status('DeleteSubnet', params) + + + # DHCP Options + + def get_all_dhcp_options(self, dhcp_options_ids=None): + """ + Retrieve information about your DhcpOptions. + + :type dhcp_options_ids: list + :param dhcp_options_ids: A list of strings with the desired DhcpOption ID's + + :rtype: list + :return: A list of :class:`boto.vpc.dhcpoptions.DhcpOptions` + """ + params = {} + if dhcp_options_ids: + self.build_list_params(params, dhcp_options_ids, 'DhcpOptionsId') + return self.get_list('DescribeDhcpOptions', params, [('item', DhcpOptions)]) + + def create_dhcp_options(self, vpc_id, cidr_block, availability_zone=None): + """ + Create a new DhcpOption + + :type vpc_id: str + :param vpc_id: The ID of the VPC where you want to create the subnet. + + :type cidr_block: str + :param cidr_block: The CIDR block you want the subnet to cover. + + :type availability_zone: str + :param availability_zone: The AZ you want the subnet in + + :rtype: The newly created DhcpOption + :return: A :class:`boto.vpc.customergateway.DhcpOption` object + """ + params = {'VpcId' : vpc_id, + 'CidrBlock' : cidr_block} + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('CreateDhcpOption', params, DhcpOptions) + + def delete_dhcp_options(self, dhcp_options_id): + """ + Delete a DHCP Options + + :type dhcp_options_id: str + :param dhcp_options_id: The ID of the DHCP Options to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'DhcpOptionsId': subnet_id} + return self.get_status('DeleteDhcpOptions', params) + + def associate_dhcp_options(self, dhcp_options_id, vpc_id): + """ + Associate a set of Dhcp Options with a VPC. + + :type dhcp_options_id: str + :param dhcp_options_id: The ID of the Dhcp Options + + :type vpc_id: str + :param vpc_id: The ID of the VPC. + + :rtype: bool + :return: True if successful + """ + params = {'DhcpOptionsId': dhcp_option, + 'VpcId' : vpc_id} + return self.get_status('AssociateDhcpOptions', params) + + # VPN Connection + + def get_all_vpn_connections(self, vpn_connection_ids=None, filters=None): + """ + Retrieve information about your VPN_CONNECTIONs. You can filter results to + return information only about those VPN_CONNECTIONs that match your search + parameters. Otherwise, all VPN_CONNECTIONs associated with your account + are returned. + + :type vpn_connection_ids: list + :param vpn_connection_ids: A list of strings with the desired VPN_CONNECTION ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the VPN_CONNECTION + pending,available,deleting,deleted + - *type*, the type of connection, currently 'ipsec.1' + - *customerGatewayId*, the ID of the customer gateway + associated with the VPN + - *vpnGatewayId*, the ID of the VPN gateway associated + with the VPN connection + + :rtype: list + :return: A list of :class:`boto.vpn_connection.vpnconnection.VpnConnection` + """ + params = {} + if vpn_connection_ids: + self.build_list_params(params, vpn_connection_ids, 'Vpn_ConnectionId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeVpnConnections', params, [('item', VPNConnection)]) + + def create_vpn_connection(self, type, customer_gateway_id, vpn_gateway_id): + """ + Create a new VPN Connection. + + :type type: str + :param type: The type of VPN Connection. Currently only 'ipsec.1' + is supported + + :type customer_gateway_id: str + :param customer_gateway_id: The ID of the customer gateway. + + :type vpn_gateway_id: str + :param vpn_gateway_id: The ID of the VPN gateway. + + :rtype: The newly created VpnConnection + :return: A :class:`boto.vpc.vpnconnection.VpnConnection` object + """ + params = {'Type' : type, + 'CustomerGatewayId' : customer_gateway_id, + 'VpnGatewayId' : vpn_gateway_id} + return self.get_object('CreateVpnConnection', params, VpnConnection) + + def delete_vpn_connection(self, vpn_connection_id): + """ + Delete a VPN Connection. + + :type vpn_connection_id: str + :param vpn_connection_id: The ID of the vpn_connection to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'VpnConnectionId': vpn_connection_id} + return self.get_status('DeleteVpnConnection', params) + + diff --git a/api/boto/vpc/customergateway.py b/api/boto/vpc/customergateway.py new file mode 100644 index 0000000..c50a616 --- /dev/null +++ b/api/boto/vpc/customergateway.py @@ -0,0 +1,54 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Customer Gateway +""" + +from boto.ec2.ec2object import EC2Object + +class CustomerGateway(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.type = None + self.state = None + self.ip_address = None + self.bgp_asn = None + + def __repr__(self): + return 'CustomerGateway:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'customerGatewayId': + self.id = value + elif name == 'ipAddress': + self.ip_address = value + elif name == 'type': + self.type = value + elif name == 'state': + self.state = value + elif name == 'bgpAsn': + self.bgp_asn = value + else: + setattr(self, name, value) + diff --git a/api/boto/vpc/dhcpoptions.py b/api/boto/vpc/dhcpoptions.py new file mode 100644 index 0000000..4fce7dc --- /dev/null +++ b/api/boto/vpc/dhcpoptions.py @@ -0,0 +1,69 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a DHCP Options set +""" + +from boto.ec2.ec2object import EC2Object + +class DhcpValueSet(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'value': + self.append(value) + +class DhcpConfigSet(dict): + + def startElement(self, name, attrs, connection): + if name == 'valueSet': + if not self.has_key(self._name): + self[self._name] = DhcpValueSet() + return self[self._name] + + def endElement(self, name, value, connection): + if name == 'key': + self._name = value + +class DhcpOptions(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.options = None + + def __repr__(self): + return 'DhcpOptions:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'dhcpConfigurationSet': + self.options = DhcpConfigSet() + return self.options + + def endElement(self, name, value, connection): + if name == 'dhcpOptionsId': + self.id = value + else: + setattr(self, name, value) + diff --git a/api/boto/vpc/subnet.py b/api/boto/vpc/subnet.py new file mode 100644 index 0000000..de8a959 --- /dev/null +++ b/api/boto/vpc/subnet.py @@ -0,0 +1,54 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Subnet +""" + +from boto.ec2.ec2object import EC2Object + +class Subnet(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.state = None + self.cidr_block = None + self.available_ip_address_count = 0 + self.availability_zone = None + + def __repr__(self): + return 'Subnet:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'subnetId': + self.id = value + elif name == 'state': + self.state = value + elif name == 'cidrBlock': + self.cidr_block = value + elif name == 'availableIpAddressCount': + self.available_ip_address_count = int(value) + elif name == 'availabilityZone': + self.availability_zone = value + else: + setattr(self, name, value) + diff --git a/api/boto/vpc/vpc.py b/api/boto/vpc/vpc.py new file mode 100644 index 0000000..152cff3 --- /dev/null +++ b/api/boto/vpc/vpc.py @@ -0,0 +1,54 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Virtual Private Cloud. +""" + +from boto.ec2.ec2object import EC2Object + +class VPC(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.dhcp_options_id = None + self.state = None + self.cidr_block = None + + def __repr__(self): + return 'VPC:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'vpcId': + self.id = value + elif name == 'dhcpOptionsId': + self.dhcp_options_id = value + elif name == 'state': + self.state = value + elif name == 'cidrBlock': + self.cidr_block = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_vpc(self.id) + diff --git a/api/boto/vpc/vpnconnection.py b/api/boto/vpc/vpnconnection.py new file mode 100644 index 0000000..42739d9 --- /dev/null +++ b/api/boto/vpc/vpnconnection.py @@ -0,0 +1,60 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a VPN Connectionn +""" + +from boto.ec2.ec2object import EC2Object + +class VpnConnection(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.state = None + self.customer_gateway_configuration = None + self.type = None + self.customer_gateway_id = None + self.vpn_gateway_id = Nonen + + def __repr__(self): + return 'VpnConnection:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'vpnConnectionId': + self.id = value + elif name == 'state': + self.state = value + elif name == 'CustomerGatewayConfiguration': + self.customer_gateway_configuration = value + elif name == 'type': + self.type = value + elif name == 'customerGatewayId': + self.customer_gateway_id = value + elif name == 'vpnGatewayId': + self.vpn_gateway_id = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_vpn_connection(self.id) + diff --git a/api/boto/vpc/vpngateway.py b/api/boto/vpc/vpngateway.py new file mode 100644 index 0000000..0fa0a9e --- /dev/null +++ b/api/boto/vpc/vpngateway.py @@ -0,0 +1,80 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Vpn Gateway +""" + +from boto.ec2.ec2object import EC2Object + +class Attachment(object): + + def __init__(self, connection=None): + self.vpc_id = None + self.state = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'vpcId': + self.vpc_id = value + elif name == 'state': + self.state = value + else: + setattr(self, name, value) + +class VpnGateway(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.type = None + self.state = None + self.availability_zone = None + self.attachments = [] + + def __repr__(self): + return 'VpnGateway:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'item': + att = Attachment() + self.attachments.append(att) + return att + + def endElement(self, name, value, connection): + if name == 'vpnGatewayId': + self.id = value + elif name == 'type': + self.type = value + elif name == 'state': + self.state = value + elif name == 'availabilityZone': + self.availability_zone = value + elif name == 'attachments': + pass + else: + setattr(self, name, value) + + def attach(self, vpc_id): + return self.connection.attach_vpn_gateway(self.id, vpc_id) + diff --git a/api/dokillapi.sh b/api/dokillapi.sh new file mode 100755 index 0000000..880cf85 --- /dev/null +++ b/api/dokillapi.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +TEE="tee -a /data/logs/api.log" +ROTATELOG="/usr/sbin/rotatelogs -l -f /data/logs/api-rot/api.log.%Y-%m-%d_%H.%M.%S 200M" + +echo "######################## KILL API "`date`" ##########################" | $TEE | $ROTATELOG + +kill -9 `cat pid` diff --git a/api/dostartapi.sh b/api/dostartapi.sh new file mode 100755 index 0000000..babe4b2 --- /dev/null +++ b/api/dostartapi.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +TEE="tee -a /data/logs/api.log" +ROTATELOG="/usr/sbin/rotatelogs -l -f /data/logs/api-rot/api.log.%Y-%m-%d_%H.%M.%S 200M" +ERRORLOG="/data/logs/apierrors.log" + +echo | $TEE | $ROTATELOG +echo "######################## START API "`date`" ##########################" | $TEE | $ROTATELOG +echo "######################## START API "`date`" ##########################" > $ERRORLOG + +export PYTHONPATH=.:.. + +nohup uwsgi-python2.6 -C -s /var/nginx/api-uwsgi.sock -i -M -w wsgi -z 30 -p 150 -l 50 -L -R 10000 -b 8192 --no-orphans --pidfile pid 2>$ERRORLOG | $TEE | $ROTATELOG & + + diff --git a/api/keepalive.sh b/api/keepalive.sh new file mode 100755 index 0000000..f596c5c --- /dev/null +++ b/api/keepalive.sh @@ -0,0 +1,16 @@ +if [ -f "DONTSTART" ]; then + exit 0; +fi + +count=`ps aux|grep api-uwsgi.sock | grep -v grep | wc -l` +response=`curl http://api.indextank.com/ 2> /dev/null` + +if [ "$response" != "\"IndexTank API. Please refer to the documentation: http://indextank.com/documentation/api\"" ]; then + date + echo Restarting webapp. Process count is $count, response was: + echo "$response" + ./dokillapi.sh + sleep 5s + ./dostartapi.sh +fi + diff --git a/api/kill_webapp.sh b/api/kill_webapp.sh new file mode 100755 index 0000000..5174d9d --- /dev/null +++ b/api/kill_webapp.sh @@ -0,0 +1,3 @@ +#!/bin/bash +touch DONTSTART +./dokillapi.sh diff --git a/api/lib/__init__.py b/api/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/lib/authorizenet.py b/api/lib/authorizenet.py new file mode 100644 index 0000000..00bcd6c --- /dev/null +++ b/api/lib/authorizenet.py @@ -0,0 +1,153 @@ +from xml.dom.minidom import Document, parseString +import httplib +import urlparse + + +class AuthorizeNet: + """ + Basic client for Authorize.net's Automated Recurring Billing (ARB) service + """ + + def __init__(self): + from django.conf import settings + f = open("authorize.settings.prod") if not settings.DEBUG else open("authorize.settings.debug") + for line in f: + line = line.strip() + if len(line) > 0 and not line.startswith('#'): + parts = line.split('=',1) + var = parts[0].strip() + val = parts[1].strip() + if var in ['host_url','api_login_id','transaction_key']: + cmd = 'self.%s = %s' % (var,val) + exec(cmd) + + def subscription_create(self, refId, name, length, unit, startDate, totalOccurrences, trialOccurrences, + amount, trialAmount, cardNumber, expirationDate, firstName, lastName, company, + address, city, state, zip, country): + doc,root = self._new_doc("ARBCreateSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + subscription = self._add_node(doc, root, 'subscription') + self._add_text_node(doc, subscription, 'name', name) + paymentSchedule = self._add_node(doc, subscription, 'paymentSchedule') + interval = self._add_node(doc, paymentSchedule, 'interval') + self._add_text_node(doc, interval, 'length', length) + self._add_text_node(doc, interval, 'unit', unit) + self._add_text_node(doc, paymentSchedule, 'startDate', startDate) + self._add_text_node(doc, paymentSchedule, 'totalOccurrences', totalOccurrences) + self._add_text_node(doc, paymentSchedule, 'trialOccurrences', trialOccurrences) + self._add_text_node(doc, subscription, 'amount', amount) + self._add_text_node(doc, subscription, 'trialAmount', trialAmount) + payment = self._add_node(doc, subscription, 'payment') + creditcard = self._add_node(doc, payment, 'creditCard') + self._add_text_node(doc, creditcard, 'cardNumber', cardNumber) + self._add_text_node(doc, creditcard, 'expirationDate', expirationDate) + billto = self._add_node(doc, subscription, 'billTo') + self._add_text_node(doc, billto, 'firstName', firstName) + self._add_text_node(doc, billto, 'lastName', lastName) + self._add_text_node(doc, billto, 'company', company) + self._add_text_node(doc, billto, 'address', address) + self._add_text_node(doc, billto, 'city', city) + self._add_text_node(doc, billto, 'state', state) + self._add_text_node(doc, billto, 'zip', zip) + self._add_text_node(doc, billto, 'country', country) + res = self._send_xml(doc.toxml()) + subscriptionId = res.getElementsByTagName('subscriptionId')[0].childNodes[0].nodeValue + return subscriptionId + + + def subscription_update(self, refId, subscriptionId, name, startDate, totalOccurrences, trialOccurrences, + amount, trialAmount, cardNumber, expirationDate, firstName, lastName, company, + address, city, state, zip, country): + doc,root = self._new_doc("ARBUpdateSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + self._add_text_node(doc, root, 'subscriptionId', subscriptionId) + subscription = self._add_node(doc, root, 'subscription') + if name: + self._add_text_node(doc, subscription, 'name', name) + if startDate or totalOccurrences or trialOccurrences: + paymentSchedule = self._add_node(doc, subscription, 'paymentSchedule') + if startDate: + self._add_text_node(doc, paymentSchedule, 'startDate', startDate) + if totalOccurrences: + self._add_text_node(doc, paymentSchedule, 'totalOccurrences', totalOccurrences) + if trialOccurrences: + self._add_text_node(doc, paymentSchedule, 'trialOccurrences', trialOccurrences) + if amount: + self._add_text_node(doc, subscription, 'amount', amount) + if trialAmount: + self._add_text_node(doc, subscription, 'trialAmount', trialAmount) + if cardNumber and expirationDate: + payment = self._add_node(doc, subscription, 'payment') + creditcard = self._add_node(doc, payment, 'creditCard') + self._add_text_node(doc, creditcard, 'cardNumber', cardNumber) + self._add_text_node(doc, creditcard, 'expirationDate', expirationDate) + if firstName and lastName: + billto = self._add_node(doc, subscription, 'billTo') + self._add_text_node(doc, billto, 'firstName', firstName) + self._add_text_node(doc, billto, 'lastName', lastName) + if company: + self._add_text_node(doc, billto, 'company', company) + if address and city and state and zip and country: + self._add_text_node(doc, billto, 'address', address) + self._add_text_node(doc, billto, 'city', city) + self._add_text_node(doc, billto, 'state', state) + self._add_text_node(doc, billto, 'zip', zip) + self._add_text_node(doc, billto, 'country', country) + self._send_xml(doc.toxml()) + + + def subscription_cancel(self, refId, subscriptionId): + doc,root = self._new_doc("ARBCancelSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + self._add_text_node(doc, root, 'subscriptionId', subscriptionId) + self._send_xml(doc.toxml()) + + + def _add_node(self, doc, node, name): + elem = doc.createElement(name) + node.appendChild(elem) + return elem + + def _add_text_node(self, doc, node, name, text): + elem = self._add_node(doc, node, name) + text_node = doc.createTextNode(text) + elem.appendChild(text_node) + return elem + + def _new_doc(self, operation): + doc = Document() + root = doc.createElement(operation) + root.setAttribute('xmlns','AnetApi/xml/v1/schema/AnetApiSchema.xsd') + doc.appendChild(root) + auth = self._add_node(doc, root, 'merchantAuthentication') + self._add_text_node(doc, auth, 'name', self.api_login_id) + self._add_text_node(doc, auth, 'transactionKey', self.transaction_key) + return doc, root + + def _send_xml(self, xml): + splits = urlparse.urlsplit(self.host_url) + print "connection.request('POST', "+self.host_url+", xml, {'Content-Type':'text/xml'})" + print "xml: "+xml + connection = httplib.HTTPSConnection(splits.hostname) + connection.request('POST', self.host_url, xml, {'Content-Type':'text/xml'}) + response = connection.getresponse() + response.body = response.read() + connection.close() + print "resp: "+response.body + res = parseString(response.body) + ok = res.getElementsByTagName('resultCode')[0].childNodes[0].nodeValue == "Ok" + if not ok: + code = res.getElementsByTagName('message')[0].childNodes[0].childNodes[0].nodeValue + msg = res.getElementsByTagName('message')[0].childNodes[1].childNodes[0].nodeValue + " (%s)"%code + raise BillingException(msg,code) + return res + + +class BillingException(Exception): + def __init__(self, msg, code): + self.msg = msg + self.code = code + def __str__(self): + return repr(self.msg) + + diff --git a/api/lib/encoder.py b/api/lib/encoder.py new file mode 100644 index 0000000..f6bb4dd --- /dev/null +++ b/api/lib/encoder.py @@ -0,0 +1,74 @@ +# Short URL Generator + +#DEFAULT_ALPHABET = 'JedR8LNFY2j6MrhkBSADUyfP5amuH9xQCX4VqbgpsGtnW7vc3TwKE' +DEFAULT_ALPHABET = 'ed82j6rhkyf5amu9x4qbgpstn7vc3w1ioz' +DEFAULT_BLOCK_SIZE = 22 + +class Encoder(object): + def __init__(self, alphabet=DEFAULT_ALPHABET, block_size=DEFAULT_BLOCK_SIZE): + self.alphabet = alphabet + self.block_size = block_size + self.mask = (1 << block_size) - 1 + self.mapping = range(block_size) + self.mapping.reverse() + def encode_url(self, n, min_length=0): + return self.enbase(self.encode(n), min_length) + def decode_url(self, n): + return self.decode(self.debase(n)) + def encode(self, n): + return (n & ~self.mask) | self._encode(n & self.mask) + def _encode(self, n): + result = 0 + for i, b in enumerate(self.mapping): + if n & (1 << i): + result |= (1 << b) + return result + def decode(self, n): + return (n & ~self.mask) | self._decode(n & self.mask) + def _decode(self, n): + result = 0 + for i, b in enumerate(self.mapping): + if n & (1 << b): + result |= (1 << i) + return result + def enbase(self, x, min_length=0): + result = self._enbase(x) + padding = self.alphabet[0] * (min_length - len(result)) + return '%s%s' % (padding, result) + def _enbase(self, x): + n = len(self.alphabet) + if x < n: + return self.alphabet[x] + return self.enbase(x/n) + self.alphabet[x%n] + def debase(self, x): + n = len(self.alphabet) + result = 0 + for i, c in enumerate(reversed(x)): + result += self.alphabet.index(c) * (n**i) + return result + +DEFAULT_ENCODER = Encoder() + +def encode(n): + return DEFAULT_ENCODER.encode(n) + +def decode(n): + return DEFAULT_ENCODER.decode(n) + +def enbase(n, min_length=0): + return DEFAULT_ENCODER.enbase(n, min_length) + +def debase(n): + return DEFAULT_ENCODER.debase(n) + +def encode_url(n, min_length=0): + return DEFAULT_ENCODER.encode_url(n, min_length) + +def decode_url(n): + return DEFAULT_ENCODER.decode_url(n) + +def to_key(n): + return enbase(encode(n)) + +def from_key(n): + return decode(debase(n)) diff --git a/api/lib/error_logging.py b/api/lib/error_logging.py new file mode 100644 index 0000000..dbf927b --- /dev/null +++ b/api/lib/error_logging.py @@ -0,0 +1,13 @@ +import traceback +from lib import flaptor_logging +from django.http import HttpResponse + +logger = flaptor_logging.get_logger('error_logging') + +class ViewErrorLoggingMiddleware: + + def process_view(self, request, view_func, view_args, view_kwargs): + self.view_name = view_func.__name__ + def process_exception(self, request, exception): + logger.error('UNEXPECTED EXCEPTION in view "%s". Exception is: %s', self.view_name, repr(traceback.print_exc())) + return HttpResponse('{"status":"ERROR", "message":"Unexpected error."}') diff --git a/api/lib/exceptions.py b/api/lib/exceptions.py new file mode 100644 index 0000000..c6ff0a7 --- /dev/null +++ b/api/lib/exceptions.py @@ -0,0 +1,8 @@ + + +class CloudException(Exception): + pass + +class NoIndexerException(CloudException): + pass + diff --git a/api/lib/flaptor_logging.py b/api/lib/flaptor_logging.py new file mode 100644 index 0000000..1af893c --- /dev/null +++ b/api/lib/flaptor_logging.py @@ -0,0 +1,100 @@ +import logging as pylogging +from logging import config +import os + +usingNativeLogger = True + +__loggers = {} + + + +def get_logger(name, force_new=False): + '''Get the Logger instance for a given name''' + global __loggers + if __loggers is None: + __loggers = {} + if force_new: + return pylogging.getLogger(name) + if not __loggers.has_key(name): + __loggers[name] = pylogging.getLogger(name) + return __loggers[name] + +class SpecialFormatter(pylogging.Formatter): + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) + RESET_SEQ = "\033[0m" + COLOR_SEQ = "\033[37;4%dm" + PIDCOLOR_SEQ = "\033[1;3%dm" + BOLD_SEQ = "\033[1m" + COLORS = { + 'WARN': YELLOW, + 'INFO': GREEN, + 'DEBU': BLUE, + 'CRIT': RED, + 'ERRO': RED + } + + def __init__(self, *args, **kwargs): + pylogging.Formatter.__init__(self, *args, **kwargs) + def format(self, record): + if not hasattr(record, 'prefix'): record.prefix = '' + if not hasattr(record, 'suffix'): record.suffix = '' + if not hasattr(record, 'compname'): record.compname = '' + record.pid = os.getpid() + + record.levelname = record.levelname[:4] + + r = pylogging.Formatter.format(self, record) + if record.levelname in SpecialFormatter.COLORS: + levelcolor = SpecialFormatter.COLOR_SEQ % (SpecialFormatter.COLORS[record.levelname]) + r = r.replace('$LEVELCOLOR', levelcolor) + r = r.replace('$RESET', SpecialFormatter.RESET_SEQ) + else: + r = r.replace('$COLOR', '') + r = r.replace('$RESET', '') + pidcolor = SpecialFormatter.COLOR_SEQ % (1 + (record.pid % 5)) + r = r.replace('$PIDCOLOR', pidcolor) + r = r.replace('$BOLD', SpecialFormatter.BOLD_SEQ) + return r + +pylogging.SpecialFormatter = SpecialFormatter + +if usingNativeLogger: + try: + config.fileConfig('logging.conf') + except Exception, e: + print e + +#class NativePythonLogger: +# def __init__(self, name): +# '''Creates a new Logger for the given name. +# Do not call this method directly, instead use +# get_logger(name) to get the appropriate instance''' +# self.name = name +# self.__logger = pylogging.getLogger(name) +# #self.updateLevel(5) +# +# def updateLevel(self, level): +# self.__level = level +# if level == 1: +# self.__logger.setLevel(pylogging.CRITICAL) +# elif level == 2: +# self.__logger.setLevel(pylogging.INFO) +# elif level == 3: +# self.__logger.setLevel(pylogging.WARNING) +# elif level == 4: +# self.__logger.setLevel(pylogging.INFO) +# elif level == 5: +# self.__logger.setLevel(pylogging.DEBUG) +# +# def debug(self, format_str, *values): +# self.__logger.debug(format_str, *values) +# def info(self, format_str, *values): +# self.__logger.info(format_str, *values) +# def warn(self, format_str, *values): +# self.__logger.warn(format_str, *values) +# def error(self, format_str, *values): +# self.__logger.error(format_str, *values) +# def exception(self, format_str, *values): +# self.__logger.exception(format_str, *values) +# def fatal(self, format_str, *values): +# self.__logger.critical(format_str, *values) diff --git a/api/lib/mail.py b/api/lib/mail.py new file mode 100644 index 0000000..b1d3a08 --- /dev/null +++ b/api/lib/mail.py @@ -0,0 +1,101 @@ +from django.core.mail import send_mail + +try: + ENV=open('/data/env.name').readline().strip() +except Exception: + ENV='PROD+EXC' + +def _no_fail(method, *args, **kwargs): + def decorated(*args, **kwargs): + try: + return method(*args, **kwargs) + except Exception, e: + print e + return + return decorated + + + +@_no_fail +def report_payment_data(account): + activity_report = 'An Account has entered payment data\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + account.package.name + '\n' + activity_report += 'Email: ' + account.user.email + '\n' + activity_report += 'API KEY: ' + account.apikey + '' + + report_activity('Payment Data for ' + account.package.name + ' Account (' + account.user.email + ')', activity_report) + +@_no_fail +def report_new_account(account): + activity_report = 'A new Account was created\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + account.package.name + '\n' + activity_report += 'Email: ' + account.user.email + '\n' + activity_report += 'API KEY: ' + account.apikey + '' + + report_activity('New ' + account.package.name + ' Account (' + account.user.email + ')', activity_report) + +@_no_fail +def report_new_index(index): + activity_report = 'A new Index was created\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + index.account.package.name + '\n' + activity_report += 'User Email: ' + index.account.user.email + '\n' + activity_report += 'Index Name: ' + index.name + '' + + report_activity('Index activity (' + index.code + ')', activity_report, 'l') + +@_no_fail +def report_new_deploy(deploy): + activity_report = 'A new Deploy is now controllable\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + deploy.index.account.package.name + '\n' + activity_report += 'User Email: ' + deploy.index.account.user.email + '\n' + activity_report += 'Index Name: ' + deploy.index.name + '\n' + activity_report += 'Worker: #' + str(deploy.worker.id) + '\n' + activity_report += ('Deploy: %r' % deploy) + '\n' + activity_report += ('Container Index: %r' % deploy.index) + '\n' + + report_activity('Index activity (' + deploy.index.code + ')', activity_report, 'l') + +@_no_fail +def report_delete_index(index): + activity_report = 'An Index has been deleted\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + index.account.package.name + '\n' + activity_report += 'User Email: ' + index.account.user.email + '\n' + activity_report += 'Index Name: ' + index.name + '\n' + + report_activity('Index activity (' + index.code + ')', activity_report, 'l') + +@_no_fail +def report_new_worker(worker): + activity_report = 'A new Worker was created\n' + activity_report += '---------------------------\n' + activity_report += repr(worker) + + report_activity('New Worker (%d)' % (worker.pk), activity_report, 't') + +@_no_fail +def report_automatic_redeploy(deploy, initial_xmx, new_xmx): + activity_report = 'Automatic redeploy.\n' + activity_report += '---------------------------\n' + activity_report += 'initial xmx value: %d\n' % (initial_xmx) + activity_report += 'new xmx value: %d\n' % (new_xmx) + activity_report += repr(deploy) + + report_activity('Automatic redeploy', activity_report, 't') + +@_no_fail +def report_activity(subject, body, type='b'): + if type == 'b': + mail_to = 'activity@indextank.com' + elif type == 't': + mail_to = 'activitytech@indextank.com' + elif type == 'l': + mail_to = 'lowactivity@indextank.com' + else: + raise Exception('Wrong report type') + + send_mail(ENV + ' - ' + subject, body, 'IndexTank Activity ', [mail_to], fail_silently=False) diff --git a/api/lib/monitor.py b/api/lib/monitor.py new file mode 100644 index 0000000..3fe9818 --- /dev/null +++ b/api/lib/monitor.py @@ -0,0 +1,148 @@ + +from threading import Thread +from traceback import format_tb +import time, datetime +import sys +import shelve + +from django.core.mail import send_mail + +from lib import flaptor_logging + +try: + ENV=open('/data/env.name').readline().strip() +except Exception: + ENV='PROD+EXC' + +#helper functions +def is_prod(): + return ENV == 'PROD' or ENV == 'QoS_Monitor' + +def env_name(): + if ENV == 'PROD': + return 'PRODUCTION' + elif ENV == 'QoS_Monitor': + return 'QoS_Monitor' + else: + return ENV + +class Monitor(Thread): + def __init__(self, pagerduty_email='api-monitor@flaptor.pagerduty.com'): + super(Monitor, self).__init__() + self.name = self.__class__.__name__ + self.statuses = shelve.open('/data/monitor-%s.shelf' % self.name) + self.logger = flaptor_logging.get_logger(self.name) + self.failure_threshold = 1 + self.fatal_failure_threshold = 0 + self.severity = 'WARNING' + self.title_template = '%s::%s: [%s] %s' + self.pagerduty_email = pagerduty_email + + def iterable(self): + return [None] + + def run(self): + self.step = 1 + while True: + starttime = int(time.time()) + try: + self.logger.info("running cycle %d", self.step) + for object in self.iterable(): + self._monitor(object) + self.report_ok("unexpected error in monitor cycle") + self.clean() + except Exception: + self.logger.exception("Unexpected error while executing cycle") + self.report_bad("unexpected error in monitor cycle", 1, 0, 'UNEXPECTED ERROR IN THE CYCLE OF %s\n\n%s' % (self.name, self.describe_error())) + self.step += 1 + self.statuses.sync() + time.sleep(max(0, self.period - (int(time.time()) - starttime))) + + def clean(self): + for title, status in self.statuses.items(): + if not status['working']: + if status['last_update'] != self.step: + self.report_ok(title) + else: + del self.statuses[title] + + + def _monitor(self, object): + try: + if self.monitor(object): + self.report_ok(str(self.alert_title(object))) + else: + self.report_bad(str(self.alert_title(object)), self.failure_threshold, self.fatal_failure_threshold, self.alert_msg(object)) + self.report_ok("unexpected error in monitor") + except Exception, e: + self.logger.exception("Unexpected error while executing monitor. Exception is: %s" % (e)) + message = 'UNEXPECTED ERROR IN THE MONITORING OF %s FOR TITLE: %s\n\n%s' % (self.name, self.alert_title(object), self.describe_error()) + self.report_bad("unexpected error in monitor", 1, 'WARNING', message) + + def describe_error(self): + exc_type, exc_value, exc_traceback = sys.exc_info() + return 'EXCEPTION: %s : %s\ntraceback:\n%s' % (exc_type, exc_value, ''.join(format_tb(exc_traceback))) + + def update_status(self, key, **kwargs): + self.statuses[key] = kwargs + + def send_alert(self, title, message, severity): + try: + if is_prod(): + if severity == 'FATAL': + name = 'FATAL ALERT (%s)' % env_name() + else: + name = 'ALERT (%s)' % env_name() + else: + name = '%s test alert' % ENV + + title = self.title_template % (ENV, self.name, severity, title) + message += '\n\n--------SENT AT ' + str(datetime.datetime.now()) + to = ['alerts@indextank.com'] + if severity == 'FATAL' and is_prod(): + to.append('alerts+fatal@indextank.com') + to.append(self.pagerduty_email) + send_mail(title, message, '"%s" ' % name, to, fail_silently=False) + self.logger.info('Sending alert for title: %s\n============\n%s', title, message) + except Exception, e: + self.logger.exception("Unexpected error while sending alerts. Exception is: %s" % (e)) + + def report_ok(self, title): + if title in self.statuses and not self.statuses[title]['working'] and (self.statuses[title]['alerted'] or self.statuses[title]['alerted_fatal']): + # it has just been resolved + self.send_alert(title, 'The problem is no longer reported. The last message was:\n %s' % (self.statuses[title]['message']), self.severity) + if title in self.statuses: + del self.statuses[title] + + def report_bad(self, title, threshold, fatal_threshold, message): + if title in self.statuses and not self.statuses[title]['working']: + # this object had already failed, let's grab the first step in which it failed + first_failure = self.statuses[title]['first_failure'] + has_alerted = self.statuses[title]['alerted'] + has_alerted_fatal = self.statuses[title]['alerted_fatal'] + else: + # this object was fine, first failure is now + first_failure = self.step + has_alerted = False + has_alerted_fatal = False + + + should_alert = self.step - first_failure + 1 >= threshold + should_alert_fatal = fatal_threshold > 0 and self.step - first_failure + 1 >= fatal_threshold + + if should_alert_fatal: + if not has_alerted_fatal: + has_alerted_fatal = True + if is_prod(): + self.send_alert(title, 'A new problem on IndexTank has been detected:\n %s' % (message), 'FATAL') + else: + self.logger.info('Fatal error was found but alert has already been sent') + elif should_alert: + if not has_alerted: + has_alerted = True + self.send_alert(title, 'A new problem on IndexTank has been detected:\n %s' % (message), self.severity) + else: + self.logger.info('Error was found but alert has already been sent') + + # save current state of the object (is_failed, message, first_failure, last_update) + self.update_status(title, working=False, last_update=self.step, message=message, first_failure=first_failure, alerted=has_alerted, alerted_fatal=has_alerted_fatal) diff --git a/api/logging.conf b/api/logging.conf new file mode 100644 index 0000000..838ebbb --- /dev/null +++ b/api/logging.conf @@ -0,0 +1,50 @@ +[loggers] +keys=root,rpc,boto,errors + +[handlers] +keys=consoleHandler,errorHandler + +[formatters] +keys=simpleFormatter,errorFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler + +[logger_rpc] +level=INFO +handlers=consoleHandler +propagate=0 +qualname=RPC + +[logger_boto] +level=INFO +handlers=consoleHandler +propagate=0 +qualname=boto + +[logger_errors] +level=INFO +handlers=errorHandler +propagate=0 +qualname=Errors + +[handler_consoleHandler] +class=StreamHandler +formatter=simpleFormatter +args=(sys.stdout,) + +[handler_errorHandler] +class=StreamHandler +formatter=errorFormatter +args=(sys.stderr,) + +[formatter_simpleFormatter] +format=%(pid)+5s %(asctime)s %(name)+8.8s:%(levelname)s%(prefix)s %(message)-90s %(suffix)s@%(filename)s:%(lineno)s +datefmt=%d/%m-%H.%M.%S +class=logging.SpecialFormatter + +[formatter_errorFormatter] +format=%(pid)+5s %(asctime)s %(compname)+8.8s:%(levelname)s%(prefix)s %(message)-90s %(suffix)s@%(filename)s:%(lineno)s +datefmt=%d/%m-%H.%M.%S +class=logging.SpecialFormatter diff --git a/api/manage.py b/api/manage.py new file mode 100644 index 0000000..db37c60 --- /dev/null +++ b/api/manage.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +from django.core.management import execute_manager + +from os import environ +from sys import argv + +environ['DJANGO_LOCAL'] = '' + +if argv[1] == 'runserver': + environ['DJANGO_LOCAL'] = '1' +if argv[1].startswith('local'): + environ['DJANGO_LOCAL'] = '1' + argv[1] = argv[1][5:] + +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) + sys.exit(1) + +if __name__ == "__main__": + execute_manager(settings) diff --git a/api/models.py b/api/models.py new file mode 100644 index 0000000..0908f60 --- /dev/null +++ b/api/models.py @@ -0,0 +1,938 @@ +import hashlib +import random +import binascii + +from lib.indextank.client import ApiClient, IndexAlreadyExists +from lib.authorizenet import AuthorizeNet, BillingException + +from django.db import models +from django.contrib.auth.models import User +from django.utils import simplejson as json +from django.db import IntegrityError +from django.db.models.aggregates import Sum, Count + +from lib import encoder, flaptor_logging + +from django.conf import settings +from datetime import datetime + +logger = flaptor_logging.get_logger('Models') + +# idea taken from https://www.grc.com/passwords.htm +def generate_apikey(id): + key = "2A1A8AE7CAEFAC47D6F74920CE4B0CE46430CDA6CF03D254C1C29402D727E570" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:14] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return unique_part + '-' + random_part + +def generate_onetimepass(id): + key = "CAEFAC47D6F7D727E57024920CE4B0CE46430CDA6CF03D254C1C29402A1A8AE7" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:5] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return unique_part + random_part + +def generate_forgotpass(id): + key = "E57024920CE4B0CE4643CAEFAC47D6F7D7270CDA6CF03D254C1C29402A1A8AE7" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:6] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return random_part + unique_part + + +# StoreFront models +class Account(models.Model): + apikey = models.CharField(max_length=22, unique=True) + creation_time = models.DateTimeField() + package = models.ForeignKey('Package', null=True) + status = models.CharField(max_length=30, null=False) + provisioner = models.ForeignKey('Provisioner', null=True) + + configuration = models.ForeignKey('IndexConfiguration', null=True) + default_analyzer = models.ForeignKey('Analyzer', null=True, related_name="accounts") + + class Statuses: + operational = 'OPERATIONAL' + creating = 'CREATING' + closed = 'CLOSED' + + def __repr__(self): + return 'Account (%s):\n\tuser_email: %s\n\tapikey: %s\n\tcreation_time: %s\n\tstatus: %s\n\tpackage: %s\n\tconfiguration: %s\n' % (self.id, PFUser.objects.filter(account=self)[0].email, str(self.apikey), str(self.creation_time), str(self.status), self.package.name, self.configuration.description) + + def __str__(self): + return '(apikey: %s; creation_time: %s; status: %s)' % (str(self.apikey), str(self.creation_time), str(self.status)) + + def count_indexes(self): + return self.indexes.aggregate(cnt=Count('id'))['cnt'] + + def count_documents(self): + return self.indexes.aggregate(cnt=Sum('current_docs_number'))['cnt'] + + def is_operational(self): + return self.status == Account.Statuses.operational + + def is_heroku(self): + # HACK UNTIL HEROKU IS A PROVISIONER + return self.package.code.startswith('HEROKU_') + #return self.provisioner and self.provisioner.name == 'heroku' + + @classmethod + def create_account(cls, dt, email=None, password=None): + account = Account() + + account.creation_time = datetime.now() + account.status = Account.Statuses.creating + account.save() + + account.apikey = generate_apikey(account.id) + account.save() + + unique_part, random_part = account.apikey.split('-', 1) + if email is None: + email = '%s@indextank.com' % unique_part + + if password is None: + password = random_part + + try: + user = User.objects.create_user(email, '', password) + except IntegrityError, e: + account.delete() + raise e + + try: + pfu = PFUser() + pfu.account = account + pfu.user = user + pfu.email = email + + pfu.save() + except IntegrityError, e: + account.delete() + user.delete() + raise e + + return account, pfu + + def create_index(self, index_name, public_search=None): + index = Index() + + # basic index data + index.populate_for_account(self) + index.name = index_name + index.creation_time = datetime.now() + index.language_code = 'en' + index.status = Index.States.new + if not public_search is None: + index.public_api = public_search + + # test for name uniqueness + # raises IntegrityError if the index name already exists + index.save() + + # define the default function + function = ScoreFunction() + function.index = index + function.name = '0' + function.definition = '-age' + function.save() + + # deduce code from id + index.code = encoder.to_key(index.id) + index.save() + + return index + + def create_demo_index(self): + try: + dataset = DataSet.objects.get(code='DEMO') + except DataSet.DoesNotExist: + logger.exception('DemoIndex dataset not present in database. Aborting demo index creation') + return + + index = self.create_index('DemoIndex') + + index.public_api = True + index.save() + + population = IndexPopulation() + population.index = index + population.status = IndexPopulation.Statuses.created + population.dataset = dataset + population.time = datetime.now() + population.populated_size = 0 + + population.save() + + def close(self): + # Dropping an account implies: + + # - removing the payment information from the account + # - removing the subscriptions from authorize.net + for info in self.payment_informations.all(): + auth = AuthorizeNet() + for subscription in info.subscriptions.all(): + auth.subscription_cancel(subscription.reference_id, subscription.subscription_id) + subscription.delete() + info.delete() + + + # - changing the status to CLOSED + self.status = Account.Statuses.closed + + # - removing and stopping the indexes for the account + for index in self.indexes.all(): + self.drop_index(index) + + # - notify + # send_notification(//close account) + + # - FIXME: handle authorize net errors! + + + self.save() + + def drop_index(self, index): + client = ApiClient(self.get_private_apiurl()) + client.delete_index(index.name) + + def apply_package(self, package): + self.package = package + + self.configuration = package.configuration + + def update_apikey(self): + self.apikey = generate_apikey(self.id) + + def get_private_apikey(self): + return self.apikey.split('-', 1)[1] + + def get_public_apikey(self): + return self.apikey.split('-', 1)[0] + + def get_private_apiurl(self): + return 'http://:%s@%s.api.indextank.com' % (self.get_private_apikey(), self.get_public_apikey()) + + def get_public_apiurl(self): + return 'http://%s.api.indextank.com' % self.get_public_apikey() + + class Meta: + db_table = 'storefront_account' + +class AccountPayingInformation(models.Model): + account = models.ForeignKey('Account', related_name='payment_informations') + + first_name = models.CharField(max_length=50, null=True) + last_name = models.CharField(max_length=50, null=True) + address = models.CharField(max_length=60, null=True) + city = models.CharField(max_length=60, null=True) + state = models.CharField(max_length=2, null=True) + zip_code = models.CharField(max_length=60, null=True) + country = models.CharField(max_length=2, null=True) + + company = models.CharField(max_length=50, null=True) + + credit_card_last_digits = models.CharField(max_length=4, null=True) + contact_email = models.EmailField(max_length=255, null=True) + + #custom subscription + monthly_amount = models.DecimalField(max_digits=8, decimal_places=2, null=True) + subscription_status = models.CharField(max_length=30, null=True) + subscription_type = models.CharField(max_length=30, null=True) + + + class Meta: + db_table = 'storefront_accountpayinginformation' + + +class PaymentSubscription(models.Model): + account = models.ForeignKey('AccountPayingInformation', related_name='subscriptions') + + # authorizenet id + subscription_id = models.CharField(max_length=20, null=False, blank=False) + # indextank id + reference_id = models.CharField(max_length=13, null=False, blank=False) + + amount = models.DecimalField(max_digits=8, decimal_places=2, null=False) + + # Frequency + start_date = models.DateTimeField() + frequency_length = models.IntegerField(null=False) + frequency_unit = models.CharField(max_length=10, null=False, blank=False) + + class Meta: + db_table = 'storefront_paymentsubscription' + + +class EffectivePayment(models.Model): + account = models.ForeignKey('AccountPayingInformation', related_name='payments') + + transaction_date = models.DateTimeField() + + # authorizenet data + transaction_id = models.CharField(max_length=12, null=False, blank=False) + customer_id = models.CharField(max_length=8, null=False, blank=False) + transaction_message = models.CharField(max_length=300, null=True) + subscription_id = models.CharField(max_length=20, null=False, blank=False) + subscription_payment_number = models.IntegerField(null=False) + first_name = models.CharField(max_length=50, null=False, blank=False) + last_name = models.CharField(max_length=50, null=False, blank=False) + address = models.CharField(max_length=60, null=True) + city = models.CharField(max_length=60, null=True) + state = models.CharField(max_length=2, null=True) + zip_code = models.CharField(max_length=60, null=True) + country = models.CharField(max_length=2, null=True) + company = models.CharField(max_length=50, null=True) + + # Inherited data (from account information + credit_card_last_digits = models.CharField(max_length=4, null=False, blank=False) + contact_email = models.EmailField(max_length=255) + + amount = models.DecimalField(max_digits=8, decimal_places=2, null=False) + + class Meta: + db_table = 'storefront_effectivepayment' + +class DataSet(models.Model): + name = models.CharField(null=True, max_length=50, unique=True) + code = models.CharField(null=True, max_length=15, unique=True) + filename = models.CharField(null=True, max_length=100, unique=True) + size = models.IntegerField(default=0) + + class Meta: + db_table = 'storefront_dataset' + +class IndexPopulation(models.Model): + index = models.ForeignKey('Index', related_name='datasets') + dataset = models.ForeignKey('DataSet', related_name='indexes') + time = models.DateTimeField() + populated_size = models.IntegerField(default=0) + + status = models.CharField(max_length=50,null=True) + + class Statuses: + created = 'CREATED' + populating = 'POPULATING' + finished = 'FINISHED' + + class Meta: + db_table = 'storefront_indexpopulation' + + +class Index(models.Model): + account = models.ForeignKey('Account', related_name='indexes') + code = models.CharField(null=True, max_length=22, unique=True) + name = models.CharField(max_length=50) + language_code = models.CharField(max_length=2) + creation_time = models.DateTimeField() + + analyzer_config = models.TextField(null=True) + configuration = models.ForeignKey('IndexConfiguration', null=True) + public_api = models.BooleanField(default=False, null=False) + + status = models.CharField(max_length=50) + + deleted = models.BooleanField(default=False, null=False) + + class States: + new = 'NEW' + live = 'LIVE' + hibernate_requested = 'HIBERNATE_REQUESTED' + hibernated = 'HIBERNATED' + waking_up = 'WAKING_UP' + + def get_json_for_analyzer(self): + if self.analyzer_config is None: + return None + configuration = json.loads(self.analyzer_config) + final_configuration = {} + + if configuration.has_key('per_field'): + per_field_final = {} + per_field = configuration.get('per_field') + for field in per_field.keys(): + per_field_final[field] = Index.get_analyzer(per_field[field]) + final_configuration['perField'] = per_field_final + final_configuration['default'] = Index.get_analyzer(per_field.get('default')) + else: + final_configuration = Index.get_analyzer(configuration) + + return final_configuration + + @classmethod + def get_analyzer(cls, configuration): + analyzer_map = {} + code = configuration.get('code') + if code is None: + raise ValueError('Analyzer configuration has no "code" key') + + try: + analyzer = AnalyzerComponent.objects.get(code=code) + except AnalyzerComponent.DoesNotExist: + raise ValueError('Analyzer configuration "code" key doesn\'t match any analyzers') + + analyzer_map['factory'] = analyzer.factory + analyzer_map['configuration'] = json.loads(analyzer.config) + + if configuration.has_key('filters'): + filters_list = [] + for filter in configuration.get('filters'): + filters_list.append(Index.get_analyzer(filter)) + analyzer_map['configuration']['filters'] = filters_list + + return analyzer_map + +# allows_adds = models.BooleanField(null=False,default=True) +# allows_queries = models.BooleanField(null=False,default=True) + + # index creation data +# allows_snippets = models.BooleanField() +# +# allows_autocomplete = models.BooleanField(default=True) +# autocomplete_type = models.models.CharField(max_length=10, null=True) # NEW +# +# allows_faceting = models.BooleanField() +# facets_bits = models.IntegerField(null=True) # NEW +# +# max_variables = models.IntegerField(null=False) # NEW +# +# max_memory_mb = models.IntegerField(null=False) # NEW +# rti_documents_number = models.IntegerField(null=False) # NEW + + # statistics + current_size = models.FloatField(default=0) + current_docs_number = models.IntegerField(default=0) + queries_per_day = models.FloatField(default=0) + + #demo + base_port = models.IntegerField(null=True) + + def __repr__(self): + return 'Index (%s):\n\tname: %s\n\tcode: %s\n\tcreation_time: %s\n\tconfiguration: %s\n\taccount\'s package: %s\ncurrent deploys: %r' % (self.id, self.name, self.code, self.creation_time, self.configuration.description, self.account.package.name, self.deploys.all()) + + def is_populating(self): + for population in self.datasets.all(): + if not population.status == IndexPopulation.Statuses.finished: + return True + return False + + def is_demo(self): + return self.name == 'DemoIndex' and self.datasets.count() > 0 + + + def is_ready(self): + ''' + Returns True if the end-user can use the index. + (this means for read and write, and it's meant to + be shown in the storefront page). Internally, this + means that at least one deployment for this index + is readable, and at least one is writable. + ''' + return self.is_writable() and self.is_readable() + + def is_hibernated(self): + return self.status in (Index.States.hibernated, Index.States.waking_up) + + def is_writable(self): + ''' + Returns true if there's at least one index that can be written. + ''' + for deploy in self.deploys.all(): + if deploy.is_writable(): + return True + + def is_readable(self): + ''' + Returns true if there's at least one index that can be read. + ''' + for deploy in self.deploys.all(): + if deploy.is_readable(): + return True + + def populate_for_account(self, account): + self.account = account + self.configuration = account.configuration + if account.default_analyzer is not None: + self.analyzer_config = account.default_analyzer.configuration + + def searchable_deploy(self): + '''Returns a single deploy that can be used to search. If no deploy is searcheable + it returns None. Note that if more than one deploy is searcheable, there are no warranties + of wich one will be returned.''' + # TODO : should change once deploy roles are implemented + ds = self.deploys.all() + ds = [d for d in ds if d.is_readable()] + return ds[0] if ds else None + + def indexable_deploys(self): + '''Returns the list of all deploys that should be updated (adds/updates/deletes/etc)''' + # TODO : should change once deploy roles are implemented + ds = self.deploys.all() + ds = [d for d in ds if d.is_writable()] + return ds + + def get_functions_dict(self): + return dict((str(f.name), f.definition) for f in self.scorefunctions.all()) + + def get_debug_info(self): + info = 'Index: %s [%s]\n' % (self.name, self.code) +\ + 'Account: %s\n' % self.account.user.email +\ + 'Deploys:\n' + for d in self.deploys.all(): + info += ' [deploy:%d] %s on [worker:%s] %s:%s' % (d.id, d.status, d.worker.id, d.worker.wan_dns, d.base_port) + return info + + def update_status(self, new_status): + print 'updating status %s for %r' % (new_status, self) + logger.debug('Updating status to %s for idnex %r', new_status, self) + Index.objects.filter(id=self.id).update(status=new_status) + + def mark_deleted(self): + Index.objects.filter(id=self.id).update(deleted=True) + + class AutocompleTypes: + created = 'DOCUMENTS' + initializing = 'QUERIES' + + class Meta: + unique_together = (('code','account'),('name','account')) + db_table = 'storefront_index' + +class Insight(models.Model): + index = models.ForeignKey(Index, related_name='insights') + code = models.CharField(max_length=30, null=False) + data = models.TextField(null=False) + last_update = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = ('index', 'code') + db_table = 'storefront_insight' + +class IndexConfiguration(models.Model): + description = models.TextField(null=False) + creation_date = models.DateField() + json_configuration = models.TextField(null=False) + + def __repr__(self): + j_map = json.loads(self.json_configuration) + mapStr = '{\n' + for m in j_map: + mapStr += '\t\t%s -> %s\n' % (m, j_map[m]) + mapStr += '\t}\n' + return 'IndexConfiguration (%s):\n\tdescription: %s\n\tcreation_date: %s\n\tjson_configuration: %s\n' % (self.id, self.description, str(self.creation_date), mapStr) + + def __str__(self): + return '(description: %s; creation_date: %s; json_configuration: %s)' % (self.description, str(self.creation_date), self.json_configuration) + + def get_data(self): + map = json.loads(self.json_configuration) + data = {} + for k,v in map.items(): + data[str(k)] = v + data['ram'] = data.get('xmx',0) + data.get('bdb_cache',0) + return data + def set_data(self, data): + self.json_configuration = json.dumps(data) + + class Meta: + db_table = 'storefront_indexconfiguration' + +class Analyzer(models.Model): + account = models.ForeignKey('Account', related_name='analyzers') + code = models.CharField(max_length=64) + configuration = models.TextField() + + class Meta: + db_table = 'storefront_analyzer' + +class AnalyzerComponent(models.Model): + code = models.CharField(max_length=15, unique=True) + name = models.CharField(max_length=200) + description = models.CharField(max_length=1000) + config = models.TextField(null=False,blank=False) + factory = models.CharField(max_length=200) + type = models.CharField(max_length=20) + enabled = models.BooleanField() + + class Types: + tokenizer = 'TOKENIZER' + filter = 'FILTER' + + class Meta: + db_table = 'storefront_analyzercomponent' + +def create_analyzer(code, name, config, factory, type, enabled): + analyzer = None + try: + analyzer = AnalyzerComponent.objects.get(code=code) + + analyzer.name = name + analyzer.config = config + analyzer.factory = factory + analyzer.type = type + analyzer.enabled = enabled + + analyzer.save() + except AnalyzerComponent.DoesNotExist: + analyzer = AnalyzerComponent(code=code, name=name, config=config, type=type, enabled=enabled) + analyzer.save() + +class Package(models.Model): + ''' + Packages define what a user have the right to when creating an Account and how does the indexes in that Account + behave. + There are two sections for what the Package configures. A fixed section with the control and limits information + that is used by nebu, storefront or api (base_price, index_max_size, searches_per_day, max_indexes). A dynamic + section that is handled by the IndexConfiguration object. The information of that section is passed to the IndexEngine + as it is and handled by it. + ''' + name = models.CharField(max_length=50) + code = models.CharField(max_length=30) + base_price = models.FloatField() + index_max_size = models.IntegerField() + searches_per_day = models.IntegerField() + max_indexes = models.IntegerField() + + configuration = models.ForeignKey('IndexConfiguration', null=True) + + def __repr__(self): + return 'Package (%s):\n\tname: %s\n\tcode: %s\n\tbase_price: %.2f\n\tindex_max_size: %i\n\tsearches_per_day: %i\n\tmax_indexes: %i\n' % (self.id, self.name, self.code, self.base_price, self.index_max_size, self.searches_per_day, self.max_indexes) + + def __str__(self): + return '(name: %s; code: %s; base_price: %.2f; index_max_size: %i; searches_per_day: %i; max_indexes: %i)' % (self.name, self.code, self.base_price, self.index_max_size, self.searches_per_day, self.max_indexes) + + def max_size_mb(self): + return self.index_max_size * settings.INDEX_SIZE_RATIO + class Meta: + db_table = 'storefront_package' + +class ScoreFunction(models.Model): + index = models.ForeignKey(Index, related_name='scorefunctions') + name = models.IntegerField(null=False) # TODO the java API expects an int. But a String may be nicer for name. + definition = models.CharField(max_length=255, blank=False, null=True) + + class Meta: + db_table = 'storefront_scorefunction' + unique_together = (('index','name')) + + +def create_configuration(description, data, creation_date=None): + configuration = IndexConfiguration() + configuration.description = description + configuration.creation_date = creation_date or datetime.now() + configuration.json_configuration = json.dumps(data) + + configuration.save() + return configuration + +def create_package(code, name, base_price, index_max_size, searches_per_day, max_indexes, configuration_map): +# The configuration_map will only be considered if the package if new or if it didn't already have a configuration + + package = None + try: + package = Package.objects.get(code=code) + + package.name = name + package.base_price = base_price + package.index_max_size = index_max_size + package.searches_per_day = searches_per_day + package.max_indexes = max_indexes + + if not package.configuration: + package.configuration = create_configuration('package:' + code, configuration_map) + + package.save() + except Package.DoesNotExist: + configuration = create_configuration('package:' + code, configuration_map) + package = Package(code=code, base_price=base_price, index_max_size=index_max_size, searches_per_day=searches_per_day, max_indexes=max_indexes, configuration=configuration) + package.save() + +def create_provisioner(name, token, email, plans): + provisioner = None + try: + provisioner = Provisioner.objects.get(name=name) + except Provisioner.DoesNotExist: + provisioner = Provisioner() + provisioner.name = name + provisioner.token = token + provisioner.email = email + provisioner.save() + + provisioner.plans.all().delete() + for plan, code in plans.items(): + pp = ProvisionerPlan() + pp.plan = plan + pp.provisioner = provisioner + pp.package = Package.objects.get(code=code) + pp.save() + + +class AccountMovement(models.Model): + account = models.ForeignKey('Account', related_name='movements') + class Meta: + db_table = 'storefront_accountmovement' + +class ActionLog(models.Model): + account = models.ForeignKey('Account', related_name='actions') + class Meta: + db_table = 'storefront_actionlog' + +class PFUser(models.Model): + user = models.ForeignKey(User, unique=True) + account = models.OneToOneField('Account', related_name='user') + email = models.EmailField(unique=True, max_length=255) + change_password = models.BooleanField(default=False, null=False) + class Meta: + db_table = 'storefront_pfuser' + + + +MAX_USABLE_RAM_PERCENTAGE = 0.9 +# Nebulyzer stuff +class Worker(models.Model): + ''' + Describes an amazon ec2 instance. + ''' + instance_name = models.CharField(max_length=50,null=False,blank=False) + lan_dns = models.CharField(max_length=100,null=False,blank=False) + wan_dns = models.CharField(max_length=100,null=False,blank=False) + status = models.CharField(max_length=30) + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) + #physical memory in MegaBytes + ram = models.IntegerField() + + class States: + created = 'CREATED' + initializing = 'INITIALIZING' + updating = 'UPDATING' + controllable = 'CONTROLLABLE' + decommissioning = 'DECOMMISSIONING' + dying = 'DYING' + dead = 'DEAD' + + class Meta: + db_table = 'storefront_worker' + + def get_usable_ram(self): + '''Return the amount of ram that can be used in this machine for + indexengines. It's calculated as a fixed percentage of the physical + ram. Value returned in MegaBytes''' + return MAX_USABLE_RAM_PERCENTAGE * self.ram + + def get_used_ram(self): + xmx = self.deploys.aggregate(xmx=Sum('effective_xmx'))['xmx'] + bdb = self.deploys.aggregate(bdb=Sum('effective_bdb'))['bdb'] + if xmx == None: + xmx = 0 + if bdb == None: + bdb = 0 + return xmx + bdb + + def is_assignable(self): + return self.status != Worker.States.decommissioning + + def is_ready(self): + return self.status in [Worker.States.controllable, Worker.States.decommissioning] + + def __repr__(self): + return 'Worker (%s):\n\tinstance_name: %s\n\tlan_dns: %s\n\twan_dns: %s\n\tstatus: %s\n\ttimestamp: %s\n\tram: %s\n' %(self.pk, self.instance_name, self.lan_dns, self.wan_dns, self.status, self.timestamp, self.ram) + +class Service(models.Model): + name = models.CharField(max_length=50,null=False,blank=False) + type = models.CharField(max_length=50,null=True,blank=True ) + host = models.CharField(max_length=100,null=False,blank=False) + port = models.IntegerField() + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) + + class Meta: + db_table = 'storefront_service' + + def __repr__(self): + return 'Service (%s):\n\tname: %s\n\ttype: %s\n\thost: %s\n\tport: %s\n\ttimestamp: %s\n' % (self.pk, self.name, self.type, self.host, self.port, self.timestamp) + + +# CPU Stats +class WorkerMountInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="disk_infos") + timestamp = models.DateTimeField() + + mount = models.CharField(max_length=100,null=False,blank=False) + available = models.IntegerField() + used = models.IntegerField() + + class Meta: + db_table = 'storefront_workermountinfo' + + +class WorkerLoadInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="load_infos") + timestamp = models.DateTimeField() + + load_average = models.FloatField() + + class Meta: + db_table = 'storefront_workerloadinfo' + +class WorkerIndexInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="indexes_infos") + timestamp = models.DateTimeField() + + deploy = models.ForeignKey('Deploy', related_name="index_infos") + used_disk = models.IntegerField() + used_mem = models.IntegerField() + + class Meta: + db_table = 'storefront_workerindexinfo' + + +class Deploy(models.Model): + ''' + Describes a deploy of an index on a worker, and it's status. + The idea is that an index can be moving from one worker to another, + so queries and indexing requests have to be mapped to one or more + index engines. + ''' + index = models.ForeignKey(Index, related_name="deploys") + worker = models.ForeignKey(Worker, related_name="deploys") + base_port = models.IntegerField() + status = models.CharField(max_length=30) + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) # Last time we updated this deploy. + parent = models.ForeignKey('self', related_name='children', null=True) # For moving deploys. + effective_xmx = models.IntegerField() + effective_bdb = models.IntegerField() + dying = models.BooleanField(default=False, null=False) + + # TODO add role fields + #searching_role = models.BooleanField() + #indexing_role = models.BooleanField() + + def __repr__(self): + return 'Deploy (%s):\n\tparent deploy: %s\n\tindex code: %s\n\tstatus: %s\n\tworker ip: %s\n\tport: %d\n\teffective_xmx: %d\n\teffective_bdb: %d\n' % (self.id, self.parent_id, self.index.code, self.status, self.worker.lan_dns, self.base_port, self.effective_xmx, self.effective_bdb) + + def __unicode__(self): + return "Deploy: %s on %s:%d" % (self.status, self.worker.lan_dns, self.base_port) + + def is_readable(self): + '''Returns true if a search can be performed on this deployment, and + the returned data is up to date''' + return self.status == Deploy.States.controllable or \ + self.status == Deploy.States.move_requested or \ + self.status == Deploy.States.moving or \ + (self.status == Deploy.States.recovering and not self.parent) + + def is_writable(self): + '''Returns True if new data has to be written to this deployment.''' + return self.status == Deploy.States.controllable or \ + self.status == Deploy.States.recovering or \ + self.status == Deploy.States.move_requested or \ + self.status == Deploy.States.moving + + def total_ram(self): + return self.effective_xmx + self.effective_bdb + + def update_status(self, new_status): + print 'updating status %s for %r' % (new_status, self) + logger.debug('Updating status to %s for deploy %r', new_status, self) + Deploy.objects.filter(id=self.id).update(status=new_status, timestamp=datetime.now()) + + def update_parent(self, new_parent): + logger.debug('Updating parent to %s for deploy %r', new_parent, self) + Deploy.objects.filter(id=self.id).update(parent=new_parent) + + class States: + created = 'CREATED' + initializing = 'INITIALIZING' + recovering = 'RECOVERING' + resurrecting = 'RESURRECTING' + controllable = 'CONTROLLABLE' + move_requested = 'MOVE_REQUESTED' + moving = 'MOVING' + decommissioning = 'DECOMMISSIONING' + + class Meta: + db_table = 'storefront_deploy' + +class BetaTestRequest(models.Model): + email = models.EmailField(unique=True, max_length=255) + site_url = models.CharField(max_length=200,null=False,blank=False) + summary = models.TextField(null=False,blank=False) + + request_date = models.DateTimeField(default=datetime.now) + status = models.CharField(max_length=50,null=True) + + class Meta: + db_table = 'storefront_betatestrequest' + +class BetaInvitation(models.Model): + password = models.CharField(max_length=20, null=True) + account = models.ForeignKey('Account', null=True) + assigned_customer = models.CharField(max_length=50, null=True) + beta_requester = models.ForeignKey('BetaTestRequest', null=True, related_name="invitation") + + invitation_date = models.DateTimeField(default=datetime.now) + forced_package = models.ForeignKey('Package', null=False) + + class Meta: + db_table = 'storefront_signupotp' + +class ContactInfo(models.Model): + name = models.CharField(max_length=64) + email = models.EmailField(unique=True, max_length=255) + request_date = models.DateTimeField(default=datetime.now) + source = models.CharField(max_length=64, null=True) + + class Meta: + db_table = 'storefront_contactinfo' + + + +class Provisioner(models.Model): + name = models.CharField(max_length=64) + token = models.CharField(max_length=64, null=False, blank=False) + email = models.EmailField(max_length=255) # contact info for the provisioner + + class Meta: + db_table = "storefront_provisioner" + +class ProvisionerPlan(models.Model): + plan = models.CharField(max_length=50) + provisioner = models.ForeignKey('Provisioner', related_name='plans') + package = models.ForeignKey('Package') + + class Meta: + db_table = "storefront_provisionerplan" + +class BlogPostInfo(models.Model): + title = models.CharField(max_length=200) + url = models.CharField(max_length=1024) + date = models.DateTimeField() + author = models.CharField(max_length=64) + + class Meta: + db_table = 'storefront_blogpost' + diff --git a/api/restapi.py b/api/restapi.py new file mode 100644 index 0000000..b6025c9 --- /dev/null +++ b/api/restapi.py @@ -0,0 +1,1647 @@ +import datetime + +from django.contrib.auth.management.commands.createsuperuser import is_valid_email +from django.http import HttpResponse, Http404, HttpResponseRedirect +from django.db import IntegrityError + +from models import Account, ScoreFunction, Package, Service + +from flaptor.indextank.rpc import ttypes +from views import mixpanel_event_track + +from restresource import ProvisionerResource, Resource, JsonResponse, non_empty_argument, int_argument, required_data, optional_data, int_querystring, json_querystring, querystring_argument, required_querystring_argument, check_public_api, get_index_param, get_index_param_or404, wakeup_if_hibernated, authorized_method + +from lib import encoder +from lib import mail +from lib.indextank.client import ApiClient + +import rpc +import re +from api import models +import time +import storage + +LOG_STORAGE_ENABLED = True + +""" Data validation and parsing functions """ +def _encode_utf8(s): + try: + return s.encode('utf-8') + except: + try: + str(s).decode('utf-8') + return str(s) + except: + try: + return str(s).encode('utf-8') + except: + return None + +def __validate_boolean(field_name): + def dovalidate(arg): + if type(arg) != bool: + return HttpResponse('"Invalid \\"%s\\" argument, it should be a json boolean"' % field_name, status=400) + return dovalidate + +def __validate_docid(docid): + """ + Validates that a document id is a string, a unicode, or an int (for backwards compatibility). + It can't be empty, nor longer than 1024 bytes. + Valid inputs + >>> __validate_docid("a") + >>> __validate_docid("\xc3\xb1") + >>> __validate_docid(u"\xc3\xb1") + >>> # for backwards compatibility + >>> __validate_docid(123) + >>> __validate_docid(0) + >>> __validate_docid(-1) + + Validate length + >>> __validate_docid("a"*1024) + >>> __validate_docid(u"a"*1024) + >>> # 512 2-byte chars are ok + >>> __validate_docid("\xc3\xb1"*512) + >>> e = __validate_docid("a"*1025) + >>> isinstance(e, HttpResponse) + True + >>> e = __validate_docid(u"\xc3"*1025) + >>> isinstance(e, HttpResponse) + True + >>> # 512 2-byte chars are not ok + >>> e = __validate_docid("\xc3\xb1"*513) + >>> isinstance(e, HttpResponse) + True + + Validate emptyness + >>> e = __validate_docid(" ") + >>> isinstance(e, HttpResponse) + True + >>> e = __validate_docid("") + >>> isinstance(e, HttpResponse) + True + >>> e = __validate_docid(" "*80) + >>> isinstance(e, HttpResponse) + True + + Validate not supported types + >>> e = __validate_docid(80.0) + >>> isinstance(e, HttpResponse) + True + >>> e = __validate_docid([1,2,3]) + >>> isinstance(e, HttpResponse) + True + >>> e = __validate_docid({"a":"b"}) + >>> isinstance(e, HttpResponse) + True + + Validate None + >>> e = __validate_docid(None) + >>> isinstance(e, HttpResponse) + True + + + """ + if type(docid) in [int, long]: + docid = str(docid) + + if not type(docid) in [str,unicode]: + return HttpResponse('"Invalid docid, it should be a String."', status=400) + + if docid.strip() == '': + return HttpResponse('"Invalid docid, it shouldnt be empty."', status=400) + + udocid = _encode_utf8(docid) + if len(udocid) > 1024: + return HttpResponse('"Invalid docid, it shouldnt be longer than 1024 bytes. It was %d"'%len(udocid), status=400) + + +def __parse_docid(docid): + if not type(docid) in [str,unicode]: + docid = str(docid) + return docid + +def __str_is_integer(val): + try: + i = int(val) + return True + except: + return False + +def __validate_fields(fields): + """ + Validates that a document fields is a dictionary with string (or unicode) keys and string (or unicode) values. + The only exception is 'timestamp', that can be an int or a string representation of an int. + + The sum of the sizes of all the field values can not be bigger than 100kb. + Returns nothing, unless a validation error was found. In that case, it returns an HttpResponse with the error as body. + + Validate documents without errors + >>> __validate_fields({'field1':'value1', 'field2':'value2'}) + >>> __validate_fields({'text':'', 'title':u''}) + >>> __validate_fields({'text':u'just one field'}) + >>> __validate_fields({'field1':'value1 and value2 and value 3 or value4', 'field2':'value2', 'field3':'123'}) + + Validate documents with errors on field values (int, float, array, dict and None) + As the input for this method comes from json, there can't be objects as values + >>> __validate_fields({'text': 123}) + >>> __validate_fields({'text': 42.8}) + + >>> e = __validate_fields({'text': ['123']}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_fields({'text': None}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_fields({'text': {'k':'v'}}) + >>> isinstance(e,HttpResponse) + True + + Validate documents with errors on field names (int, float and None) + As the input for this method comes from json, there can't be objects as keys + >>> e = __validate_fields({None: '123'}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_fields({123: 'text'}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_fields({42.5: 'value'}) + >>> isinstance(e,HttpResponse) + True + + Validate timestamps + >>> __validate_fields({'timestamp': 123, 'field1':'value1'}) + >>> __validate_fields({'timestamp': '123', 'fieldN':'valueN'}) + >>> __validate_fields({'timestamp': -123, 'field1':'value1'}) + >>> __validate_fields({'timestamp': '-123', 'fieldN':'valueN'}) + >>> e = __validate_fields({'timestamp': 'something', 'fieldN': 'valueN'}) + >>> isinstance(e,HttpResponse) + True + + Validate document size + >>> __validate_fields({'text': 'a'*1024}) + >>> __validate_fields({'text': 'a'*1024, 'title':'b'*1024}) + >>> # this is the boundary case for 1 field + >>> __validate_fields({'text': 'a'*1024*100}) + >>> # a boundary case for 2 fields .. 1 * 9 * 1024 + 10 * 9 * 1024 + >>> __validate_fields({'text': 'a'*9*1024, 'title': 'a b c d e '*9*1024}) + >>> # a boundary case for 2-byte chars on fields + >>> __validate_fields({'text': '\xc3\xb1'*100*512}) + >>> # this is an error case for 1 field + >>> e = __validate_fields({'text': 'a'*1024*101}) + >>> isinstance(e,HttpResponse) + True + >>> # this is an error case for 2 fields + >>> e = __validate_fields({'text': 'a'*50*1024, 'title': 'a b c d e '*9*1024}) + >>> isinstance(e,HttpResponse) + True + >>> # this is an error case for 10 fields .. 10 * ( 1024 * 11 ) + >>> fields = {} + >>> fields.update([("text%d"%i,"123 456 789"*1024) for i in range(0,10)]) + >>> e = __validate_fields(fields) + >>> isinstance(e,HttpResponse) + True + >>> # this is an error case for 2-byte chars on fields + >>> e = __validate_fields({"text":"\xc3\xb1"*100*513}) + >>> isinstance(e,HttpResponse) + True + >>> # disallow no fields + >>> e = __validate_fields({}) + >>> isinstance(e,HttpResponse) + True + + >>> # disallow None, Arrays, Numbers and Strings + >>> e = __validate_fields(None) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_fields([1, 2, 3]) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_fields(123) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_fields("this is some text") + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_fields(True) + >>> isinstance(e,HttpResponse) + True + + """ + + if not fields: + return HttpResponse('"At least one field is required. If you\'re trying to delete the document, you should use the delete api"', status=400) + + + if not isinstance(fields, dict): + return HttpResponse('"fields should be a JSON Object, e.g: {\'text\': \'something\'} "', status=400) + + + for k, v in fields.iteritems(): + # timestamp gets special treatment, it should be an integer + if 'timestamp' == k: + if not type(v) == int and not (type(v) == str and __str_is_integer(v)): + return HttpResponse('"Invalid timestamp: %s. It should be an integer."' % fields['timestamp'], status=400) + continue + + # any other key should be a string or unicode + if not isinstance(k,str) and not isinstance(k,unicode): + return HttpResponse('"Name for field %s is not a String"' % (k), status=400) + + + if isinstance(v,int) or isinstance(v,float): + v = str(v) + if not isinstance(v,str) and not isinstance(v,unicode): + return HttpResponse('"Value for field %s is not a String nor a number"' % (k), status=400) + + ev = _encode_utf8(v) + ek = _encode_utf8(k) + + if ek is None: + return HttpResponse('"Invalid name for field %s"' % (k), status=400) + if ev is None: + return HttpResponse('"Invalid content for field %s: %s"' % (k, v), status=400) + + # verify document size + doc_size = sum(map(lambda (k,v) : len(v) if type(v) in [str, unicode] else 0, fields.iteritems())) + if doc_size > 1024 * 100: + return HttpResponse('"Invalid document size. It shouldn\'t be bigger than 100KB. Got %d bytes"' % (doc_size), status=400) + + +def __parse_fields(fields): + if 'timestamp' not in fields: + fields['timestamp'] = str(int(time.time())) + for k, v in fields.iteritems(): + fields[k] = _encode_utf8(v) + return fields + +def __validate_categories(categories): + """ + Validates that a document categories is a dictionary with string (or unicode) keys and string (or unicode) values. + Returns nothing, unless a validation error was found. In that case, it returns an HttpResponse with the error as body + + Validate categories without errors + >>> __validate_categories({'field1':'value1', 'field2':'value2'}) + >>> __validate_categories({'text':'', 'title':u''}) + >>> __validate_categories({'text':u'just one field'}) + >>> __validate_categories({'field1':'value1 and value2 and value 3 or value4', 'field2':'value2', 'field3':'123'}) + + Validate documents with errors on category values (int, float, array, dict and None) + As the input for this method comes from json, there can't be objects as values + >>> e = __validate_categories({'text': 123}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_categories({'text': 42.8}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_categories({'text': ['123']}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_categories({'text': None}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_categories({'text': {'k':'v'}}) + >>> isinstance(e,HttpResponse) + True + + Validate documents with errors on category names (int, float and None) + As the input for this method comes from json, there can't be objects as keys + >>> e = __validate_categories({None: '123'}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_categories({123: 'text'}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_categories({42.5: 'value'}) + >>> isinstance(e,HttpResponse) + True + """ + for k, v in categories.iteritems(): + ev = _encode_utf8(v) + ek = _encode_utf8(k) + + if not isinstance(k,str) and not isinstance(k,unicode): + return HttpResponse('"Name for category %s is not a String"' % (k), status=400) + + if not isinstance(v,str) and not isinstance(v,unicode): + return HttpResponse('"Value for category %s is not a String"' % (k), status=400) + + if ek is None: + return HttpResponse('"Invalid name for category %s"' % (k), status=400) + if ev is None: + return HttpResponse('"Invalid content for category %s: %s"' % (k, v), status=400) + +def __parse_categories(categories): + parsed = {} + for k, v in categories.iteritems(): + parsed[k] = _encode_utf8(v).strip() + return parsed + + +def __validate_variables(variables): + """ + Validates that variables for a document is a dict with string representations of positive ints as keys and floats as values + + variable mappings without errors + >>> __validate_variables({"4":8, "5":2.5}) + >>> __validate_variables({"1":2}) + >>> __validate_variables({"10":2}) + >>> # the next line is kinda valid. "2" can be parsed to float. + >>> __validate_variables({"1":"2"}) + + + variable mappings with errors on keys + >>> e = __validate_variables({"-10":2}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_variables({"var1":2}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_variables({"10":2, "v3":4.5}) + >>> isinstance(e,HttpResponse) + True + + variable mappings with errors on values + >>> e = __validate_variables({"1":[2.5]}) + >>> isinstance(e,HttpResponse) + True + >>> e = __validate_variables({"1":2, "2":{2.5:2}}) + >>> isinstance(e,HttpResponse) + True + + """ + for k, v in variables.iteritems(): + if not k.isdigit(): + return HttpResponse('"Invalid variable index: %s. It should be integer."' % k, status=400) + try: + float(v) + except (ValueError, TypeError): + return HttpResponse('"Invalid variable value for index %s: %s. It should be a float"' % (k,v), status=400) + + +def __parse_variables(variables): + parsed = {} + for k, v in variables.iteritems(): + parsed[int(k)] = float(v) + return parsed + +def __validate_document(document): + if type(document) is not dict: + return HttpResponse('"Document should be a JSON object"', status=400) + if 'docid' not in document: + return HttpResponse('"Document should have a docid attribute"', status=400) + if 'fields' not in document: + return HttpResponse('"Document should have a fields attribute"', status=400) + response = None + response = response or __validate_docid(document['docid']) + response = response or __validate_fields(document['fields']) + if 'variables' in document: + response = response or __validate_variables(document['variables']) + if 'categories' in document: + response = response or __validate_categories(document['categories']) + if response: + return response + +def __parse_document(document): + document['docid'] = __parse_docid(document['docid']) + document['fields'] = __parse_fields(document['fields']) + if 'variables' in document: + document['variables'] = __parse_variables(document['variables']) + if 'categories' in document: + document['categories'] = __parse_categories(document['categories']) + return document + +def __validate_query(query): + """ + Validates that a query is a string or a unicode. + It can't be empty. + Valid inputs + >>> __validate_query("a") + >>> __validate_query("\xc3\xb1") + >>> __validate_query(u"\xc3\xb1") + + Validate emptyness + >>> e = __validate_query(" ") + >>> isinstance(e, HttpResponse) + True + >>> e = __validate_query("") + >>> isinstance(e, HttpResponse) + True + >>> e = __validate_query(" "*80) + >>> isinstance(e, HttpResponse) + True + + Validate not supported types + >>> e = __validate_query(80.0) + >>> isinstance(e, HttpResponse) + True + >>> e = __validate_query([1,2,3]) + >>> isinstance(e, HttpResponse) + True + >>> e = __validate_query({"a":"b"}) + >>> isinstance(e, HttpResponse) + True + + Validate None + >>> e = __validate_query(None) + >>> isinstance(e, HttpResponse) + True + """ + + if query is None: + return HttpResponse('"Invalid query. It cannot be NULL"', status=400) + + if type(query) not in [str,unicode]: + return HttpResponse('"Invalid query. It MUST be a String"', status=400) + + if query.strip() == '' : + return HttpResponse('"Invalid query. It cannot be a empty"', status=400) + +def __parse_query(query): + return _encode_utf8(query.lower()) + +""" Argument validation decorators """ +required_index_name = non_empty_argument('index_name', 'Index name cannot be empty') + +""" Data validation decorators """ +required_categories_data = required_data('categories', __validate_categories, __parse_categories) +required_variables_data = required_data('variables', __validate_variables, __parse_variables) +required_fields_data = required_data('fields', __validate_fields, __parse_fields) +required_docid_data = required_data('docid', __validate_docid, __parse_docid) +optional_public_search_data = optional_data('public_search', __validate_boolean('public_search')) +optional_docid_data = optional_data('docid', __validate_docid, __parse_docid) +required_definition_data = required_data('definition') +required_query_data = required_data('query', __validate_query, __parse_query) +optional_variables_data = optional_data('variables', __validate_variables, __parse_variables) +required_integer_function = int_argument('function', 'Function number should be a non-negative integer') + +def required_documents(func): + def decorated(self, request, data, **kwargs): + if type(data) is list: + if not data: + return HttpResponse('"Invalid batch insert. At least one document is required"', status=400) + for i in xrange(len(data)): + response = __validate_document(data[i]) + if response: + response.content = '"Invalid batch insert, in document #%d of %d: %s"' % (i+1, len(data), response.content[1:-1]) + return response + data[i] = __parse_document(data[i]) + kwargs['batch_mode'] = True + kwargs['documents'] = data + else: + response = __validate_document(data) + if response: + return response + kwargs['batch_mode'] = False + kwargs['documents'] = [__parse_document(data)] + return func(self, request, data=data, **kwargs) + return decorated + +def required_docids(func): + def decorated(self, request, data, **kwargs): + + if not data: + docids = request.GET.getlist('docid') + if docids: + if len(docids) > 1: + data = [] + for docid in docids: + data.append({'docid':docid}) + else: + data = {'docid':docids[0]} + else: + return HttpResponse('"If no body is given, you should include a docid argument in the querystring"', status=400) + + if type(data) is list: + if not data: + return HttpResponse('"Invalid batch delete. At least one docid is required"', status=400) + for i in xrange(len(data)): + if not data[i].has_key('docid'): + return HttpResponse('"Invalid batch delete. Document #%d of %d doesn\'t have a docid parameter"' % (i+1, len(data)), status=400) + + response = __validate_docid(data[i]['docid']) + if response: + response.content = '"Invalid batch delete, in docid #%d of %d: %s"' % (i+1, len(data), response.content[1:-1]) + return response + data[i]['docid'] = __parse_docid(data[i]['docid']) + kwargs['bulk_mode'] = True + kwargs['documents'] = data + else: + if 'docid' in data: + response = __validate_docid(data['docid']) + if response: + response.content + return response + data['docid'] = __parse_docid(data['docid']) + kwargs['bulk_mode'] = False + kwargs['documents'] = [data] + else: + return HttpResponse('"Argument docid is required in the request body"', status=400) + return func(self, request, data=data, **kwargs) + return decorated + +""" Shared bulk delete code """ +def delete_docs_from_index(resource, index, documents): + """ + resource: A Resource (from restresource.py). Needed just to LOG errors + index: The index to delete documents from. Every deploy for that index will be hit. + documents: list of documents to delete. **Need** 'docid' on each of them. + """ + indexers = rpc.get_indexer_clients(index) + + responses = [] + + for doc in documents: + ret = {'deleted': True} + try: + for indexer in indexers: + indexer.delDoc(doc['docid']) + except Exception: + resource.logger.exception('"Failed to delete %s on %s (%s)', doc['docid'], index.code, index.name) + resource.error_logger.exception('"Failed to delete %s on %s (%s)', doc['docid'], index.code, index.name) + ret['deleted'] = False + ret['error'] = '"Currently unable to delete the requested document"' + + responses.append(ret) + return responses + +""" Util functions for request processing """ +def metadata_for_index(index): + return dict(code=index.code, creation_time=index.creation_time.isoformat(), started=index.is_ready(), size=index.current_docs_number, public_search=index.public_api, status=index.status) + +def build_logrecord_for_add(index, document): + return ttypes.LogRecord(index_code=index.code, + docid=document['docid'], + deleted=False, + fields=document['fields'], + variables=document.get('variables', {}), + categories=document.get('categories', {})) + +def build_logrecord_for_delete(index, document): + return ttypes.LogRecord(index_code=index.code, + docid=document['docid'], + deleted=True) +log_writers = None +log_addresses = None + + +def send_log_storage_batch(resource, index, records): + global log_addresses + global log_writers + try: + storage_services = Service.objects.filter(name='storage') + if storage_services: + addresses = set((service.host, int(service.port), service.type) for service in storage_services) + if addresses != log_addresses: + log_writers = [(rpc.getReconnectingLogWriterClient(h,p), t == 'optional') for h,p,t in addresses] + log_addresses = addresses + for writer, optional in log_writers: + try: + writer.send_batch(ttypes.LogBatch(records=records)) + except: + if optional: + resource.logger.exception('Optional storage failed to receive batch - IGNORING. %d records for index %s', len(records), index.code) + resource.error_logger.exception('Optional storage failed to receive batch - IGNORING. %d records for index %s', len(records), index.code) + else: + raise + + return True + else: + resource.logger.error('No storage services found. %d records for index %s', len(records), index.code) + resource.error_logger.error('No storage services found. %d records for index %s', len(records), index.code) + return False + except: + resource.logger.exception('Error sending batch to log storage. %d records for index %s', len(records), index.code) + resource.error_logger.exception('Error sending batch to log storage. %d records for index %s', len(records), index.code) + return False + + +""" + Version resource ====================================================== +""" +class Version(Resource): + authenticated = False + def GET(self, request, version): + return HttpResponse('"API V %s : Documentation"' % version) + +""" + Indexes resource ====================================================== +""" +class Indexes(Resource): + authenticated = True + def GET(self, request, version): + metadata = {} + for index in self.get_account().indexes.all(): + metadata[index.name] = metadata_for_index(index) + # 200 OK : json of the indexes metadata + return JsonResponse(metadata) + +""" + Index resource ====================================================== +""" +class Index(Resource): + authenticated = True + + # gets index metadata + @required_index_name + @get_index_param_or404 + def GET(self, request, version, index): + return JsonResponse(metadata_for_index(index)) + + # creates a new index for the given name + @required_index_name + @optional_public_search_data + def PUT(self, request, data, version, index_name, public_search=None): + account = self.get_account() + + indexes = account.indexes.filter(name=index_name) + if indexes.count() > 1: + self.logger.exception('Inconsistent state: more than one index with name %s', index_name) + self.error_logger.exception('Inconsistent state: more than one index with name %s', index_name) + + return HttpResponse('Unable to create/update your index. Please contact us.', status=500) + elif indexes.count() == 1: + if not public_search is None: + index = indexes[0] + index.public_api = public_search + index.save() + + # 204 OK: index already exists + return HttpResponse(status=204) + else: + current_count = account.indexes.filter(deleted=False).count() + max_count = account.package.max_indexes + + if not current_count < max_count: + msg = '"Unable to create. Too many indexes for the account (maximum: %d)"' % max_count + return HttpResponse(msg, status=409) #conflict + + # basic index data + try: + index = account.create_index(index_name, public_search) + except IntegrityError: + return HttpResponse(status=204) # already exists + + mail.report_new_index(index) + mixpanel_event_track('use', { 'plan' : account.package.code, 'use-type': 'index creation', 'distinct_id': account.user.email}) + + # 201 OK : index created + return JsonResponse(metadata_for_index(index), status=201) + + + @required_index_name + @get_index_param + def DELETE(self, request, data, version, index): + if not index: + return HttpResponse(status=204) + rpc.get_deploy_manager().delete_index(index.code) + mail.report_delete_index(index) + mixpanel_event_track('use', { 'plan' : index.account.package.code, 'use-type': 'index deletion', 'distinct_id': index.account.user.email}) + return HttpResponse() + + +""" + Document resource ====================================================== +""" +class Document(Resource): + authenticated = True + + @required_index_name + @required_querystring_argument('docid') + @get_index_param_or404 + def GET(self, request, version, index, docid): + self.logger.debug('id=%s', docid) + doc = storage.storage_get(index.code, docid) + if doc is None: + raise Http404 + self.set_message('Retrieved document %s' % docid) + return JsonResponse(doc) + + def _insert_document(self, index, indexers, document): + tdoc = ttypes.Document(fields=document['fields']) + docid = document['docid'] + variables = document.get('variables', {}) + categories = document.get('categories', {}) + + success = { 'added': True } + try: + for indexer in indexers: + indexer.addDoc(docid, tdoc, int(tdoc.fields['timestamp']), variables) + if categories: + indexer.updateCategories(docid, categories) + + except Exception, e: + self.logger.exception('Failed to index %s on %s (%s)', docid, index.code, index.name) + self.error_logger.exception('Failed to index %s on %s (%s)', docid, index.code, index.name) + success['added'] = False + success['error'] = '"Currently unable to index the requested document"' + + return success + + def _validate_variables(self, index, documents): + max_variables = index.configuration.get_data().get('max_variables') + for i in xrange(len(documents)): + if 'variables' in documents[i]: + for k in documents[i]['variables'].keys(): + if k < 0 or k >= max_variables: + if len(documents) == 1: + return HttpResponse('"Invalid key in variables: \'%d\' (it should be in the range [0..%d]"' % (k, max_variables-1), status=400) + else: + return HttpResponse('"Invalid batch insert, in document #%d of %d: Invalid variable index %d. Valid keys are in range [0-%d]"' % (i+1, len(documents), k, max_variables-1), status=400) + + @required_index_name + @required_documents + @get_index_param_or404 + @wakeup_if_hibernated + def PUT(self, request, data, version, index, documents, batch_mode): + if batch_mode: + self.logger.debug('batch insert: %d docs', len(documents)) + else: + self.logger.debug('id=%s fields=%s variables=%s categories=%s', documents[0].get('docid'), documents[0].get('fields'), documents[0].get('variables'), documents[0].get('categories')) + + if not index.is_ready(): + return HttpResponse('"Index is not ready."', status=409) + + response = self._validate_variables(index, documents) + if response: + return response + + rt = [] + + indexers = rpc.get_indexer_clients(index) + + if LOG_STORAGE_ENABLED: + records = [build_logrecord_for_add(index, d) for d in documents] + if not send_log_storage_batch(self, index, records): + return HttpResponse('"Currently unable to insert the requested document(s)."', status=503) + + for document in documents: + rt.append(self._insert_document(index, indexers, document)) + + if not batch_mode: + if rt[0]['added']: + return HttpResponse() + else: + return HttpResponse(rt[0]['error'], status=503) + + return JsonResponse(rt) + + + + + @required_index_name + @required_docids + @get_index_param_or404 + @wakeup_if_hibernated + def DELETE(self, request, data, version, index, documents, bulk_mode): + if bulk_mode: + self.logger.debug('bulk delete: %s docids', len(documents)) + else: + self.logger.debug('id=%s qsid=%s', data['docid'], request.GET.get('docid')) + + if not index.is_ready(): + return HttpResponse('"Index is not ready"', status=409) + + if LOG_STORAGE_ENABLED: + records = [build_logrecord_for_delete(index, d) for d in documents] + + if not send_log_storage_batch(self, index, records): + return HttpResponse('"Currently unable to delete the requested document."', status=503) + + indexers = rpc.get_indexer_clients(index) + + responses = delete_docs_from_index(self, index, documents) + + if not bulk_mode: + if responses[0]['deleted']: + return HttpResponse() + else: + return HttpResponse(responses[0]['error'], status=503) + + return JsonResponse(responses) + + +""" + Categories resource ====================================================== +""" +class Categories(Resource): + authenticated = True + + @required_index_name + @required_docid_data + @required_categories_data + @get_index_param_or404 + @wakeup_if_hibernated + def PUT(self, request, data, version, categories, index, docid): + self.logger.debug('id=%s categories=%s', docid, categories) + + if not index.is_writable(): + return HttpResponse('"Index is not ready"', status=409) + + if LOG_STORAGE_ENABLED: + records = [ttypes.LogRecord(index_code=index.code, docid=docid, deleted=False, categories=categories)] + if not send_log_storage_batch(self, index, records): + return HttpResponse('"Currently unable to update the requested document."', status=503) + + + indexers = rpc.get_indexer_clients(index) + + failed = False + for indexer in indexers: + try: + indexer.updateCategories(docid, categories) + except Exception, e: + if isinstance(e, ttypes.IndextankException): + return HttpResponse(e.message, status=400) + else: + self.logger.exception('Failed to update variables for %s on %s (%s)', docid, index.code, index.name) + self.error_logger.exception('Failed to update variables for %s on %s (%s)', docid, index.code, index.name) + failed = True + break + + if failed: + return HttpResponse('"Currently unable to update the categories for the requested document."', status=503) + else: + self.set_message('Updated categories for %s' % docid) + return HttpResponse() + + +""" + Variables resource ====================================================== +""" +class Variables(Resource): + authenticated = True + + @required_index_name + @required_docid_data + @required_variables_data + @get_index_param_or404 + @wakeup_if_hibernated + def PUT(self, request, data, version, variables, index, docid): + self.logger.debug('id=%s variables=%s', docid, variables) + + if not index.is_writable(): + return HttpResponse('"Index is not ready"', status=409) + + if LOG_STORAGE_ENABLED: + records = [ttypes.LogRecord(index_code=index.code, docid=docid, deleted=False, variables=variables)] + if not send_log_storage_batch(self, index, records): + return HttpResponse('"Currently unable to update the requested document."', status=503) + + + indexers = rpc.get_indexer_clients(index) + + failed = False + for indexer in indexers: + try: + indexer.updateBoost(docid, variables) + except Exception, e: + if isinstance(e, ttypes.IndextankException) and e.message.startswith('Invalid boost index'): + return HttpResponse(e.message.replace('boost', 'variable'), status=400) + else: + self.logger.exception('Failed to update variables for %s on %s (%s)', docid, index.code, index.name) + self.error_logger.exception('Failed to update variables for %s on %s (%s)', docid, index.code, index.name) + failed = True + break + + if failed: + return HttpResponse('"Currently unable to index the requested document."', status=503) + else: + self.set_message('Updated variables for %s' % docid) + return HttpResponse() + +""" + Functions resource ====================================================== +""" +class Functions(Resource): + # gets the functions metadata + @required_index_name + @get_index_param_or404 + @wakeup_if_hibernated + def GET(self, request, version, index): + #asking for writability in a get sounds odd... but jorge and spike + #think it's ok for functions. + if not index.is_writable(): + return HttpResponse('"Index not ready to list functions."', status=409) + + indexer = rpc.get_indexer_clients(index)[0] + functions = indexer.listScoreFunctions() + return JsonResponse(functions) + + +""" + Function resource ====================================================== +""" +class Function(Resource): + authenticated = True + + + # TODO GET could return the function definition + + # writes a function for the given number + @required_index_name + @required_integer_function + @required_definition_data + @get_index_param_or404 + @wakeup_if_hibernated + def PUT(self, request, data, version, index, function, definition): + self.logger.debug('f=%d', function) + + if (function < 0): + return HttpResponse('"Function index cannot be negative."', status=400) + + if not index.is_writable(): + return HttpResponse('"Index is not writable"', status=409) + + indexers = rpc.get_indexer_clients(index) + + failed = False + for indexer in indexers: + try: + indexer.addScoreFunction(function, definition) + except: + self.logger.warn('Failed to add function %s with definition: %s', function, data) + failed = True + break + + if failed: + return HttpResponse('"Unable to add the requested function. Check your syntax."', status=400) + else: + sf, _ = ScoreFunction.objects.get_or_create(index=index, name=function) + sf.definition = definition + sf.save() + + self.set_message('set to %s' % (definition)) + return HttpResponse() + + @required_index_name + @required_integer_function + @get_index_param_or404 + @wakeup_if_hibernated + def DELETE(self, request, data, version, index, function): + self.logger.debug('f=%d', function) + + if (function < 0): + return HttpResponse('"Function index cannot be negative."', status=400) + + if not index.is_writable(): + return HttpResponse('"Index is not writable"', status=409) + + indexers = rpc.get_indexer_clients(index) + + failed = False + for indexer in indexers: + try: + indexer.removeScoreFunction(function) + except: + self.logger.exception('Failed to remove function %s', function) + self.error_logger.exception('Failed to remove function %s', function) + failed = True + break + + if failed: + return HttpResponse('"Failed to remove the requested function."', status=503) + else: + models.ScoreFunction.objects.filter(index=index, name=function).delete() + return HttpResponse() + +builtin_len = len +""" + Search resource ====================================================== +""" +class Search(Resource): + authenticated = False + + @required_index_name + @required_querystring_argument('q') + @int_querystring('start') + @int_querystring('function') + @json_querystring('category_filters') + def DELETE(self, request, version, index_name, q, start=0, function=0, category_filters={}, data=None): + #kwarg 'data' is added in Resource.dispatch for 'DELETE' requests + self.logger.debug('f=%d pag=%d q=%s', function, start, q) + index = self.get_index_or_404(index_name) + + authorize_response = self._check_authorize(request) + if authorize_response: + return authorize_response + + if not index.is_readable(): + return HttpResponse('"Index is not searchable"', status=409) + + if not index.is_writable(): + return HttpResponse('"Index is not writable"', status=409) + q = _encode_utf8(q) + + if start > 5000: + return HttpResponse('"Invalid start parameters (start shouldn\'t exceed 5000)"', status=400) + + query_vars = {} + for i in xrange(10): + k = 'var%d' % i + if k in request.GET: + try: + v = float(request.GET[k]) + except ValueError: + return HttpResponse('"Invalid query variable, it should be a decimal number (var%d = %s)"', status=400) + query_vars[i] = v + + # range facet_filters + function_filters = [] + docvar_filters = [] + docvar_prefix = '' + function_prefix = 'filter_function' + + for key in request.GET.keys(): + doc_match = re.match('filter_docvar(\\d+)',key) + func_match = re.match('filter_function(\\d+)',key) + if doc_match: + self.logger.info('got docvar filter: ' + request.GET[key]) + extra_filters = self._get_range_filters(int(doc_match.group(1)), request.GET[key]) + if extra_filters == None: + return HttpResponse('"Invalid document variable range filter (' + request.GET[key] + ')"', status=400) + docvar_filters.extend(extra_filters) + elif func_match: + extra_filters = self._get_range_filters(int(func_match.group(1)), request.GET[key]) + if extra_filters == None: + return HttpResponse('"Invalid function range filter (' + request.GET[key] + ')"', status=400) + function_filters.extend(extra_filters) + + facet_filters = [] + for k,v in category_filters.items(): + if type(v) is str or type(v) is unicode: + facet_filters.append(ttypes.CategoryFilter(k, v)) + elif type(v) is list: + for element in v: + facet_filters.append(ttypes.CategoryFilter(k, element)) + else: + return HttpResponse('"Invalid facets filter"', status=400) + + extra_parameters = {} + + searcher = rpc.get_searcher_client(index) + if searcher is None: + self.logger.warn('cannot find searcher for index %s (%s)', index.name, index.code) + self.error_logger.warn('cannot find searcher for index %s (%s)', index.name, index.code) + return HttpResponse('"Currently unable to perform the requested query"', status=503) + + len = 1000 + iterations = None + while True: + if iterations is not None: + iterations -= 1 + if iterations < 0: + return HttpResponse('"Currently unable to perform the requested query - some or all documents may not have been deleted."', status=503) + try: + rs = searcher.search(q, start, len, function, query_vars, facet_filters, docvar_filters, function_filters, extra_parameters) + except ttypes.InvalidQueryException, iqe: + return HttpResponse('"Invalid query: %s"' % q, status=400) + except ttypes.MissingQueryVariableException, qve: + return HttpResponse('"%s"' % qve.message, status=400) + except ttypes.IndextankException, ite: + if ite.message == 'Invalid query': + return HttpResponse('"Invalid query: %s"' % q, status=400) + else: + self.logger.exception('Unexpected IndextankException while performing query') + self.error_logger.exception('Unexpected IndextankException while performing query') + if iterations is None: + return HttpResponse('"Currently unable to perform the requested query."', status=503) + else: + continue + + if iterations is None: + iterations = ((rs.matches - start) / len) * 2 + + self.logger.debug('query delete: %s docids', rs.matches) + + if LOG_STORAGE_ENABLED: + records = [build_logrecord_for_delete(index, d) for d in rs.docs] + + if not send_log_storage_batch(self, index, records): + continue + + delete_docs_from_index(self, index, rs.docs) + if (rs.matches - start) < len: + break + + return HttpResponse() + + @required_index_name + @required_querystring_argument('q') + @int_querystring('start') + @int_querystring('len') + @int_querystring('function') + @querystring_argument('fetch') + @querystring_argument('fetch_variables') + @querystring_argument('fetch_categories') + @querystring_argument('snippet') + @querystring_argument('snippet_type') + @json_querystring('category_filters') + @querystring_argument('callback') + @get_index_param_or404 + @check_public_api + @wakeup_if_hibernated + def GET(self, request, version, index, q, start=0, len=10, function=0, fetch='', fetch_variables='', fetch_categories='', snippet='', snippet_type='', category_filters={}, callback=None): + self.logger.debug('f=%d pag=%d:%d q=%s snippet=%s fetch=%s fetch_variables=%s fetch_categories=%s ', function, start, start+len, q, snippet, fetch, fetch_variables, fetch_categories) + + dymenabled = index.configuration.get_data().get('didyoumean') + + if not index.is_readable(): + return HttpResponse('"Index is not searchable"', status=409) + + q = _encode_utf8(q) + q = self._sanitize_query(q) + + if start + len > 5000: + return HttpResponse('"Invalid start and len parameters (start+len shouldn\'t exceed 5000)"', status=400) + + query_vars = {} + for i in xrange(10): + k = 'var%d' % i + if k in request.GET: + try: + v = float(request.GET[k]) + except ValueError: + return HttpResponse('"Invalid query variable, it should be a decimal number (var%d = %s)"', status=400) + query_vars[i] = v + + # range facet_filters + function_filters = [] + docvar_filters = [] + docvar_prefix = '' + function_prefix = 'filter_function' + + for key in request.GET.keys(): + doc_match = re.match('filter_docvar(\\d+)',key) + func_match = re.match('filter_function(\\d+)',key) + if doc_match: + self.logger.info('got docvar filter: ' + request.GET[key]) + extra_filters = self._get_range_filters(int(doc_match.group(1)), request.GET[key]) + if extra_filters == None: + return HttpResponse('"Invalid document variable range filter (' + request.GET[key] + ')"', status=400) + docvar_filters.extend(extra_filters) + elif func_match: + extra_filters = self._get_range_filters(int(func_match.group(1)), request.GET[key]) + if extra_filters == None: + return HttpResponse('"Invalid function range filter (' + request.GET[key] + ')"', status=400) + function_filters.extend(extra_filters) + + facet_filters = [] + for k,v in category_filters.items(): + if type(v) is str or type(v) is unicode: + facet_filters.append(ttypes.CategoryFilter(k, v)) + elif type(v) is list: + for element in v: + facet_filters.append(ttypes.CategoryFilter(k, element)) + else: + return HttpResponse('"Invalid facets filter"', status=400) + + extra_parameters = {} + if snippet: + extra_parameters['snippet_fields'] = snippet + if snippet_type: + extra_parameters['snippet_type'] = snippet_type + if fetch: + extra_parameters['fetch_fields'] = ','.join([f.strip() for f in fetch.split(',')]) + if fetch_variables: + extra_parameters['fetch_variables'] = fetch_variables + if fetch_categories: + extra_parameters['fetch_categories'] = fetch_categories + + + searcher = rpc.get_searcher_client(index) + if searcher is None: + self.logger.warn('cannot find searcher for index %s (%s)', index.name, index.code) + self.error_logger.warn('cannot find searcher for index %s (%s)', index.name, index.code) + return HttpResponse('"Currently unable to perform the requested search"', status=503) + + try: + t1 = time.time() + rs = searcher.search(q, start, len, function, query_vars, facet_filters, docvar_filters, function_filters, extra_parameters) + t2 = time.time() + except ttypes.InvalidQueryException, iqe: + return HttpResponse('"Invalid query: %s"' % q, status=400) + except ttypes.MissingQueryVariableException, qve: + return HttpResponse('"%s"' % qve.message, status=400) + except ttypes.IndextankException, ite: + if ite.message == 'Invalid query': + return HttpResponse('"Invalid query: %s"' % q, status=400) + else: + self.logger.exception('Unexpected IndextankException while performing query') + self.error_logger.exception('Unexpected IndextankException while performing query') + return HttpResponse('"Currently unable to perform the requested search"', status=503) + + + formatted_time = '%.3f' % (t2-t1) + for i in xrange(builtin_len(rs.docs)): + if rs.variables: + for k,v in rs.variables[i].iteritems(): + rs.docs[i]['variable_%d' % k] = v + if rs.categories: + for k,v in rs.categories[i].iteritems(): + rs.docs[i]['category_%s' % k] = v + if rs.scores: + rs.docs[i]['query_relevance_score'] = rs.scores[i] + rsp = dict(matches=rs.matches, results=rs.docs, search_time=formatted_time, facets=rs.facets, query=q) + #only add didyoumean to customers who have it enabled. + if dymenabled: + rsp['didyoumean'] = rs.didyoumean + + return JsonResponse(rsp, callback=callback) + + def _get_range_filters(self, id, filter_string): + filter_strings = filter_string.split(',') + range_filters = [] + for filter in filter_strings: + parts = filter.split(':') + if (len(parts) != 2): + return None + + if parts[0] == '*': + floor = 0 + no_floor = True + else: + try: + floor = float(parts[0]) + no_floor = False + except ValueError: + return None + + if parts[1] == '*': + ceil = 0 + no_ceil = True + else: + try: + ceil = float(parts[1]) + no_ceil = False + except ValueError: + return None + + range_filters.append(ttypes.RangeFilter(key=id, floor=floor, no_floor=no_floor, ceil=ceil, no_ceil=no_ceil)) + + return range_filters + + def _sanitize_query(self, q): + s = self._try_to_sanitize_parentheses_and_quotes(q) + s = s.replace(r'{', r'\{') + s = s.replace(r'}', r'\}') + s = s.replace(r'[', r'\[') + s = s.replace(r']', r'\]') + s = s.replace(r'!', r'\!') + s = s.replace(r'?', r'\?') + s = s.replace(r'~', r'\~') + #s = s.replace(r'*', r'\*') + s = s.replace(r': ', r'\: ') + s = s.replace(r' :', r' \:') + s = s.replace(r'ssh:/', r'ssh\:/') + s = s.replace(r'ftp:/', r'ftp\:/') + s = s.replace(r'http:/', r'http\:/') + s = s.replace(r'https:/', r'https\:/') + if s != q: + self.logger.debug('query sanitized it was (%s) now (%s)', q, s) + return s + + def _try_to_sanitize_parentheses_and_quotes(self, q): + # should solve most 'solveable' queries + lp = q.count('(') + rp = q.count(')') + while (lp > rp): + q = q.rstrip() + q += ')' + rp += 1 + while (lp < rp): + q = '(' + q + lp += 1 + while '()' in q: + q = q.replace(r'()',r'') + if (q.count('"') % 2): + q += '"' + return q + +""" + Promote resource ====================================================== +""" +class Promote(Resource): + authenticated = True + + @required_index_name + @required_docid_data + @required_query_data + @get_index_param_or404 + @wakeup_if_hibernated + def PUT(self, request, data, version, index, docid, query): + self.logger.debug('id=%s q=%s', docid, query) + + if not index.is_writable(): + return HttpResponse('"Index is not writable"', status=409) + + indexers = rpc.get_indexer_clients(index) + + failed = False + for indexer in indexers: + try: + indexer.promoteResult(docid, query) + except: + self.logger.exception('"Failed to promote result %s for query %s"', docid, query) + self.error_logger.exception('"Failed to promote result %s for query %s"', docid, query) + failed = True + break + + if failed: + return HttpResponse('"Currently unable to perform the requested promote."', status=503) + else: + return HttpResponse() + +""" + InstantLinks resource ====================================================== +""" +class InstantLinks(Resource): + authenticated = False + + @required_index_name + @required_querystring_argument('query', parse=_encode_utf8) + @querystring_argument('fetch', parse=_encode_utf8) + @querystring_argument('callback', parse=_encode_utf8) + @querystring_argument('field', parse=_encode_utf8) + @get_index_param_or404 + @wakeup_if_hibernated + def GET(self, request, version, index, query, fetch='name', callback=None, field='name'): + self.logger.debug('query=%s callback=%s field=%s fetch=%s', query, callback, field, fetch) + + if not index.is_readable(): + return HttpResponse('"Index is not readable."', status=409) + + searcher = rpc.get_searcher_client(index) + if searcher is None: + self.logger.warn('cannot find searcher for index %s (%s)', index.name, index.code) + return HttpResponse('"Currently unable to perform the requested search"', status=503) + + suggestor = rpc.get_suggestor_client(index) + try: + suggestions = suggestor.complete(query, field) + if builtin_len(suggestions) == 0: + rsp = dict(matches=0, results=[],search_time='', facets={}) + return JsonResponse(rsp, callback=callback) + + # got suggestions .. build a query + print 'suggestions=%s' % suggestions + query = ' OR '.join( map( lambda x: "(%s)"%x, suggestions)) + query = '%s:(%s)' % (field, query) + print 'query=%s' % query + extra_parameters = {} + extra_parameters['fetch_fields'] = fetch + try: + rs = searcher.search(query, 0, 4, 0, {}, {}, {}, {}, extra_parameters) + except ttypes.IndextankException, ite: + self.logger.exception('Unexpected IndextankException while performing query for instantlinks') + self.error_logger.exception('Unexpected IndextankException while performing query for instantlinks') + return HttpResponse('"Currently unable to perform the requested search"', status=503) + rsp = dict(matches=rs.matches, results=rs.docs, search_time='', facets=rs.facets) + return JsonResponse(rsp, callback=callback) + except: + self.logger.exception('Failed to provide instant links for "%s"', query) + self.error_logger.exception('Failed to provide instant links for "%s"', query) + return HttpResponse('"Currently unable to perform the requested instant links."', status=503) + + +""" + AutoComplete resource ====================================================== +""" +class AutoComplete(Resource): + authenticated = False + + @required_index_name + @required_querystring_argument('query', parse=_encode_utf8) + @querystring_argument('callback', parse=_encode_utf8) + @querystring_argument('field', parse=_encode_utf8) + @get_index_param_or404 + @wakeup_if_hibernated + def GET(self, request, version, index, query, callback=None, field='text'): + self.logger.debug('query=%s callback=%s', query, callback) + + if not index.is_readable(): + return HttpResponse('"Index is not readable."', status=409) + + suggestor = rpc.get_suggestor_client(index) + try: + suggestions = suggestor.complete(query, field) + return JsonResponse({'suggestions': suggestions, 'query': query}, callback=callback) + except: + self.logger.exception('Failed to provide autocomplete for "%s"', query) + self.error_logger.exception('Failed to provide autocomplete for "%s"', query) + return HttpResponse('"Currently unable to perform the requested autocomplete."', status=503) + + + + +""" + Provisioner for CloudFoundry. This for the transition until we adapt CF to conform to our default provisioner. +""" +class TransitionCloudFoundryProvisioner(ProvisionerResource): + + @authorized_method + @required_querystring_argument("code") + def GET(self, request, code, **kwargs): + # fetch the account we want info about + account = Account.objects.get(apikey__startswith=code+"-") + + # make sure the provisioner for the account is the same requesting its deletion + if self.provisioner != account.provisioner: + return HttpResponse('You are not the provisioner for this account', status=403) + + # create the creds + creds = { + 'INDEXTANK_PRIVATE_API_URL' : account.get_private_apiurl(), + 'INDEXTANK_PUBLIC_API_URL' : account.get_public_apiurl(), + 'INDEXTANK_INDEX_NAMES' : [i.name for i in account.indexes.all()] + } + + return JsonResponse({"code": account.get_public_apikey(), "config": creds}) + + @authorized_method + @required_data("plan") + def PUT(self, request, data, **kwargs): + # only default package right now + if data['plan'].upper() != "FREE": + return HttpResponse("Only FREE plan allowed", status=400) + + account, _ = Account.create_account(datetime.datetime.now()) + account.status = Account.Statuses.operational + account.provisioner = self.provisioner + + # apply package. see validation above + account.apply_package(Package.objects.get(name=data['plan'].upper())) + + account.save() + + # create an index, using ApiClient + client = ApiClient(account.get_private_apiurl()) + i = client.create_index("idx") + + # ok, write credentials on response + creds = { + 'INDEXTANK_PRIVATE_API_URL' : account.get_private_apiurl(), + 'INDEXTANK_PUBLIC_API_URL' : account.get_public_apiurl(), + 'INDEXTANK_INDEX_NAMES': ["idx"] + } + + return JsonResponse({"code": account.get_public_apikey(), "config": creds}) + + @authorized_method + @required_data("code") + def DELETE(self, request, data, **kwargs): + # fetch the account that should be deleted + account = Account.objects.get(apikey__startswith=data['code']+"-") + + # make sure the provisioner for the account is the same requesting its deletion + if self.provisioner != account.provisioner: + return HttpResponse('You are not the provisioner for this account', status=403) + + # ok, lets do it + account.close() + + return HttpResponse(status=204) + + +class BaseProvisioner(ProvisionerResource): + def _is_valid_provisioner(self, provisioner): + return True + + def _validate(self, request, data, plan, **kwargs): + if not self._is_valid_provisioner(self.provisioner): + return HttpResponse('"Invalid provisioner"', status=400) + if self.provisioner.plans.filter(plan=plan.upper()).count() == 0: + return HttpResponse('"Plan %s NOT allowed"' % plan, status=400) + + def _get_email(self, request, data, plan, **kwargs): + return None + + def _get_id(self, account): + return account.get_public_apikey() + + @authorized_method + @required_data("plan") + def POST(self, request, data, plan=None, **kwargs): + response = self._validate(request, data, plan, **kwargs) + if response: + return response + + package = self.provisioner.plans.get(plan=plan.upper()).package + account, _ = Account.create_account(datetime.datetime.now(), email=self._get_email(request, data, plan, **kwargs)) + + account.status = Account.Statuses.operational + account.provisioner = self.provisioner + account.apply_package(package) + account.save() + + mixpanel_event_track('signup', { 'plan' : package.code, 'signup_place': self.provisioner.name, 'distinct_id': account.user.email}) + + # create an index, using ApiClient + client = ApiClient(account.get_private_apiurl()) + i = client.create_index("idx", options={ 'public_search':True }) + + # ok, write credentials on response + creds = { + 'INDEXTANK_API_URL' : account.get_private_apiurl(), + 'INDEXTANK_PRIVATE_API_URL' : account.get_private_apiurl(), + 'INDEXTANK_PUBLIC_API_URL' : account.get_public_apiurl(), + } + + return JsonResponse({"id": self._get_id(account), "config": creds}) + + @authorized_method + def PUT(self, request, data, id=None): + return HttpResponse('{ "message": "Sorry, we do not support plan upgrades" }', status=503) + + +class EmailProvisionerMixin(object): + def _validate(self, request, data, plan, **kwargs): + r = super(EmailProvisionerMixin, self)._validate(request, data, plan, **kwargs) + return r or self._validate_email(data.get('email', None)) + + def _validate_email(self, email): + if email is None: + return HttpResponse('"An email is required"', status=400) + try: + is_valid_email(email) + except: + return HttpResponse('"Invalid email!"', status=400) + + def _get_email(self, request, data, plan, **kwargs): + return data['email'] + +class SSOProvisionerMixin(object): + def GET(self, request, id=None): + timestamp = int(request.GET.get('timestamp',0)) + token = request.GET.get('token','') + navdata = request.GET.get('nav-data','') + return HttpResponseRedirect("http://indextank.com/provider/resources/%s?token=%s×tamp=%s&nav-data=%s"%(id,token,timestamp,navdata)) # TODO fix this + +class DeleteProvisionerMixin(object): + def _get_account(self, id): + return Account.objects.get(apikey__startswith=("%s-" % id)) + + @authorized_method + def DELETE(self, request, data, id=None): + # fetch the account that should be deleted + account = self._get_account(id) + + # make sure the provisioner for the account is the same requesting its deletion + if self.provisioner != account.provisioner: + return HttpResponse('You are not the provisioner for this account', status=403) + + # ok, lets do it + account.close() + + return HttpResponse() + +class FetchCredentialsProvisionerMixin(object): + @authorized_method + @required_querystring_argument("id") + def GET(self, request, code, **kwargs): + # fetch the account we want info about + account = self._get_account(id) + + # make sure the provisioner for the account is the same requesting its deletion + if self.provisioner != account.provisioner: + return HttpResponse('You are not the provisioner for this account', status=403) + + # create the creds + creds = { + 'INDEXTANK_PRIVATE_API_URL' : account.get_private_apiurl(), + 'INDEXTANK_PUBLIC_API_URL' : account.get_public_apiurl(), + 'INDEXTANK_INDEX_NAMES' : [i.name for i in account.indexes.all()] + } + + return JsonResponse({"id": account.get_public_apikey(), "config": creds}) + + +class PublicProvisioner(EmailProvisionerMixin, BaseProvisioner): + pass + +class DisabledProvisioner(BaseProvisioner): + @authorized_method + @required_data("plan") + def POST(self, request, data, plan=None, **kwargs): + return HttpResponse('"Provisioning is currently disabled"', status=503) + +class CloudFoundryProvisioner(DeleteProvisionerMixin, FetchCredentialsProvisionerMixin, DisabledProvisioner): + def _is_valid_provisioner(self, provisioner): + return provisioner.name == 'cloudfoundry' + +class AppHarborProvisioner(DeleteProvisionerMixin, SSOProvisionerMixin, DisabledProvisioner): + def _is_valid_provisioner(self, provisioner): + return provisioner.name == 'appharbor' + +class HerokuProvisioner(DeleteProvisionerMixin, SSOProvisionerMixin, BaseProvisioner): + def _is_valid_provisioner(self, provisioner): + return provisioner.name == 'heroku' + def _get_id(self, account): + return account.id + def _get_account(self, id): + return Account.objects.get(id=id) + +def default(request): + return HttpResponse('"IndexTank API. Please refer to the documentation: http://indextank.com/documentation/api"') + + + diff --git a/api/restresource.py b/api/restresource.py new file mode 100644 index 0000000..1cdbd36 --- /dev/null +++ b/api/restresource.py @@ -0,0 +1,544 @@ +import re +import time +import base64 + +from django.http import Http404, HttpResponse, HttpResponseNotAllowed, HttpResponseServerError, HttpResponseNotFound +from django.utils import simplejson as json +from django.shortcuts import get_object_or_404 + +from models import Account, Index, Provisioner +from lib import encoder +from lib.flaptor_logging import get_logger +import logging + +class HttpMethodNotAllowed(Exception): + """ + Signals that request.method was not part of + the list of permitted methods. + """ + +class AccountFilter(logging.Filter): + def __init__(self, resource): + logging.Filter.__init__(self) + self.resource = resource + + def filter(self, record): + record.prefix = ' $PIDCOLOR%s$RESET' % (self.resource.get_code()) + if self.resource.executing: + # indent internal log lines + record.msg = ' ' + record.msg + else: + record.prefix += '$BOLD' + record.suffix = '$RESET' + return True + +class FakeLogger(): + ''' + Logger wrapper that adds: + - pid + - coloring + ''' + def __init__(self, logger, resource, compname=None): + self.logger = logger + self.resource = resource + self.compname = compname or logger.name + + def _add_extra(self, msg, kwargs): + prefix = ' $PIDCOLOR%s$RESET' % (self.resource.get_code()) + suffix = '' + if self.resource.executing: + # indent internal log lines + msg = ' ' + msg + else: + prefix += '$BOLD' + suffix += '$RESET' + if 'extra' in kwargs: + kwargs['extra']['prefix'] = prefix + kwargs['extra']['suffix'] = suffix + kwargs['extra']['compname'] = self.compname + else: + kwargs['extra'] = dict(prefix=prefix, suffix=suffix, compname=self.compname) + return msg + + def debug(self, msg, *args, **kwargs): + msg = self._add_extra(msg, kwargs) + self.logger.debug(msg, *args, **kwargs) + + def info(self, msg, *args, **kwargs): + msg = self._add_extra(msg, kwargs) + self.logger.info(msg, *args, **kwargs) + + def warn(self, msg, *args, **kwargs): + msg = self._add_extra(msg, kwargs) + self.logger.warn(msg, *args, **kwargs) + + def error(self, msg, *args, **kwargs): + msg = self._add_extra(msg, kwargs) + self.logger.error(msg, *args, **kwargs) + + def fatal(self, msg, *args, **kwargs): + msg = self._add_extra(msg, kwargs) + self.logger.fatal(msg, *args, **kwargs) + + def exception(self, msg, *args, **kwargs): + msg = self._add_extra(msg, kwargs) + kwargs['exc_info'] = 1 + self.logger.error(msg, *args, **kwargs) + +class Resource(object): + authenticated = False + mimetype = None + permitted_methods = 'GET' + + def __init__(self): + self.__name__ = self.__class__.__name__ + self._account = None + self.permitted_methods = [] + if hasattr(self, 'GET'): + self.permitted_methods.append('GET') + if hasattr(self, 'POST'): + self.permitted_methods.append('POST') + if hasattr(self, 'PUT'): + self.permitted_methods.append('PUT') + if not hasattr(self, 'POST'): + self.permitted_methods.append('POST') + self.POST = self.PUT + if hasattr(self, 'DELETE'): + self.permitted_methods.append('DELETE') + self.logger = FakeLogger(get_logger(self.__name__), self) + self.error_logger = FakeLogger(get_logger('Errors'), self, compname=self.__name__) + self.executing = True + + def dispatch(self, request, target, **kwargs): + ''' + Delegates to the appropriate method (PUT, GET, DELETE) + if the derived class defines the handler + Returns and HttpMethodNotAllowed if not. + For all methods but get, it also parses the body as json, and + passes it to the handler method. Returns http 400 if it's + unparsable. + ''' + request_method = request.method.upper() + if request_method not in self.permitted_methods: + raise HttpMethodNotAllowed + + if request_method == 'GET': + return target.GET(request, **kwargs) + elif request_method == 'POST' or request_method == 'PUT' or request_method == 'DELETE': + data = request._get_raw_post_data() + if data: + try: + data = json.loads(data) + except ValueError, e: + return HttpResponse('"Invalid JSON input: %s"' % e, status=400) + else: + data = {} + if request_method == 'DELETE': + return target.DELETE(request, data=data, **kwargs) + if request_method == 'POST': + return target.POST(request, data=data, **kwargs) + else: + return target.PUT(request, data=data, **kwargs) + else: + raise Http404 + + def get_code(self): + ''' + Returns the account code (the public part of + the api-key + ''' + return self._code + + def get_account(self): + ''' + Returns an Account object for this call. + If account cannot be validated, it returns 404. + ''' + if self._account_id and not self._account: + self._account = get_object_or_404(Account, pk=self._account_id) + return self._account + + def get_index(self, index_name): + ''' + Returns an Index Object for this call. + If and index with index_name cannot be found for this account, + it returns None. + ''' + idx = Index.objects.filter(account=self._account_id, name=index_name) + if len(idx): + return idx[0] + else: + return None + + def get_index_or_404(self, index_name): + return get_object_or_404(Index, account=self._account_id, name=index_name) + + def __parse_account(self, request): + host = request.get_host() + code = re.search(r'([^\.]+)\.api\..*', host) + self._code = None + self._account_id = None + self._account = None + if code: + self._code = code.group(1) + self._account_id = encoder.from_key(self._code) + + def _authorize(self, request, force=False): + auth = request.META.get('HTTP_AUTHORIZATION') + if self._account_id and auth: + auth = auth.split() + if auth and len(auth) > 1 and auth[0].lower() == "basic": + parts = base64.b64decode(auth[1]).split(':') + if len(parts) == 2: + key = '%s-%s' % (self._code, parts[1]) + #self.logger.debug('AUTH: comparing %s vs %s', key, self.get_account().apikey) + if self.get_account().apikey == key: + return True + return False + + def __call__(self, request, **kwargs): + self._start_time = time.time() + self.__parse_account(request) + + self.message = None + + response = HttpResponseServerError('Unreachable response') + + + authorize_response = self.validate_request(request) + + if authorize_response: + response = authorize_response + else: + try: + response = self.dispatch(request, self, **kwargs) + except HttpMethodNotAllowed: + response = HttpResponseNotAllowed(self.permitted_methods) + response.mimetype = self.mimetype + except Http404: + response = HttpResponseNotFound() + except Exception: + self.logger.exception('Unexpected error while processing request') + self.error_logger.exception('Unexpected error while processing request') + response = HttpResponseServerError('Unexpected error') + elapsed_time = time.time() - self._start_time + + if self.message is None: + self.message = response.content[:120] + + self.executing = False + self.logger.info('[%d] in %.3fs for [%s %s] : %s', response.status_code, elapsed_time, request.method, request.path, self.message) + self.executing = True + return response + + def _check_authorize(self, request): + """ + Checks authorization. Returns None if ok and the proper response + if it does not pass. + """ + response = None + if not self._authorize(request): + response = HttpResponse('"Authorization Required"', mimetype=self.mimetype, status=401) + response['WWW-Authenticate'] = 'Basic realm=""' + + return response + + def validate_request(self, request): + response = None + if self.authenticated: + response = self._check_authorize(request) + if response is None and self.get_account().status != Account.Statuses.operational: + response = HttpResponse('"Account not active"', mimetype=self.mimetype, status=409) + return response + + def set_message(self, message): + self.message = message + + @classmethod + def as_view(cls, **initkwargs): + def view(request, *args, **kwargs): + instance = cls(**initkwargs) + return instance(request, *args, **kwargs) + return view + + +class JsonResponse(HttpResponse): + def __init__(self, json_object, *args, **kwargs): + body = json.dumps(json_object) + if not ('mimetype' in kwargs and kwargs['mimetype']): + if 'callback' in kwargs and kwargs['callback']: + kwargs['mimetype'] = 'application/javascript' + else: + kwargs['mimetype'] = 'application/json' + + if 'callback' in kwargs: + callback = kwargs.pop('callback') + if callback: + body = '%s(%s)' % (callback, body) + super(JsonResponse, self).__init__(body, *args, **kwargs) + +class ProvisionerResource(Resource): + provisioner = None + + def get_code(self): + ''' + Returs a human readable string identifying the provider. + ''' + return self.provisioner.name if self.provisioner else '?????' + + def _authorize(self, request): + auth = request.META.get('HTTP_AUTHORIZATION') + if auth: + auth = auth.split() + if auth and len(auth) > 1 and auth[0].lower() == "basic": + parts = base64.b64decode(auth[1]).split(':') + if len(parts) == 2: + token = parts[1] + provisioner = Provisioner.objects.filter(token=token) + if provisioner: + self.provisioner = provisioner[0] + return True + return False + + def validate_request(self, request): + return None + + +""" + Standard validators and parsers +""" + +def int_validator(message, status=400): + ''' + Returns a validation function that, on error, + return an HttpResponse with the given status code + and message. + + >>> validator = int_validator('foo', status=401) + >>> validator(1) + >>> response = validator('bar') + >>> response.code + 401 + + ''' + def validate_int(string): + if not string.isdigit(): + return HttpResponse(message, status=status) + return validate_int + +def json_validator(message, status=400): + ''' + Return a validation function for json, that on error + returns an HttpResponse with the given status code and + message + + >>> validator = json_validator('foo', status=401) + >>> validator('{"menu": {"foo": "bar", "bar": "foo"}}') + >>> response = validator("{'menu': {'foo': 'bar', 'bar': 'foo'}}") + >>> response.code + 401 + + ''' + def validate_json(string): + try: + data = json.loads(string) + except ValueError, e: + return HttpResponse(message, status=status) + return validate_json + +ALWAYS_VALID = lambda x: None +IDENTITY = lambda x: x + +""" + Data extraction and validation for PUT requests decorators +""" +def optional_data(name, validate=ALWAYS_VALID, parse=IDENTITY): + return __check_data(name, validate, parse, False) +def required_data(name, validate=ALWAYS_VALID, parse=IDENTITY): + return __check_data(name, validate, parse, True) + +def __check_data(name, validate, parse, required): + def decorate(func): + def decorated(self, request, data, **kwargs): + if name in data: + response = validate(data[name]) + if response: + return response + kwargs[name] = parse(data[name]) + elif required: + return HttpResponse('"Argument %s is required in the request body"' % name, status=400) + return func(self, request, data=data, **kwargs) + return decorated + return decorate + +""" + Keyword argument validation decorators +""" +def validate_argument(name, validate, parse): + ''' + Decorator that validates a request arguments, and returns a + parsed version. + Arguments: + name -- The name of the parameter to validate. + validate -- a validation function that returns None for sucess + or an HttpResponse for failure + parse -- The parsing function. + ''' + def decorate(func): + def decorated(self, request, **kwargs): + if name in kwargs: + response = validate(kwargs[name]) + if response: + return response + kwargs[name] = parse(kwargs[name]) + return func(self, request, **kwargs) + return decorated + return decorate + +def utf8_argument(name, message=None, status=400): + ''' + Decorator that validates and parses utf-8 arguments into str. + Arguments: + message -- ignored + status -- ignored + ''' + def validate_utf8(string): + try: + string.encode('utf-8') + except: + return HttpResponse('"%s should be valid utf-8. Was: %s"' % (name, string)) + def encode_utf8(string): + return string.encode('utf-8') + return validate_argument(name, validate_utf8, encode_utf8) + +def int_argument(name, message=None, status=400): + return validate_argument(name, int_validator(message, status), int) + +def non_empty_argument(name, message, status=400): + def validate_non_empty(string): + if string.strip() == '': + return HttpResponse(message, status=status) + return validate_argument(name, validate_non_empty, IDENTITY) + +""" + GET querystring argument validation decorators +""" +def required_querystring_argument(name, validate=ALWAYS_VALID, parse=IDENTITY): + def decorate(func): + def decorated(self, request, **kwargs): + if name in request.GET: + response = validate(request.GET[name]) + if response: + return response + kwargs[name] = parse(request.GET[name]) + else: + return HttpResponse('"Argument %s is required in the QueryString"' % (name), status=400) + return func(self, request, **kwargs) + return decorated + return decorate + +def querystring_argument(name, validate=ALWAYS_VALID, parse=IDENTITY): + def decorate(func): + def decorated(self, request, **kwargs): + if name in request.GET: + response = validate(request.GET[name]) + if response: + return response + kwargs[name] = parse(request.GET[name]) + return func(self, request, **kwargs) + return decorated + return decorate + +def int_querystring(name, message=None, status=400): + if not message: + message = 'Query string argument %s should be a non negative integer.' % name + return querystring_argument(name, int_validator(message, status), int) + +def json_querystring(name, message=None, status=400): + if not message: + message = 'Query string argument %s should be a valid json.' % name + return querystring_argument(name, json_validator(message, status), json.loads) + + +def authorized_method(func): + ''' + Decorator that forces request to be authorized. + Using it on a method ensures that in the case that the + user isn't authorized, the body of the method won't be run. + ''' + def decorated(self, request, **kwargs): + return self._check_authorize(request) or func(self, request, **kwargs) + return decorated + +def check_public_api(func): + ''' + Checks that the defined index has public access enabled OR + that the request is (privately) authorized. + + Depends on the "index" param so get_index_param_or404 should + be called first. + + In case that the index doesn't have public access enabled and + the request is not authorized, it return 404. + ''' + def decorated(self, request, **kwargs): + if 'index' in kwargs: + index = kwargs['index'] + if not index.public_api: + authorize_response = self._check_authorize(request) + if authorize_response: + return authorize_response + else: + return HttpResponse('"Index name cannot be empty"', status=400) + return func(self, request, **kwargs) + return decorated + +def get_index_param(func): + ''' + Decorator that validates the existence of an index_name + param, and parses it into the "index" variable. + If no index with that name exist, it parses "None" into + the index param, but the decorated function does run. + ''' + def decorated(self, request, **kwargs): + if 'index_name' in kwargs: + index = self.get_index(kwargs['index_name']) + kwargs['index'] = index + del kwargs['index_name'] + return func(self, request, **kwargs) + return decorated + +def get_index_param_or404(func): + ''' + Decorator that validates the existence of an index_name + param, and parses it into the "index" variable. + In case of error, an 404 HttpResponse is returned and + the decorated function doesn't run. + ''' + def decorated(self, request, **kwargs): + if 'index_name' in kwargs: + index = self.get_index_or_404(kwargs['index_name']) + kwargs['index'] = index + del kwargs['index_name'] + return func(self, request, **kwargs) + return decorated + +def wakeup_if_hibernated(func): + ''' + Decorator that makes sure an index is not hibernated. + It takes the "index" argument from the request, and checks + the index's state. If it was hibernated, it triggers wake up + and returns a 503 response. + ''' + def decorated(self, request, **kwargs): + if 'index' in kwargs: + index = kwargs['index'] + if index.status in (Index.States.hibernated, Index.States.waking_up): + if index.status == Index.States.hibernated: + index.update_status(Index.States.waking_up) + return HttpResponse('"Wellcome back! Your index has been hibernating and is now waking up. Please retry in a few seconds"', status=503) + return func(self, request, **kwargs) + else: + return HttpResponse('"Index name cannot be empty"', status=400) + return decorated + diff --git a/api/rpc.py b/api/rpc.py new file mode 100644 index 0000000..2fd7649 --- /dev/null +++ b/api/rpc.py @@ -0,0 +1,169 @@ +from flaptor.indextank.rpc import Indexer, Searcher, Suggestor, Storage, LogWriter, WorkerManager,\ + DeployManager, Controller, FrontendManager + +from flaptor.indextank.rpc.ttypes import NebuException, IndextankException + +''' =========================== + THRIFT STUFF + =========================== ''' +from thrift.transport import TSocket, TTransport +from thrift.protocol import TBinaryProtocol +from lib import flaptor_logging, exceptions +from thrift.transport.TTransport import TTransportException +from socket import socket +from socket import error as SocketError + + +logger = flaptor_logging.get_logger('RPC') + +# Missing a way to close transport +def getThriftControllerClient(host, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host,19010, timeout_ms) + client = Controller.Client(protocol) + transport.open() + return client + +# Missing a way to close transport +def getThriftIndexerClient(host, base_port, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host, base_port + 1, timeout_ms) + client = Indexer.Client(protocol) + transport.open() + return client + +def getThriftSearcherClient(host, base_port, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host, base_port + 2, timeout_ms) + client = Searcher.Client(protocol) + transport.open() + return client + +def getThriftSuggestorClient(host, base_port): + protocol, transport = __getThriftProtocolTransport(host, base_port + 3) + client = Suggestor.Client(protocol) + transport.open() + return client + +storage_port = 10000 +def getThriftStorageClient(): + protocol, transport = __getThriftProtocolTransport('storage',storage_port) + client = Storage.Client(protocol) + transport.open() + return client + +def getThriftLogWriterClient(host, port, timeout_ms=500): + protocol, transport = __getThriftProtocolTransport(host,port,timeout_ms) + client = LogWriter.Client(protocol) + transport.open() + return client + +def getThriftLogReaderClient(host, port, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host,port,timeout_ms) + client = LogWriter.Client(protocol) + transport.open() + return client + +class ReconnectingClient: + def __init__(self, factory): + self.factory = factory + self.delegate = None #factory() + + def __getattr__(self, name): + import types + if self.delegate is None: + self.delegate = self.factory() + att = getattr(self.delegate, name) + if type(att) is types.MethodType: + def wrap(*args, **kwargs): + try: + return att(*args, **kwargs) + except (NebuException, IndextankException): + logger.warn('raising catcheable exception') + raise + except (TTransportException, IOError, SocketError): + logger.warn('failed to run %s, reconnecting once', name) + self.delegate = self.factory() + att2 = getattr(self.delegate, name) + return att2(*args, **kwargs) + except Exception: + logger.exception('Unexpected failure to run %s, reconnecting once', name) + self.delegate = self.factory() + att2 = getattr(self.delegate, name) + return att2(*args, **kwargs) + + return wrap + else: + return att + +def getReconnectingStorageClient(): + return ReconnectingClient(getThriftStorageClient) + +def getReconnectingLogWriterClient(host, port): + return ReconnectingClient(lambda: getThriftLogWriterClient(host, port)) + +worker_manager_port = 8799 +def getThriftWorkerManagerClient(host): + protocol, transport = __getThriftProtocolTransport(host,worker_manager_port) + client = WorkerManager.Client(protocol) + transport.open() + return client + +deploymanager_port = 8899 +def get_deploy_manager(): + protocol, transport = __getThriftProtocolTransport('deploymanager',deploymanager_port) + client = DeployManager.Client(protocol) + transport.open() + return client + + +def __getThriftProtocolTransport(host, port=0, timeout_ms=None): + ''' returns protocol,transport''' + # Make socket + transport = TSocket.TSocket(host, port) + + if timeout_ms is not None: + transport.setTimeout(timeout_ms) + + # Buffering is critical. Raw sockets are very slow + transport = TTransport.TBufferedTransport(transport) + + # Wrap in a protocol + protocol = TBinaryProtocol.TBinaryProtocol(transport) + return protocol, transport + + +def get_searcher_client(index, timeout_ms=None): + ''' + This method returns a single searcherclient, or None + ''' + deploy = index.searchable_deploy() + if deploy: + return getThriftSearcherClient(deploy.worker.lan_dns, int(deploy.base_port), timeout_ms) + else: + return None + +def get_worker_controller(worker, timeout_ms=None): + return getThriftControllerClient(worker. lan_dns) + +def get_suggestor_client(index): + ''' + This method returns a single suggestorclient, or None + ''' + deploy = index.searchable_deploy() + if deploy: + return getThriftSuggestorClient(deploy.worker.lan_dns, int(deploy.base_port)) + else: + return None + +def get_indexer_clients(index, timeout_ms=1000): + ''' + This method returns the list of all indexerclients that should be updated + on add,delete,update, and category updates. + @raise exceptions.NoIndexerException if this index has no writable deploy. + ''' + deploys = index.indexable_deploys() + retval = [] + for d in deploys: + retval.append(getThriftIndexerClient(d.worker.lan_dns, int(d.base_port), timeout_ms)) + if retval: + return retval + else: + raise exceptions.NoIndexerException() diff --git a/api/safe_reload.sh b/api/safe_reload.sh new file mode 100755 index 0000000..d5a7368 --- /dev/null +++ b/api/safe_reload.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# kill all api workers (those that match the pattern but are not `cat pid`) +# the master should then take care of respawning workers +echo "slowly killing workers and allowing the master to respawn them" +for pid in `pgrep -f 'api-uwsgi' | grep -v \`cat pid\``; do + count=`pgrep -f 'api-uwsgi' | grep -v \`cat pid\` | wc -l` + echo "killing $pid - live workers: $count" + kill $pid + sleep 1 +done \ No newline at end of file diff --git a/api/settings.py b/api/settings.py new file mode 100644 index 0000000..15ef978 --- /dev/null +++ b/api/settings.py @@ -0,0 +1,80 @@ +# Django settings for burbio project. + +from os import environ + +LOCAL = (environ.get('DJANGO_LOCAL') == '1') + +DEBUG = False +TEMPLATE_DEBUG = DEBUG + +ADMINS = ( + # ('Your Name', 'your_email@domain.com'), +) + +MANAGERS = ADMINS + +DATABASE_ENGINE = 'mysql' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'ado_mssql'. +DATABASE_NAME = 'indextank' # Or path to database file if using sqlite3. +DATABASE_USER = '****' # Not used with sqlite3. +DATABASE_PASSWORD = '****' # Not used with sqlite3. +DATABASE_HOST = 'database' # Set to empty string for localhost. Not used with sqlite3. +DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'Etc/GMT+0' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.load_template_source', + 'django.template.loaders.app_directories.load_template_source', +# 'django.template.loaders.eggs.load_template_source', +) + +MIDDLEWARE_CLASSES = ( + 'api.lib.error_logging.ViewErrorLoggingMiddleware', + 'django.middleware.gzip.GZipMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.locale.LocaleMiddleware', +) + +ROOT_URLCONF = 'api.urls' + +TEMPLATE_DIRS = ( + 'templates' + # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +INSTALLED_APPS = ( + 'api', + 'django.contrib.contenttypes', + 'django.contrib.auth' +) + +STORAGE_ENV = 'PROD' + +STATIC_URLS = [ '/_static' ] +ALLOWED_INCLUDE_ROOTS = ('static') + +USER_COOKIE_NAME = "pf_user" +COMMON_DOMAIN = 'localhost' +#SESSION_COOKIE_DOMAIN = COMMON_DOMAIN +FORCE_SCRIPT_NAME = '' +USE_MULTITHREADED_SERVER = True +LOGGER_CONFIG_FILE='logging.conf' + +EMAIL_HOST='localhost' +EMAIL_PORT=25 +EMAIL_HOST_USER='email%localhost' +EMAIL_HOST_PASSWORD='****' diff --git a/api/start_webapp.sh b/api/start_webapp.sh new file mode 100755 index 0000000..9243e21 --- /dev/null +++ b/api/start_webapp.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +./dostartapi.sh +rm -f DONTSTART diff --git a/api/static/README.v1 b/api/static/README.v1 new file mode 100644 index 0000000..33c8e1e --- /dev/null +++ b/api/static/README.v1 @@ -0,0 +1,104 @@ +Indextank API Version 1.0 +------------------------- + +Each IndexTank account has its own domain which can be found in the account's dashboard. +This domain looks like this: + http://.api.indextank.com/ +We'll call this PUBLIC_DOMAIN + +This public domain can be authenticated via Basic auth: + http://:@.api.indextank.com/ +We'll call this AUTHENTICATED_DOMAIN + +Calls to the AUTHENTICATED_DOMAIN will return 401 if the authentication fails. + +Available methods +----------------- + +GET http://api.indextank.com/v1 + Returns this README file + +GET /v1/indexes + Returns a JSON map from index_name to their metadata + +PUT /v1/indexes/ + Creates an index for the given name. + 201: It has been created and returns a JSON string with the new index code + 204: There already existed and index for that name + 409: The max number of indexes has been reached for the account + +GET /v1/indexes/ + Returns JSON metadata for that index + 404: if it doesn't exist + +DELETE /v1/indexes/ + Deletes the index for the given name + 204: if it didn't exist + +PUT /v1/indexes//docs + Expects the following arguments in the JSON body + - docid: a string with the document identifier + - fields: map from field names to their content + - variables: map from variable index to their float value + Indexes the given document in using the given docid and fields + +DELETE /v1/indexes//docs + Expects the following arguments in the JSON body + - docid: a string with the document identifier + Removes the document from the index. + +PUT /v1/indexes//docs/variables + Expects the following parameters in the JSON body + - docid: a string with the document identifier + - variables: map from variable index to their float value + Update the variables of the document. + +GET /v1/indexes//search + Searches the given query in that index + Required parameters in querystring: + - q: the query to perform + Optional parameters in querystring: + - start: integer with the first result to return (default is 0) + - len: integer with the number of results to return (default is 10) + - function: index of the relevance function to be used + - snippet: comma separated list of field names for which to return snippets (requires snipettinng feature) + - fetch: comma separated list of field names for which to return the full content (requires snipettinng feature) + 400: if the query is invalid + +PUT /v1/indexes//promote + Expects the following parameters in the JSON body + - docid: a string with the document identifier + - query: the query for which to promote the document + Promotes the document to the top of the given + +GET /v1/indexes//search//functions + Returns a list of the relevance function indexes and their definitions + +PUT /v1/indexes//search//functions/ + Expects a JSON string with the function definition in the body + Updates the function to the given definition + 400: if the definition is invalid + +DELETE /v1/indexes//search//functions/ + Removes the definition of the function + +GET /v1/indexes//stats + Returns usage stats for the given index + +GET /v1/indexes//query_log + Returns a portion of the query log for the given index + +GET /v1/indexes//autocomplete + Required parameters in querystring: + - fragment: the fragment for which to provide autocomplete suggestions + Optional parameters in querystring: + - callback: the name of the callback function. This causes the output to + be JSONP + Returns a JSON with the suggested queries for the given fragment + Intended to be used with AJAX. + NOTE: this request in unauthenticated since it needs to be made public + to function. if an authenticated request is made it will be + rejected to prevent private key exposition. If you suspect your + key has been expose please regenerate it in your dashboard and + update your clients accordingly. + \ No newline at end of file diff --git a/api/storage.py b/api/storage.py new file mode 100644 index 0000000..2e8614d --- /dev/null +++ b/api/storage.py @@ -0,0 +1,107 @@ + +''' ========================= + Interaction with SimpleDB + ========================= ''' + +import boto +import hashlib +import zlib +import time +import sys +import traceback +import datetime + +from amazon_credential import AMAZON_USER, AMAZON_PASSWORD + +from django.conf import settings + +def get_connection(): + return boto.connect_sdb(AMAZON_USER, AMAZON_PASSWORD) + +def get_ids(index_code, doc_id): + md5 = hashlib.md5() + md5.update(index_code); + md5.update(doc_id); + domain_num = zlib.crc32(md5.digest()) % 100 + domain_id = str(domain_num).rjust(3,'0') + item_id = index_code+'|'+doc_id + return domain_id, item_id + +VALUE_MAX_LENGTH = 1024 +def storage_add(index_code, doc_id, content): + domain_id, item_id = get_ids(index_code, doc_id) + try: + sdb = get_connection() + domain = sdb.get_domain(domain_id) + item = domain.new_item(item_id) + item['timestamp'] = time.time() + limit = VALUE_MAX_LENGTH-2 + for key, txt in content.iteritems(): + n = 0 + i = 1 + while len(txt) > n: + part = '['+txt[n:n+limit]+']' + item['_'+str(i)+'_'+key] = part + n += limit + i += 1 + item.save() + return "" + except: + return sys.exc_info() + +def storage_get(index_code, doc_id): + domain_id, item_id = get_ids(index_code, doc_id) + try: + sdb = get_connection() + domain = sdb.get_domain(domain_id) + item = domain.get_item('|%s|DOC|%s' % (settings.STORAGE_ENV, item_id)) + return item_to_doc(item) if item is not None else None + except: + traceback.print_exc() + return {}, sys.exc_info() + +def storage_del(index_code, doc_id): + domain_id, item_id = get_ids(index_code, doc_id) + try: + sdb = get_connection() + domain = sdb.get_domain(domain_id) + domain.delete_attributes('DOC|'+item_id) + return "" + except: + return sys.exc_info() + + +def item_to_doc(item): + doc = {} + vars = {} + cats = {} + item = dict(item) + + fields = item['item_fields'].split(',') + + for k, v in item.iteritems(): + if k.startswith('user_boost_'): + vars[k[11:]] = v + if k.startswith('user_category_'): + cats[k[14:]] = v + + for f in fields: + v = '' + p = 1 + while True: + part = item.get('_%s_%d' % (f, p)) + if part: + v += part + p += 1 + else: + break + if f == 'timestamp': + doc[f] = '%s (%s)' % (v, datetime.datetime.fromtimestamp(int(v))) + else: + doc[f] = v + + return { 'fields': doc, 'variables': vars, 'categories': cats } + + + + diff --git a/api/testapi.py b/api/testapi.py new file mode 100644 index 0000000..da815ce --- /dev/null +++ b/api/testapi.py @@ -0,0 +1,44 @@ +from lib.indextank.client import ApiClient +import sys +import time + +if len(sys.argv) != 2: + print 'Usage: testapi.py ' + +api = ApiClient(sys.argv[1]) +index = api.get_index('testapi.py') + +if index.exists(): + print 'deleting previous index' + index.delete_index() + +print 'creating index' +index.create_index() + +print 'waiting to start...' +while not index.has_started(): + time.sleep(1) + +print 'adding docs' +index.add_document(1, {'text': 'a b c1'}, variables={0:1, 1:2, 2:3}) +index.add_document(2, {'text': 'a b c2'}, variables={0:2, 1:2, 2:2}) +index.add_document(3, {'text': 'a b c3'}, variables={0:3, 1:2, 2:1}) + +print 'adding functions' +index.add_function(1, 'd[0]') +index.add_function(2, 'd[2]') + +print 'checking functions' +assert index.search('a', scoring_function=1)['results'][0]['docid'] == '3' +assert index.search('a', scoring_function=2)['results'][0]['docid'] == '1' + +print 'checking fetch' +assert index.search('a', scoring_function=2, fetch_fields=['text'])['results'][0]['text'] == 'a b c1' + +print 'checking delete' +index.delete_document(1) + +assert index.search('a', scoring_function=2)['results'][0]['docid'] == '2' + +print 'success' + diff --git a/api/tests.py b/api/tests.py new file mode 100644 index 0000000..e788a90 --- /dev/null +++ b/api/tests.py @@ -0,0 +1,8 @@ +import unittest +import doctest +from api import restapi + +def suite(): + suite = unittest.TestSuite() + suite.addTest(doctest.DocTestSuite(restapi)) + return suite diff --git a/api/thrift/TSCons.py b/api/thrift/TSCons.py new file mode 100644 index 0000000..2404625 --- /dev/null +++ b/api/thrift/TSCons.py @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from os import path +from SCons.Builder import Builder + +def scons_env(env, add=''): + opath = path.dirname(path.abspath('$TARGET')) + lstr = 'thrift --gen cpp -o ' + opath + ' ' + add + ' $SOURCE' + cppbuild = Builder(action = lstr) + env.Append(BUILDERS = {'ThriftCpp' : cppbuild}) + +def gen_cpp(env, dir, file): + scons_env(env) + suffixes = ['_types.h', '_types.cpp'] + targets = map(lambda s: 'gen-cpp/' + file + s, suffixes) + return env.ThriftCpp(targets, dir+file+'.thrift') diff --git a/api/thrift/TSerialization.py b/api/thrift/TSerialization.py new file mode 100644 index 0000000..b19f98a --- /dev/null +++ b/api/thrift/TSerialization.py @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from protocol import TBinaryProtocol +from transport import TTransport + +def serialize(thrift_object, protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()): + transport = TTransport.TMemoryBuffer() + protocol = protocol_factory.getProtocol(transport) + thrift_object.write(protocol) + return transport.getvalue() + +def deserialize(base, buf, protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()): + transport = TTransport.TMemoryBuffer(buf) + protocol = protocol_factory.getProtocol(transport) + base.read(protocol) + return base + diff --git a/api/thrift/Thrift.py b/api/thrift/Thrift.py new file mode 100644 index 0000000..91728a7 --- /dev/null +++ b/api/thrift/Thrift.py @@ -0,0 +1,133 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys + +class TType: + STOP = 0 + VOID = 1 + BOOL = 2 + BYTE = 3 + I08 = 3 + DOUBLE = 4 + I16 = 6 + I32 = 8 + I64 = 10 + STRING = 11 + UTF7 = 11 + STRUCT = 12 + MAP = 13 + SET = 14 + LIST = 15 + UTF8 = 16 + UTF16 = 17 + +class TMessageType: + CALL = 1 + REPLY = 2 + EXCEPTION = 3 + ONEWAY = 4 + +class TProcessor: + + """Base class for procsessor, which works on two streams.""" + + def process(iprot, oprot): + pass + +class TException(Exception): + + """Base class for all thrift exceptions.""" + + # BaseException.message is deprecated in Python v[2.6,3.0) + if (2,6,0) <= sys.version_info < (3,0): + def _get_message(self): + return self._message + def _set_message(self, message): + self._message = message + message = property(_get_message, _set_message) + + def __init__(self, message=None): + Exception.__init__(self, message) + self.message = message + +class TApplicationException(TException): + + """Application level thrift exceptions.""" + + UNKNOWN = 0 + UNKNOWN_METHOD = 1 + INVALID_MESSAGE_TYPE = 2 + WRONG_METHOD_NAME = 3 + BAD_SEQUENCE_ID = 4 + MISSING_RESULT = 5 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + + def __str__(self): + if self.message: + return self.message + elif self.type == self.UNKNOWN_METHOD: + return 'Unknown method' + elif self.type == self.INVALID_MESSAGE_TYPE: + return 'Invalid message type' + elif self.type == self.WRONG_METHOD_NAME: + return 'Wrong method name' + elif self.type == self.BAD_SEQUENCE_ID: + return 'Bad sequence ID' + elif self.type == self.MISSING_RESULT: + return 'Missing result' + else: + return 'Default (unknown) TApplicationException' + + def read(self, iprot): + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.message = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.type = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + oprot.writeStructBegin('TApplicationException') + if self.message != None: + oprot.writeFieldBegin('message', TType.STRING, 1) + oprot.writeString(self.message) + oprot.writeFieldEnd() + if self.type != None: + oprot.writeFieldBegin('type', TType.I32, 2) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() diff --git a/api/thrift/__init__.py b/api/thrift/__init__.py new file mode 100644 index 0000000..48d659c --- /dev/null +++ b/api/thrift/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['Thrift', 'TSCons'] diff --git a/api/thrift/protocol/TBinaryProtocol.py b/api/thrift/protocol/TBinaryProtocol.py new file mode 100644 index 0000000..50c6aa8 --- /dev/null +++ b/api/thrift/protocol/TBinaryProtocol.py @@ -0,0 +1,259 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TProtocol import * +from struct import pack, unpack + +class TBinaryProtocol(TProtocolBase): + + """Binary implementation of the Thrift protocol driver.""" + + # NastyHaxx. Python 2.4+ on 32-bit machines forces hex constants to be + # positive, converting this into a long. If we hardcode the int value + # instead it'll stay in 32 bit-land. + + # VERSION_MASK = 0xffff0000 + VERSION_MASK = -65536 + + # VERSION_1 = 0x80010000 + VERSION_1 = -2147418112 + + TYPE_MASK = 0x000000ff + + def __init__(self, trans, strictRead=False, strictWrite=True): + TProtocolBase.__init__(self, trans) + self.strictRead = strictRead + self.strictWrite = strictWrite + + def writeMessageBegin(self, name, type, seqid): + if self.strictWrite: + self.writeI32(TBinaryProtocol.VERSION_1 | type) + self.writeString(name) + self.writeI32(seqid) + else: + self.writeString(name) + self.writeByte(type) + self.writeI32(seqid) + + def writeMessageEnd(self): + pass + + def writeStructBegin(self, name): + pass + + def writeStructEnd(self): + pass + + def writeFieldBegin(self, name, type, id): + self.writeByte(type) + self.writeI16(id) + + def writeFieldEnd(self): + pass + + def writeFieldStop(self): + self.writeByte(TType.STOP); + + def writeMapBegin(self, ktype, vtype, size): + self.writeByte(ktype) + self.writeByte(vtype) + self.writeI32(size) + + def writeMapEnd(self): + pass + + def writeListBegin(self, etype, size): + self.writeByte(etype) + self.writeI32(size) + + def writeListEnd(self): + pass + + def writeSetBegin(self, etype, size): + self.writeByte(etype) + self.writeI32(size) + + def writeSetEnd(self): + pass + + def writeBool(self, bool): + if bool: + self.writeByte(1) + else: + self.writeByte(0) + + def writeByte(self, byte): + buff = pack("!b", byte) + self.trans.write(buff) + + def writeI16(self, i16): + buff = pack("!h", i16) + self.trans.write(buff) + + def writeI32(self, i32): + buff = pack("!i", i32) + self.trans.write(buff) + + def writeI64(self, i64): + buff = pack("!q", i64) + self.trans.write(buff) + + def writeDouble(self, dub): + buff = pack("!d", dub) + self.trans.write(buff) + + def writeString(self, str): + self.writeI32(len(str)) + self.trans.write(str) + + def readMessageBegin(self): + sz = self.readI32() + if sz < 0: + version = sz & TBinaryProtocol.VERSION_MASK + if version != TBinaryProtocol.VERSION_1: + raise TProtocolException(type=TProtocolException.BAD_VERSION, message='Bad version in readMessageBegin: %d' % (sz)) + type = sz & TBinaryProtocol.TYPE_MASK + name = self.readString() + seqid = self.readI32() + else: + if self.strictRead: + raise TProtocolException(type=TProtocolException.BAD_VERSION, message='No protocol version header') + name = self.trans.readAll(sz) + type = self.readByte() + seqid = self.readI32() + return (name, type, seqid) + + def readMessageEnd(self): + pass + + def readStructBegin(self): + pass + + def readStructEnd(self): + pass + + def readFieldBegin(self): + type = self.readByte() + if type == TType.STOP: + return (None, type, 0) + id = self.readI16() + return (None, type, id) + + def readFieldEnd(self): + pass + + def readMapBegin(self): + ktype = self.readByte() + vtype = self.readByte() + size = self.readI32() + return (ktype, vtype, size) + + def readMapEnd(self): + pass + + def readListBegin(self): + etype = self.readByte() + size = self.readI32() + return (etype, size) + + def readListEnd(self): + pass + + def readSetBegin(self): + etype = self.readByte() + size = self.readI32() + return (etype, size) + + def readSetEnd(self): + pass + + def readBool(self): + byte = self.readByte() + if byte == 0: + return False + return True + + def readByte(self): + buff = self.trans.readAll(1) + val, = unpack('!b', buff) + return val + + def readI16(self): + buff = self.trans.readAll(2) + val, = unpack('!h', buff) + return val + + def readI32(self): + buff = self.trans.readAll(4) + val, = unpack('!i', buff) + return val + + def readI64(self): + buff = self.trans.readAll(8) + val, = unpack('!q', buff) + return val + + def readDouble(self): + buff = self.trans.readAll(8) + val, = unpack('!d', buff) + return val + + def readString(self): + len = self.readI32() + str = self.trans.readAll(len) + return str + + +class TBinaryProtocolFactory: + def __init__(self, strictRead=False, strictWrite=True): + self.strictRead = strictRead + self.strictWrite = strictWrite + + def getProtocol(self, trans): + prot = TBinaryProtocol(trans, self.strictRead, self.strictWrite) + return prot + + +class TBinaryProtocolAccelerated(TBinaryProtocol): + + """C-Accelerated version of TBinaryProtocol. + + This class does not override any of TBinaryProtocol's methods, + but the generated code recognizes it directly and will call into + our C module to do the encoding, bypassing this object entirely. + We inherit from TBinaryProtocol so that the normal TBinaryProtocol + encoding can happen if the fastbinary module doesn't work for some + reason. (TODO(dreiss): Make this happen sanely in more cases.) + + In order to take advantage of the C module, just use + TBinaryProtocolAccelerated instead of TBinaryProtocol. + + NOTE: This code was contributed by an external developer. + The internal Thrift team has reviewed and tested it, + but we cannot guarantee that it is production-ready. + Please feel free to report bugs and/or success stories + to the public mailing list. + """ + + pass + + +class TBinaryProtocolAcceleratedFactory: + def getProtocol(self, trans): + return TBinaryProtocolAccelerated(trans) diff --git a/api/thrift/protocol/TCompactProtocol.py b/api/thrift/protocol/TCompactProtocol.py new file mode 100644 index 0000000..fbc156a --- /dev/null +++ b/api/thrift/protocol/TCompactProtocol.py @@ -0,0 +1,368 @@ +from TProtocol import * +from struct import pack, unpack + +__all__ = ['TCompactProtocol', 'TCompactProtocolFactory'] + +CLEAR = 0 +FIELD_WRITE = 1 +VALUE_WRITE = 2 +CONTAINER_WRITE = 3 +BOOL_WRITE = 4 +FIELD_READ = 5 +CONTAINER_READ = 6 +VALUE_READ = 7 +BOOL_READ = 8 + +def make_helper(v_from, container): + def helper(func): + def nested(self, *args, **kwargs): + assert self.state in (v_from, container), (self.state, v_from, container) + return func(self, *args, **kwargs) + return nested + return helper +writer = make_helper(VALUE_WRITE, CONTAINER_WRITE) +reader = make_helper(VALUE_READ, CONTAINER_READ) + +def makeZigZag(n, bits): + return (n << 1) ^ (n >> (bits - 1)) + +def fromZigZag(n): + return (n >> 1) ^ -(n & 1) + +def writeVarint(trans, n): + out = [] + while True: + if n & ~0x7f == 0: + out.append(n) + break + else: + out.append((n & 0xff) | 0x80) + n = n >> 7 + trans.write(''.join(map(chr, out))) + +def readVarint(trans): + result = 0 + shift = 0 + while True: + x = trans.readAll(1) + byte = ord(x) + result |= (byte & 0x7f) << shift + if byte >> 7 == 0: + return result + shift += 7 + +class CompactType: + TRUE = 1 + FALSE = 2 + BYTE = 0x03 + I16 = 0x04 + I32 = 0x05 + I64 = 0x06 + DOUBLE = 0x07 + BINARY = 0x08 + LIST = 0x09 + SET = 0x0A + MAP = 0x0B + STRUCT = 0x0C + +CTYPES = {TType.BOOL: CompactType.TRUE, # used for collection + TType.BYTE: CompactType.BYTE, + TType.I16: CompactType.I16, + TType.I32: CompactType.I32, + TType.I64: CompactType.I64, + TType.DOUBLE: CompactType.DOUBLE, + TType.STRING: CompactType.BINARY, + TType.STRUCT: CompactType.STRUCT, + TType.LIST: CompactType.LIST, + TType.SET: CompactType.SET, + TType.MAP: CompactType.MAP, + } + +TTYPES = {} +for k, v in CTYPES.items(): + TTYPES[v] = k +TTYPES[CompactType.FALSE] = TType.BOOL +del k +del v + +class TCompactProtocol(TProtocolBase): + "Compact implementation of the Thrift protocol driver." + + PROTOCOL_ID = 0x82 + VERSION = 1 + VERSION_MASK = 0x1f + TYPE_MASK = 0xe0 + TYPE_SHIFT_AMOUNT = 5 + + def __init__(self, trans): + TProtocolBase.__init__(self, trans) + self.state = CLEAR + self.__last_fid = 0 + self.__bool_fid = None + self.__bool_value = None + self.__structs = [] + self.__containers = [] + + def __writeVarint(self, n): + writeVarint(self.trans, n) + + def writeMessageBegin(self, name, type, seqid): + assert self.state == CLEAR + self.__writeUByte(self.PROTOCOL_ID) + self.__writeUByte(self.VERSION | (type << self.TYPE_SHIFT_AMOUNT)) + self.__writeVarint(seqid) + self.__writeString(name) + self.state = VALUE_WRITE + + def writeMessageEnd(self): + assert self.state == VALUE_WRITE + self.state = CLEAR + + def writeStructBegin(self, name): + assert self.state in (CLEAR, CONTAINER_WRITE, VALUE_WRITE), self.state + self.__structs.append((self.state, self.__last_fid)) + self.state = FIELD_WRITE + self.__last_fid = 0 + + def writeStructEnd(self): + assert self.state == FIELD_WRITE + self.state, self.__last_fid = self.__structs.pop() + + def writeFieldStop(self): + self.__writeByte(0) + + def __writeFieldHeader(self, type, fid): + delta = fid - self.__last_fid + if 0 < delta <= 15: + self.__writeUByte(delta << 4 | type) + else: + self.__writeByte(type) + self.__writeI16(fid) + self.__last_fid = fid + + def writeFieldBegin(self, name, type, fid): + assert self.state == FIELD_WRITE, self.state + if type == TType.BOOL: + self.state = BOOL_WRITE + self.__bool_fid = fid + else: + self.state = VALUE_WRITE + self.__writeFieldHeader(CTYPES[type], fid) + + def writeFieldEnd(self): + assert self.state in (VALUE_WRITE, BOOL_WRITE), self.state + self.state = FIELD_WRITE + + def __writeUByte(self, byte): + self.trans.write(pack('!B', byte)) + + def __writeByte(self, byte): + self.trans.write(pack('!b', byte)) + + def __writeI16(self, i16): + self.__writeVarint(makeZigZag(i16, 16)) + + def __writeSize(self, i32): + self.__writeVarint(i32) + + def writeCollectionBegin(self, etype, size): + assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state + if size <= 14: + self.__writeUByte(size << 4 | CTYPES[etype]) + else: + self.__writeUByte(0xf0 | CTYPES[etype]) + self.__writeSize(size) + self.__containers.append(self.state) + self.state = CONTAINER_WRITE + writeSetBegin = writeCollectionBegin + writeListBegin = writeCollectionBegin + + def writeMapBegin(self, ktype, vtype, size): + assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state + if size == 0: + self.__writeByte(0) + else: + self.__writeSize(size) + self.__writeUByte(CTYPES[ktype] << 4 | CTYPES[vtype]) + self.__containers.append(self.state) + self.state = CONTAINER_WRITE + + def writeCollectionEnd(self): + assert self.state == CONTAINER_WRITE, self.state + self.state = self.__containers.pop() + writeMapEnd = writeCollectionEnd + writeSetEnd = writeCollectionEnd + writeListEnd = writeCollectionEnd + + def writeBool(self, bool): + if self.state == BOOL_WRITE: + self.__writeFieldHeader(types[bool], self.__bool_fid) + elif self.state == CONTAINER_WRITE: + self.__writeByte(int(bool)) + else: + raise AssertetionError, "Invalid state in compact protocol" + + writeByte = writer(__writeByte) + writeI16 = writer(__writeI16) + + @writer + def writeI32(self, i32): + self.__writeVarint(makeZigZag(i32, 32)) + + @writer + def writeI64(self, i64): + self.__writeVarint(makeZigZag(i64, 64)) + + @writer + def writeDouble(self, dub): + self.trans.write(pack('!d', dub)) + + def __writeString(self, s): + self.__writeSize(len(s)) + self.trans.write(s) + writeString = writer(__writeString) + + def readFieldBegin(self): + assert self.state == FIELD_READ, self.state + type = self.__readUByte() + if type & 0x0f == TType.STOP: + return (None, 0, 0) + delta = type >> 4 + if delta == 0: + fid = self.__readI16() + else: + fid = self.__last_fid + delta + self.__last_fid = fid + type = type & 0x0f + if type == CompactType.TRUE: + self.state = BOOL_READ + self.__bool_value = True + elif type == CompactType.FALSE: + self.state = BOOL_READ + self.__bool_value = False + else: + self.state = VALUE_READ + return (None, self.__getTType(type), fid) + + def readFieldEnd(self): + assert self.state in (VALUE_READ, BOOL_READ), self.state + self.state = FIELD_READ + + def __readUByte(self): + result, = unpack('!B', self.trans.readAll(1)) + return result + + def __readByte(self): + result, = unpack('!b', self.trans.readAll(1)) + return result + + def __readVarint(self): + return readVarint(self.trans) + + def __readZigZag(self): + return fromZigZag(self.__readVarint()) + + def __readSize(self): + result = self.__readVarint() + if result < 0: + raise TException("Length < 0") + return result + + def readMessageBegin(self): + assert self.state == CLEAR + proto_id = self.__readUByte() + if proto_id != self.PROTOCOL_ID: + raise TProtocolException(TProtocolException.BAD_VERSION, + 'Bad protocol id in the message: %d' % proto_id) + ver_type = self.__readUByte() + type = (ver_type & self.TYPE_MASK) >> self.TYPE_SHIFT_AMOUNT + version = ver_type & self.VERSION_MASK + if version != self.VERSION: + raise TProtocolException(TProtocolException.BAD_VERSION, + 'Bad version: %d (expect %d)' % (version, self.VERSION)) + seqid = self.__readVarint() + name = self.__readString() + return (name, type, seqid) + + def readMessageEnd(self): + assert self.state == VALUE_READ + assert len(self.__structs) == 0 + self.state = CLEAR + + def readStructBegin(self): + assert self.state in (CLEAR, CONTAINER_READ, VALUE_READ), self.state + self.__structs.append((self.state, self.__last_fid)) + self.state = FIELD_READ + self.__last_fid = 0 + + def readStructEnd(self): + assert self.state == FIELD_READ + self.state, self.__last_fid = self.__structs.pop() + + def readCollectionBegin(self): + assert self.state in (VALUE_READ, CONTAINER_READ), self.state + size_type = self.__readUByte() + size = size_type >> 4 + type = self.__getTType(size_type) + if size == 15: + size = self.__readSize() + self.__containers.append(self.state) + self.state = CONTAINER_READ + return type, size + readSetBegin = readCollectionBegin + readListBegin = readCollectionBegin + + def readMapBegin(self): + assert self.state in (VALUE_READ, CONTAINER_READ), self.state + size = self.__readSize() + types = 0 + if size > 0: + types = self.__readUByte() + vtype = self.__getTType(types) + ktype = self.__getTType(types >> 4) + self.__containers.append(self.state) + self.state = CONTAINER_READ + return (ktype, vtype, size) + + def readCollectionEnd(self): + assert self.state == CONTAINER_READ, self.state + self.state = self.__containers.pop() + readSetEnd = readCollectionEnd + readListEnd = readCollectionEnd + readMapEnd = readCollectionEnd + + def readBool(self): + if self.state == BOOL_READ: + return self.__bool_value + elif self.state == CONTAINER_READ: + return bool(self.__readByte()) + else: + raise AssertionError, "Invalid state in compact protocol: %d" % self.state + + readByte = reader(__readByte) + __readI16 = __readZigZag + readI16 = reader(__readZigZag) + readI32 = reader(__readZigZag) + readI64 = reader(__readZigZag) + + @reader + def readDouble(self): + buff = self.trans.readAll(8) + val, = unpack('!d', buff) + return val + + def __readString(self): + len = self.__readSize() + return self.trans.readAll(len) + readString = reader(__readString) + + def __getTType(self, byte): + return TTYPES[byte & 0x0f] + + +class TCompactProtocolFactory: + def __init__(self): + pass + + def getProtocol(self, trans): + return TCompactProtocol(trans) diff --git a/api/thrift/protocol/TProtocol.py b/api/thrift/protocol/TProtocol.py new file mode 100644 index 0000000..be3cb14 --- /dev/null +++ b/api/thrift/protocol/TProtocol.py @@ -0,0 +1,205 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from thrift.Thrift import * + +class TProtocolException(TException): + + """Custom Protocol Exception class""" + + UNKNOWN = 0 + INVALID_DATA = 1 + NEGATIVE_SIZE = 2 + SIZE_LIMIT = 3 + BAD_VERSION = 4 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + +class TProtocolBase: + + """Base class for Thrift protocol driver.""" + + def __init__(self, trans): + self.trans = trans + + def writeMessageBegin(self, name, type, seqid): + pass + + def writeMessageEnd(self): + pass + + def writeStructBegin(self, name): + pass + + def writeStructEnd(self): + pass + + def writeFieldBegin(self, name, type, id): + pass + + def writeFieldEnd(self): + pass + + def writeFieldStop(self): + pass + + def writeMapBegin(self, ktype, vtype, size): + pass + + def writeMapEnd(self): + pass + + def writeListBegin(self, etype, size): + pass + + def writeListEnd(self): + pass + + def writeSetBegin(self, etype, size): + pass + + def writeSetEnd(self): + pass + + def writeBool(self, bool): + pass + + def writeByte(self, byte): + pass + + def writeI16(self, i16): + pass + + def writeI32(self, i32): + pass + + def writeI64(self, i64): + pass + + def writeDouble(self, dub): + pass + + def writeString(self, str): + pass + + def readMessageBegin(self): + pass + + def readMessageEnd(self): + pass + + def readStructBegin(self): + pass + + def readStructEnd(self): + pass + + def readFieldBegin(self): + pass + + def readFieldEnd(self): + pass + + def readMapBegin(self): + pass + + def readMapEnd(self): + pass + + def readListBegin(self): + pass + + def readListEnd(self): + pass + + def readSetBegin(self): + pass + + def readSetEnd(self): + pass + + def readBool(self): + pass + + def readByte(self): + pass + + def readI16(self): + pass + + def readI32(self): + pass + + def readI64(self): + pass + + def readDouble(self): + pass + + def readString(self): + pass + + def skip(self, type): + if type == TType.STOP: + return + elif type == TType.BOOL: + self.readBool() + elif type == TType.BYTE: + self.readByte() + elif type == TType.I16: + self.readI16() + elif type == TType.I32: + self.readI32() + elif type == TType.I64: + self.readI64() + elif type == TType.DOUBLE: + self.readDouble() + elif type == TType.STRING: + self.readString() + elif type == TType.STRUCT: + name = self.readStructBegin() + while True: + (name, type, id) = self.readFieldBegin() + if type == TType.STOP: + break + self.skip(type) + self.readFieldEnd() + self.readStructEnd() + elif type == TType.MAP: + (ktype, vtype, size) = self.readMapBegin() + for i in range(size): + self.skip(ktype) + self.skip(vtype) + self.readMapEnd() + elif type == TType.SET: + (etype, size) = self.readSetBegin() + for i in range(size): + self.skip(etype) + self.readSetEnd() + elif type == TType.LIST: + (etype, size) = self.readListBegin() + for i in range(size): + self.skip(etype) + self.readListEnd() + +class TProtocolFactory: + def getProtocol(self, trans): + pass diff --git a/api/thrift/protocol/__init__.py b/api/thrift/protocol/__init__.py new file mode 100644 index 0000000..01bfe18 --- /dev/null +++ b/api/thrift/protocol/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TProtocol', 'TBinaryProtocol', 'fastbinary'] diff --git a/api/thrift/protocol/fastbinary.c b/api/thrift/protocol/fastbinary.c new file mode 100644 index 0000000..67b215a --- /dev/null +++ b/api/thrift/protocol/fastbinary.c @@ -0,0 +1,1203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include "cStringIO.h" +#include +#include +#include + +/* Fix endianness issues on Solaris */ +#if defined (__SVR4) && defined (__sun) + #if defined(__i386) && !defined(__i386__) + #define __i386__ + #endif + + #ifndef BIG_ENDIAN + #define BIG_ENDIAN (4321) + #endif + #ifndef LITTLE_ENDIAN + #define LITTLE_ENDIAN (1234) + #endif + + /* I386 is LE, even on Solaris */ + #if !defined(BYTE_ORDER) && defined(__i386__) + #define BYTE_ORDER LITTLE_ENDIAN + #endif +#endif + +// TODO(dreiss): defval appears to be unused. Look into removing it. +// TODO(dreiss): Make parse_spec_args recursive, and cache the output +// permanently in the object. (Malloc and orphan.) +// TODO(dreiss): Why do we need cStringIO for reading, why not just char*? +// Can cStringIO let us work with a BufferedTransport? +// TODO(dreiss): Don't ignore the rv from cwrite (maybe). + +/* ====== BEGIN UTILITIES ====== */ + +#define INIT_OUTBUF_SIZE 128 + +// Stolen out of TProtocol.h. +// It would be a huge pain to have both get this from one place. +typedef enum TType { + T_STOP = 0, + T_VOID = 1, + T_BOOL = 2, + T_BYTE = 3, + T_I08 = 3, + T_I16 = 6, + T_I32 = 8, + T_U64 = 9, + T_I64 = 10, + T_DOUBLE = 4, + T_STRING = 11, + T_UTF7 = 11, + T_STRUCT = 12, + T_MAP = 13, + T_SET = 14, + T_LIST = 15, + T_UTF8 = 16, + T_UTF16 = 17 +} TType; + +#ifndef __BYTE_ORDER +# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN) +# define __BYTE_ORDER BYTE_ORDER +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __BIG_ENDIAN BIG_ENDIAN +# else +# error "Cannot determine endianness" +# endif +#endif + +// Same comment as the enum. Sorry. +#if __BYTE_ORDER == __BIG_ENDIAN +# define ntohll(n) (n) +# define htonll(n) (n) +#elif __BYTE_ORDER == __LITTLE_ENDIAN +# if defined(__GNUC__) && defined(__GLIBC__) +# include +# define ntohll(n) bswap_64(n) +# define htonll(n) bswap_64(n) +# else /* GNUC & GLIBC */ +# define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) ) +# define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) ) +# endif /* GNUC & GLIBC */ +#else /* __BYTE_ORDER */ +# error "Can't define htonll or ntohll!" +#endif + +// Doing a benchmark shows that interning actually makes a difference, amazingly. +#define INTERN_STRING(value) _intern_ ## value + +#define INT_CONV_ERROR_OCCURRED(v) ( ((v) == -1) && PyErr_Occurred() ) +#define CHECK_RANGE(v, min, max) ( ((v) <= (max)) && ((v) >= (min)) ) + +// Py_ssize_t was not defined before Python 2.5 +#if (PY_VERSION_HEX < 0x02050000) +typedef int Py_ssize_t; +#endif + +/** + * A cache of the spec_args for a set or list, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + TType element_type; + PyObject* typeargs; +} SetListTypeArgs; + +/** + * A cache of the spec_args for a map, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + TType ktag; + TType vtag; + PyObject* ktypeargs; + PyObject* vtypeargs; +} MapTypeArgs; + +/** + * A cache of the spec_args for a struct, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + PyObject* klass; + PyObject* spec; +} StructTypeArgs; + +/** + * A cache of the item spec from a struct specification, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + int tag; + TType type; + PyObject* attrname; + PyObject* typeargs; + PyObject* defval; +} StructItemSpec; + +/** + * A cache of the two key attributes of a CReadableTransport, + * so we don't have to keep calling PyObject_GetAttr. + */ +typedef struct { + PyObject* stringiobuf; + PyObject* refill_callable; +} DecodeBuffer; + +/** Pointer to interned string to speed up attribute lookup. */ +static PyObject* INTERN_STRING(cstringio_buf); +/** Pointer to interned string to speed up attribute lookup. */ +static PyObject* INTERN_STRING(cstringio_refill); + +static inline bool +check_ssize_t_32(Py_ssize_t len) { + // error from getting the int + if (INT_CONV_ERROR_OCCURRED(len)) { + return false; + } + if (!CHECK_RANGE(len, 0, INT32_MAX)) { + PyErr_SetString(PyExc_OverflowError, "string size out of range"); + return false; + } + return true; +} + +static inline bool +parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) { + long val = PyInt_AsLong(o); + + if (INT_CONV_ERROR_OCCURRED(val)) { + return false; + } + if (!CHECK_RANGE(val, min, max)) { + PyErr_SetString(PyExc_OverflowError, "int out of range"); + return false; + } + + *ret = (int32_t) val; + return true; +} + + +/* --- FUNCTIONS TO PARSE STRUCT SPECIFICATOINS --- */ + +static bool +parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 2) { + PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args"); + return false; + } + + dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->element_type)) { + return false; + } + + dest->typeargs = PyTuple_GET_ITEM(typeargs, 1); + + return true; +} + +static bool +parse_map_args(MapTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 4) { + PyErr_SetString(PyExc_TypeError, "expecting 4 arguments for typeargs to map"); + return false; + } + + dest->ktag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->ktag)) { + return false; + } + + dest->vtag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2)); + if (INT_CONV_ERROR_OCCURRED(dest->vtag)) { + return false; + } + + dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1); + dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3); + + return true; +} + +static bool +parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 2) { + PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for struct args"); + return false; + } + + dest->klass = PyTuple_GET_ITEM(typeargs, 0); + dest->spec = PyTuple_GET_ITEM(typeargs, 1); + + return true; +} + +static int +parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) { + + // i'd like to use ParseArgs here, but it seems to be a bottleneck. + if (PyTuple_Size(spec_tuple) != 5) { + PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for spec tuple"); + return false; + } + + dest->tag = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->tag)) { + return false; + } + + dest->type = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1)); + if (INT_CONV_ERROR_OCCURRED(dest->type)) { + return false; + } + + dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2); + dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3); + dest->defval = PyTuple_GET_ITEM(spec_tuple, 4); + return true; +} + +/* ====== END UTILITIES ====== */ + + +/* ====== BEGIN WRITING FUNCTIONS ====== */ + +/* --- LOW-LEVEL WRITING FUNCTIONS --- */ + +static void writeByte(PyObject* outbuf, int8_t val) { + int8_t net = val; + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int8_t)); +} + +static void writeI16(PyObject* outbuf, int16_t val) { + int16_t net = (int16_t)htons(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int16_t)); +} + +static void writeI32(PyObject* outbuf, int32_t val) { + int32_t net = (int32_t)htonl(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int32_t)); +} + +static void writeI64(PyObject* outbuf, int64_t val) { + int64_t net = (int64_t)htonll(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int64_t)); +} + +static void writeDouble(PyObject* outbuf, double dub) { + // Unfortunately, bitwise_cast doesn't work in C. Bad C! + union { + double f; + int64_t t; + } transfer; + transfer.f = dub; + writeI64(outbuf, transfer.t); +} + + +/* --- MAIN RECURSIVE OUTPUT FUCNTION -- */ + +static int +output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) { + /* + * Refcounting Strategy: + * + * We assume that elements of the thrift_spec tuple are not going to be + * mutated, so we don't ref count those at all. Other than that, we try to + * keep a reference to all the user-created objects while we work with them. + * output_val assumes that a reference is already held. The *caller* is + * responsible for handling references + */ + + switch (type) { + + case T_BOOL: { + int v = PyObject_IsTrue(value); + if (v == -1) { + return false; + } + + writeByte(output, (int8_t) v); + break; + } + case T_I08: { + int32_t val; + + if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) { + return false; + } + + writeByte(output, (int8_t) val); + break; + } + case T_I16: { + int32_t val; + + if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) { + return false; + } + + writeI16(output, (int16_t) val); + break; + } + case T_I32: { + int32_t val; + + if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) { + return false; + } + + writeI32(output, val); + break; + } + case T_I64: { + int64_t nval = PyLong_AsLongLong(value); + + if (INT_CONV_ERROR_OCCURRED(nval)) { + return false; + } + + if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) { + PyErr_SetString(PyExc_OverflowError, "int out of range"); + return false; + } + + writeI64(output, nval); + break; + } + + case T_DOUBLE: { + double nval = PyFloat_AsDouble(value); + if (nval == -1.0 && PyErr_Occurred()) { + return false; + } + + writeDouble(output, nval); + break; + } + + case T_STRING: { + Py_ssize_t len = PyString_Size(value); + + if (!check_ssize_t_32(len)) { + return false; + } + + writeI32(output, (int32_t) len); + PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len); + break; + } + + case T_LIST: + case T_SET: { + Py_ssize_t len; + SetListTypeArgs parsedargs; + PyObject *item; + PyObject *iterator; + + if (!parse_set_list_args(&parsedargs, typeargs)) { + return false; + } + + len = PyObject_Length(value); + + if (!check_ssize_t_32(len)) { + return false; + } + + writeByte(output, parsedargs.element_type); + writeI32(output, (int32_t) len); + + iterator = PyObject_GetIter(value); + if (iterator == NULL) { + return false; + } + + while ((item = PyIter_Next(iterator))) { + if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) { + Py_DECREF(item); + Py_DECREF(iterator); + return false; + } + Py_DECREF(item); + } + + Py_DECREF(iterator); + + if (PyErr_Occurred()) { + return false; + } + + break; + } + + case T_MAP: { + PyObject *k, *v; + Py_ssize_t pos = 0; + Py_ssize_t len; + + MapTypeArgs parsedargs; + + len = PyDict_Size(value); + if (!check_ssize_t_32(len)) { + return false; + } + + if (!parse_map_args(&parsedargs, typeargs)) { + return false; + } + + writeByte(output, parsedargs.ktag); + writeByte(output, parsedargs.vtag); + writeI32(output, len); + + // TODO(bmaurer): should support any mapping, not just dicts + while (PyDict_Next(value, &pos, &k, &v)) { + // TODO(dreiss): Think hard about whether these INCREFs actually + // turn any unsafe scenarios into safe scenarios. + Py_INCREF(k); + Py_INCREF(v); + + if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs) + || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) { + Py_DECREF(k); + Py_DECREF(v); + return false; + } + Py_DECREF(k); + Py_DECREF(v); + } + break; + } + + // TODO(dreiss): Consider breaking this out as a function + // the way we did for decode_struct. + case T_STRUCT: { + StructTypeArgs parsedargs; + Py_ssize_t nspec; + Py_ssize_t i; + + if (!parse_struct_args(&parsedargs, typeargs)) { + return false; + } + + nspec = PyTuple_Size(parsedargs.spec); + + if (nspec == -1) { + return false; + } + + for (i = 0; i < nspec; i++) { + StructItemSpec parsedspec; + PyObject* spec_tuple; + PyObject* instval = NULL; + + spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i); + if (spec_tuple == Py_None) { + continue; + } + + if (!parse_struct_item_spec (&parsedspec, spec_tuple)) { + return false; + } + + instval = PyObject_GetAttr(value, parsedspec.attrname); + + if (!instval) { + return false; + } + + if (instval == Py_None) { + Py_DECREF(instval); + continue; + } + + writeByte(output, (int8_t) parsedspec.type); + writeI16(output, parsedspec.tag); + + if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) { + Py_DECREF(instval); + return false; + } + + Py_DECREF(instval); + } + + writeByte(output, (int8_t)T_STOP); + break; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return false; + + } + + return true; +} + + +/* --- TOP-LEVEL WRAPPER FOR OUTPUT -- */ + +static PyObject * +encode_binary(PyObject *self, PyObject *args) { + PyObject* enc_obj; + PyObject* type_args; + PyObject* buf; + PyObject* ret = NULL; + + if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) { + return NULL; + } + + buf = PycStringIO->NewOutput(INIT_OUTBUF_SIZE); + if (output_val(buf, enc_obj, T_STRUCT, type_args)) { + ret = PycStringIO->cgetvalue(buf); + } + + Py_DECREF(buf); + return ret; +} + +/* ====== END WRITING FUNCTIONS ====== */ + + +/* ====== BEGIN READING FUNCTIONS ====== */ + +/* --- LOW-LEVEL READING FUNCTIONS --- */ + +static void +free_decodebuf(DecodeBuffer* d) { + Py_XDECREF(d->stringiobuf); + Py_XDECREF(d->refill_callable); +} + +static bool +decode_buffer_from_obj(DecodeBuffer* dest, PyObject* obj) { + dest->stringiobuf = PyObject_GetAttr(obj, INTERN_STRING(cstringio_buf)); + if (!dest->stringiobuf) { + return false; + } + + if (!PycStringIO_InputCheck(dest->stringiobuf)) { + free_decodebuf(dest); + PyErr_SetString(PyExc_TypeError, "expecting stringio input"); + return false; + } + + dest->refill_callable = PyObject_GetAttr(obj, INTERN_STRING(cstringio_refill)); + + if(!dest->refill_callable) { + free_decodebuf(dest); + return false; + } + + if (!PyCallable_Check(dest->refill_callable)) { + free_decodebuf(dest); + PyErr_SetString(PyExc_TypeError, "expecting callable"); + return false; + } + + return true; +} + +static bool readBytes(DecodeBuffer* input, char** output, int len) { + int read; + + // TODO(dreiss): Don't fear the malloc. Think about taking a copy of + // the partial read instead of forcing the transport + // to prepend it to its buffer. + + read = PycStringIO->cread(input->stringiobuf, output, len); + + if (read == len) { + return true; + } else if (read == -1) { + return false; + } else { + PyObject* newiobuf; + + // using building functions as this is a rare codepath + newiobuf = PyObject_CallFunction( + input->refill_callable, "s#i", *output, read, len, NULL); + if (newiobuf == NULL) { + return false; + } + + // must do this *AFTER* the call so that we don't deref the io buffer + Py_CLEAR(input->stringiobuf); + input->stringiobuf = newiobuf; + + read = PycStringIO->cread(input->stringiobuf, output, len); + + if (read == len) { + return true; + } else if (read == -1) { + return false; + } else { + // TODO(dreiss): This could be a valid code path for big binary blobs. + PyErr_SetString(PyExc_TypeError, + "refill claimed to have refilled the buffer, but didn't!!"); + return false; + } + } +} + +static int8_t readByte(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int8_t))) { + return -1; + } + + return *(int8_t*) buf; +} + +static int16_t readI16(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int16_t))) { + return -1; + } + + return (int16_t) ntohs(*(int16_t*) buf); +} + +static int32_t readI32(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int32_t))) { + return -1; + } + return (int32_t) ntohl(*(int32_t*) buf); +} + + +static int64_t readI64(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int64_t))) { + return -1; + } + + return (int64_t) ntohll(*(int64_t*) buf); +} + +static double readDouble(DecodeBuffer* input) { + union { + int64_t f; + double t; + } transfer; + + transfer.f = readI64(input); + if (transfer.f == -1) { + return -1; + } + return transfer.t; +} + +static bool +checkTypeByte(DecodeBuffer* input, TType expected) { + TType got = readByte(input); + if (INT_CONV_ERROR_OCCURRED(got)) { + return false; + } + + if (expected != got) { + PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field"); + return false; + } + return true; +} + +static bool +skip(DecodeBuffer* input, TType type) { +#define SKIPBYTES(n) \ + do { \ + if (!readBytes(input, &dummy_buf, (n))) { \ + return false; \ + } \ + } while(0) + + char* dummy_buf; + + switch (type) { + + case T_BOOL: + case T_I08: SKIPBYTES(1); break; + case T_I16: SKIPBYTES(2); break; + case T_I32: SKIPBYTES(4); break; + case T_I64: + case T_DOUBLE: SKIPBYTES(8); break; + + case T_STRING: { + // TODO(dreiss): Find out if these check_ssize_t32s are really necessary. + int len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + SKIPBYTES(len); + break; + } + + case T_LIST: + case T_SET: { + TType etype; + int len, i; + + etype = readByte(input); + if (etype == -1) { + return false; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + for (i = 0; i < len; i++) { + if (!skip(input, etype)) { + return false; + } + } + break; + } + + case T_MAP: { + TType ktype, vtype; + int len, i; + + ktype = readByte(input); + if (ktype == -1) { + return false; + } + + vtype = readByte(input); + if (vtype == -1) { + return false; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + for (i = 0; i < len; i++) { + if (!(skip(input, ktype) && skip(input, vtype))) { + return false; + } + } + break; + } + + case T_STRUCT: { + while (true) { + TType type; + + type = readByte(input); + if (type == -1) { + return false; + } + + if (type == T_STOP) + break; + + SKIPBYTES(2); // tag + if (!skip(input, type)) { + return false; + } + } + break; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return false; + + } + + return true; + +#undef SKIPBYTES +} + + +/* --- HELPER FUNCTION FOR DECODE_VAL --- */ + +static PyObject* +decode_val(DecodeBuffer* input, TType type, PyObject* typeargs); + +static bool +decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) { + int spec_seq_len = PyTuple_Size(spec_seq); + if (spec_seq_len == -1) { + return false; + } + + while (true) { + TType type; + int16_t tag; + PyObject* item_spec; + PyObject* fieldval = NULL; + StructItemSpec parsedspec; + + type = readByte(input); + if (type == -1) { + return false; + } + if (type == T_STOP) { + break; + } + tag = readI16(input); + if (INT_CONV_ERROR_OCCURRED(tag)) { + return false; + } + if (tag >= 0 && tag < spec_seq_len) { + item_spec = PyTuple_GET_ITEM(spec_seq, tag); + } else { + item_spec = Py_None; + } + + if (item_spec == Py_None) { + if (!skip(input, type)) { + return false; + } else { + continue; + } + } + + if (!parse_struct_item_spec(&parsedspec, item_spec)) { + return false; + } + if (parsedspec.type != type) { + if (!skip(input, type)) { + PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading and can't be skipped"); + return false; + } else { + continue; + } + } + + fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs); + if (fieldval == NULL) { + return false; + } + + if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) { + Py_DECREF(fieldval); + return false; + } + Py_DECREF(fieldval); + } + return true; +} + + +/* --- MAIN RECURSIVE INPUT FUCNTION --- */ + +// Returns a new reference. +static PyObject* +decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) { + switch (type) { + + case T_BOOL: { + int8_t v = readByte(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + + switch (v) { + case 0: Py_RETURN_FALSE; + case 1: Py_RETURN_TRUE; + // Don't laugh. This is a potentially serious issue. + default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL; + } + break; + } + case T_I08: { + int8_t v = readByte(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + + return PyInt_FromLong(v); + } + case T_I16: { + int16_t v = readI16(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + return PyInt_FromLong(v); + } + case T_I32: { + int32_t v = readI32(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + return PyInt_FromLong(v); + } + + case T_I64: { + int64_t v = readI64(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + // TODO(dreiss): Find out if we can take this fastpath always when + // sizeof(long) == sizeof(long long). + if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) { + return PyInt_FromLong((long) v); + } + + return PyLong_FromLongLong(v); + } + + case T_DOUBLE: { + double v = readDouble(input); + if (v == -1.0 && PyErr_Occurred()) { + return false; + } + return PyFloat_FromDouble(v); + } + + case T_STRING: { + Py_ssize_t len = readI32(input); + char* buf; + if (!readBytes(input, &buf, len)) { + return NULL; + } + + return PyString_FromStringAndSize(buf, len); + } + + case T_LIST: + case T_SET: { + SetListTypeArgs parsedargs; + int32_t len; + PyObject* ret = NULL; + int i; + + if (!parse_set_list_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!checkTypeByte(input, parsedargs.element_type)) { + return NULL; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return NULL; + } + + ret = PyList_New(len); + if (!ret) { + return NULL; + } + + for (i = 0; i < len; i++) { + PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs); + if (!item) { + Py_DECREF(ret); + return NULL; + } + PyList_SET_ITEM(ret, i, item); + } + + // TODO(dreiss): Consider biting the bullet and making two separate cases + // for list and set, avoiding this post facto conversion. + if (type == T_SET) { + PyObject* setret; +#if (PY_VERSION_HEX < 0x02050000) + // hack needed for older versions + setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL); +#else + // official version + setret = PySet_New(ret); +#endif + Py_DECREF(ret); + return setret; + } + return ret; + } + + case T_MAP: { + int32_t len; + int i; + MapTypeArgs parsedargs; + PyObject* ret = NULL; + + if (!parse_map_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!checkTypeByte(input, parsedargs.ktag)) { + return NULL; + } + if (!checkTypeByte(input, parsedargs.vtag)) { + return NULL; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + ret = PyDict_New(); + if (!ret) { + goto error; + } + + for (i = 0; i < len; i++) { + PyObject* k = NULL; + PyObject* v = NULL; + k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs); + if (k == NULL) { + goto loop_error; + } + v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs); + if (v == NULL) { + goto loop_error; + } + if (PyDict_SetItem(ret, k, v) == -1) { + goto loop_error; + } + + Py_DECREF(k); + Py_DECREF(v); + continue; + + // Yuck! Destructors, anyone? + loop_error: + Py_XDECREF(k); + Py_XDECREF(v); + goto error; + } + + return ret; + + error: + Py_XDECREF(ret); + return NULL; + } + + case T_STRUCT: { + StructTypeArgs parsedargs; + if (!parse_struct_args(&parsedargs, typeargs)) { + return NULL; + } + + PyObject* ret = PyObject_CallObject(parsedargs.klass, NULL); + if (!ret) { + return NULL; + } + + if (!decode_struct(input, ret, parsedargs.spec)) { + Py_DECREF(ret); + return NULL; + } + + return ret; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return NULL; + } +} + + +/* --- TOP-LEVEL WRAPPER FOR INPUT -- */ + +static PyObject* +decode_binary(PyObject *self, PyObject *args) { + PyObject* output_obj = NULL; + PyObject* transport = NULL; + PyObject* typeargs = NULL; + StructTypeArgs parsedargs; + DecodeBuffer input = {}; + + if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) { + return NULL; + } + + if (!parse_struct_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!decode_buffer_from_obj(&input, transport)) { + return NULL; + } + + if (!decode_struct(&input, output_obj, parsedargs.spec)) { + free_decodebuf(&input); + return NULL; + } + + free_decodebuf(&input); + + Py_RETURN_NONE; +} + +/* ====== END READING FUNCTIONS ====== */ + + +/* -- PYTHON MODULE SETUP STUFF --- */ + +static PyMethodDef ThriftFastBinaryMethods[] = { + + {"encode_binary", encode_binary, METH_VARARGS, ""}, + {"decode_binary", decode_binary, METH_VARARGS, ""}, + + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +PyMODINIT_FUNC +initfastbinary(void) { +#define INIT_INTERN_STRING(value) \ + do { \ + INTERN_STRING(value) = PyString_InternFromString(#value); \ + if(!INTERN_STRING(value)) return; \ + } while(0) + + INIT_INTERN_STRING(cstringio_buf); + INIT_INTERN_STRING(cstringio_refill); +#undef INIT_INTERN_STRING + + PycString_IMPORT; + if (PycStringIO == NULL) return; + + (void) Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods); +} diff --git a/api/thrift/server/THttpServer.py b/api/thrift/server/THttpServer.py new file mode 100644 index 0000000..3047d9c --- /dev/null +++ b/api/thrift/server/THttpServer.py @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import BaseHTTPServer + +from thrift.server import TServer +from thrift.transport import TTransport + +class ResponseException(Exception): + """Allows handlers to override the HTTP response + + Normally, THttpServer always sends a 200 response. If a handler wants + to override this behavior (e.g., to simulate a misconfigured or + overloaded web server during testing), it can raise a ResponseException. + The function passed to the constructor will be called with the + RequestHandler as its only argument. + """ + def __init__(self, handler): + self.handler = handler + + +class THttpServer(TServer.TServer): + """A simple HTTP-based Thrift server + + This class is not very performant, but it is useful (for example) for + acting as a mock version of an Apache-based PHP Thrift endpoint.""" + + def __init__(self, processor, server_address, + inputProtocolFactory, outputProtocolFactory = None, + server_class = BaseHTTPServer.HTTPServer): + """Set up protocol factories and HTTP server. + + See BaseHTTPServer for server_address. + See TServer for protocol factories.""" + + if outputProtocolFactory is None: + outputProtocolFactory = inputProtocolFactory + + TServer.TServer.__init__(self, processor, None, None, None, + inputProtocolFactory, outputProtocolFactory) + + thttpserver = self + + class RequestHander(BaseHTTPServer.BaseHTTPRequestHandler): + def do_POST(self): + # Don't care about the request path. + itrans = TTransport.TFileObjectTransport(self.rfile) + otrans = TTransport.TFileObjectTransport(self.wfile) + itrans = TTransport.TBufferedTransport(itrans, int(self.headers['Content-Length'])) + otrans = TTransport.TMemoryBuffer() + iprot = thttpserver.inputProtocolFactory.getProtocol(itrans) + oprot = thttpserver.outputProtocolFactory.getProtocol(otrans) + try: + thttpserver.processor.process(iprot, oprot) + except ResponseException, exn: + exn.handler(self) + else: + self.send_response(200) + self.send_header("content-type", "application/x-thrift") + self.end_headers() + self.wfile.write(otrans.getvalue()) + + self.httpd = server_class(server_address, RequestHander) + + def serve(self): + self.httpd.serve_forever() diff --git a/api/thrift/server/TNonblockingServer.py b/api/thrift/server/TNonblockingServer.py new file mode 100644 index 0000000..ea348a0 --- /dev/null +++ b/api/thrift/server/TNonblockingServer.py @@ -0,0 +1,310 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +"""Implementation of non-blocking server. + +The main idea of the server is reciving and sending requests +only from main thread. + +It also makes thread pool server in tasks terms, not connections. +""" +import threading +import socket +import Queue +import select +import struct +import logging + +from thrift.transport import TTransport +from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory + +__all__ = ['TNonblockingServer'] + +class Worker(threading.Thread): + """Worker is a small helper to process incoming connection.""" + def __init__(self, queue): + threading.Thread.__init__(self) + self.queue = queue + + def run(self): + """Process queries from task queue, stop if processor is None.""" + while True: + try: + processor, iprot, oprot, otrans, callback = self.queue.get() + if processor is None: + break + processor.process(iprot, oprot) + callback(True, otrans.getvalue()) + except Exception: + logging.exception("Exception while processing request") + callback(False, '') + +WAIT_LEN = 0 +WAIT_MESSAGE = 1 +WAIT_PROCESS = 2 +SEND_ANSWER = 3 +CLOSED = 4 + +def locked(func): + "Decorator which locks self.lock." + def nested(self, *args, **kwargs): + self.lock.acquire() + try: + return func(self, *args, **kwargs) + finally: + self.lock.release() + return nested + +def socket_exception(func): + "Decorator close object on socket.error." + def read(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except socket.error: + self.close() + return read + +class Connection: + """Basic class is represented connection. + + It can be in state: + WAIT_LEN --- connection is reading request len. + WAIT_MESSAGE --- connection is reading request. + WAIT_PROCESS --- connection has just read whole request and + waits for call ready routine. + SEND_ANSWER --- connection is sending answer string (including length + of answer). + CLOSED --- socket was closed and connection should be deleted. + """ + def __init__(self, new_socket, wake_up): + self.socket = new_socket + self.socket.setblocking(False) + self.status = WAIT_LEN + self.len = 0 + self.message = '' + self.lock = threading.Lock() + self.wake_up = wake_up + + def _read_len(self): + """Reads length of request. + + It's really paranoic routine and it may be replaced by + self.socket.recv(4).""" + read = self.socket.recv(4 - len(self.message)) + if len(read) == 0: + # if we read 0 bytes and self.message is empty, it means client close + # connection + if len(self.message) != 0: + logging.error("can't read frame size from socket") + self.close() + return + self.message += read + if len(self.message) == 4: + self.len, = struct.unpack('!i', self.message) + if self.len < 0: + logging.error("negative frame size, it seems client"\ + " doesn't use FramedTransport") + self.close() + elif self.len == 0: + logging.error("empty frame, it's really strange") + self.close() + else: + self.message = '' + self.status = WAIT_MESSAGE + + @socket_exception + def read(self): + """Reads data from stream and switch state.""" + assert self.status in (WAIT_LEN, WAIT_MESSAGE) + if self.status == WAIT_LEN: + self._read_len() + # go back to the main loop here for simplicity instead of + # falling through, even though there is a good chance that + # the message is already available + elif self.status == WAIT_MESSAGE: + read = self.socket.recv(self.len - len(self.message)) + if len(read) == 0: + logging.error("can't read frame from socket (get %d of %d bytes)" % + (len(self.message), self.len)) + self.close() + return + self.message += read + if len(self.message) == self.len: + self.status = WAIT_PROCESS + + @socket_exception + def write(self): + """Writes data from socket and switch state.""" + assert self.status == SEND_ANSWER + sent = self.socket.send(self.message) + if sent == len(self.message): + self.status = WAIT_LEN + self.message = '' + self.len = 0 + else: + self.message = self.message[sent:] + + @locked + def ready(self, all_ok, message): + """Callback function for switching state and waking up main thread. + + This function is the only function witch can be called asynchronous. + + The ready can switch Connection to three states: + WAIT_LEN if request was oneway. + SEND_ANSWER if request was processed in normal way. + CLOSED if request throws unexpected exception. + + The one wakes up main thread. + """ + assert self.status == WAIT_PROCESS + if not all_ok: + self.close() + self.wake_up() + return + self.len = '' + if len(message) == 0: + # it was a oneway request, do not write answer + self.message = '' + self.status = WAIT_LEN + else: + self.message = struct.pack('!i', len(message)) + message + self.status = SEND_ANSWER + self.wake_up() + + @locked + def is_writeable(self): + "Returns True if connection should be added to write list of select." + return self.status == SEND_ANSWER + + # it's not necessary, but... + @locked + def is_readable(self): + "Returns True if connection should be added to read list of select." + return self.status in (WAIT_LEN, WAIT_MESSAGE) + + @locked + def is_closed(self): + "Returns True if connection is closed." + return self.status == CLOSED + + def fileno(self): + "Returns the file descriptor of the associated socket." + return self.socket.fileno() + + def close(self): + "Closes connection" + self.status = CLOSED + self.socket.close() + +class TNonblockingServer: + """Non-blocking server.""" + def __init__(self, processor, lsocket, inputProtocolFactory=None, + outputProtocolFactory=None, threads=10): + self.processor = processor + self.socket = lsocket + self.in_protocol = inputProtocolFactory or TBinaryProtocolFactory() + self.out_protocol = outputProtocolFactory or self.in_protocol + self.threads = int(threads) + self.clients = {} + self.tasks = Queue.Queue() + self._read, self._write = socket.socketpair() + self.prepared = False + + def setNumThreads(self, num): + """Set the number of worker threads that should be created.""" + # implement ThreadPool interface + assert not self.prepared, "You can't change number of threads for working server" + self.threads = num + + def prepare(self): + """Prepares server for serve requests.""" + self.socket.listen() + for _ in xrange(self.threads): + thread = Worker(self.tasks) + thread.setDaemon(True) + thread.start() + self.prepared = True + + def wake_up(self): + """Wake up main thread. + + The server usualy waits in select call in we should terminate one. + The simplest way is using socketpair. + + Select always wait to read from the first socket of socketpair. + + In this case, we can just write anything to the second socket from + socketpair.""" + self._write.send('1') + + def _select(self): + """Does select on open connections.""" + readable = [self.socket.handle.fileno(), self._read.fileno()] + writable = [] + for i, connection in self.clients.items(): + if connection.is_readable(): + readable.append(connection.fileno()) + if connection.is_writeable(): + writable.append(connection.fileno()) + if connection.is_closed(): + del self.clients[i] + return select.select(readable, writable, readable) + + def handle(self): + """Handle requests. + + WARNING! You must call prepare BEFORE calling handle. + """ + assert self.prepared, "You have to call prepare before handle" + rset, wset, xset = self._select() + for readable in rset: + if readable == self._read.fileno(): + # don't care i just need to clean readable flag + self._read.recv(1024) + elif readable == self.socket.handle.fileno(): + client = self.socket.accept().handle + self.clients[client.fileno()] = Connection(client, self.wake_up) + else: + connection = self.clients[readable] + connection.read() + if connection.status == WAIT_PROCESS: + itransport = TTransport.TMemoryBuffer(connection.message) + otransport = TTransport.TMemoryBuffer() + iprot = self.in_protocol.getProtocol(itransport) + oprot = self.out_protocol.getProtocol(otransport) + self.tasks.put([self.processor, iprot, oprot, + otransport, connection.ready]) + for writeable in wset: + self.clients[writeable].write() + for oob in xset: + self.clients[oob].close() + del self.clients[oob] + + def close(self): + """Closes the server.""" + for _ in xrange(self.threads): + self.tasks.put([None, None, None, None, None]) + self.socket.close() + self.prepared = False + + def serve(self): + """Serve forever.""" + self.prepare() + while True: + self.handle() diff --git a/api/thrift/server/TServer.py b/api/thrift/server/TServer.py new file mode 100644 index 0000000..8456e2d --- /dev/null +++ b/api/thrift/server/TServer.py @@ -0,0 +1,274 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import logging +import sys +import os +import traceback +import threading +import Queue + +from thrift.Thrift import TProcessor +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol + +class TServer: + + """Base interface for a server, which must have a serve method.""" + + """ 3 constructors for all servers: + 1) (processor, serverTransport) + 2) (processor, serverTransport, transportFactory, protocolFactory) + 3) (processor, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory)""" + def __init__(self, *args): + if (len(args) == 2): + self.__initArgs__(args[0], args[1], + TTransport.TTransportFactoryBase(), + TTransport.TTransportFactoryBase(), + TBinaryProtocol.TBinaryProtocolFactory(), + TBinaryProtocol.TBinaryProtocolFactory()) + elif (len(args) == 4): + self.__initArgs__(args[0], args[1], args[2], args[2], args[3], args[3]) + elif (len(args) == 6): + self.__initArgs__(args[0], args[1], args[2], args[3], args[4], args[5]) + + def __initArgs__(self, processor, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory): + self.processor = processor + self.serverTransport = serverTransport + self.inputTransportFactory = inputTransportFactory + self.outputTransportFactory = outputTransportFactory + self.inputProtocolFactory = inputProtocolFactory + self.outputProtocolFactory = outputProtocolFactory + + def serve(self): + pass + +class TSimpleServer(TServer): + + """Simple single-threaded server that just pumps around one transport.""" + + def __init__(self, *args): + TServer.__init__(self, *args) + + def serve(self): + self.serverTransport.listen() + while True: + client = self.serverTransport.accept() + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + +class TThreadedServer(TServer): + + """Threaded server that spawns a new thread per each connection.""" + + def __init__(self, *args, **kwargs): + TServer.__init__(self, *args) + self.daemon = kwargs.get("daemon", False) + + def serve(self): + self.serverTransport.listen() + while True: + try: + client = self.serverTransport.accept() + t = threading.Thread(target = self.handle, args=(client,)) + t.setDaemon(self.daemon) + t.start() + except KeyboardInterrupt: + raise + except Exception, x: + logging.exception(x) + + def handle(self, client): + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + +class TThreadPoolServer(TServer): + + """Server with a fixed size pool of threads which service requests.""" + + def __init__(self, *args, **kwargs): + TServer.__init__(self, *args) + self.clients = Queue.Queue() + self.threads = 10 + self.daemon = kwargs.get("daemon", False) + + def setNumThreads(self, num): + """Set the number of worker threads that should be created""" + self.threads = num + + def serveThread(self): + """Loop around getting clients from the shared queue and process them.""" + while True: + try: + client = self.clients.get() + self.serveClient(client) + except Exception, x: + logging.exception(x) + + def serveClient(self, client): + """Process input/output from a client for as long as possible""" + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + + def serve(self): + """Start a fixed number of worker threads and put client into a queue""" + for i in range(self.threads): + try: + t = threading.Thread(target = self.serveThread) + t.setDaemon(self.daemon) + t.start() + except Exception, x: + logging.exception(x) + + # Pump the socket for clients + self.serverTransport.listen() + while True: + try: + client = self.serverTransport.accept() + self.clients.put(client) + except Exception, x: + logging.exception(x) + + +class TForkingServer(TServer): + + """A Thrift server that forks a new process for each request""" + """ + This is more scalable than the threaded server as it does not cause + GIL contention. + + Note that this has different semantics from the threading server. + Specifically, updates to shared variables will no longer be shared. + It will also not work on windows. + + This code is heavily inspired by SocketServer.ForkingMixIn in the + Python stdlib. + """ + + def __init__(self, *args): + TServer.__init__(self, *args) + self.children = [] + + def serve(self): + def try_close(file): + try: + file.close() + except IOError, e: + logging.warning(e, exc_info=True) + + + self.serverTransport.listen() + while True: + client = self.serverTransport.accept() + try: + pid = os.fork() + + if pid: # parent + # add before collect, otherwise you race w/ waitpid + self.children.append(pid) + self.collect_children() + + # Parent must close socket or the connection may not get + # closed promptly + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + try_close(itrans) + try_close(otrans) + else: + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + + ecode = 0 + try: + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, e: + logging.exception(e) + ecode = 1 + finally: + try_close(itrans) + try_close(otrans) + + os._exit(ecode) + + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + + def collect_children(self): + while self.children: + try: + pid, status = os.waitpid(0, os.WNOHANG) + except os.error: + pid = None + + if pid: + self.children.remove(pid) + else: + break + + diff --git a/api/thrift/server/__init__.py b/api/thrift/server/__init__.py new file mode 100644 index 0000000..1bf6e25 --- /dev/null +++ b/api/thrift/server/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TServer', 'TNonblockingServer'] diff --git a/api/thrift/transport/THttpClient.py b/api/thrift/transport/THttpClient.py new file mode 100644 index 0000000..5026978 --- /dev/null +++ b/api/thrift/transport/THttpClient.py @@ -0,0 +1,126 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TTransport import * +from cStringIO import StringIO + +import urlparse +import httplib +import warnings +import socket + +class THttpClient(TTransportBase): + + """Http implementation of TTransport base.""" + + def __init__(self, uri_or_host, port=None, path=None): + """THttpClient supports two different types constructor parameters. + + THttpClient(host, port, path) - deprecated + THttpClient(uri) + + Only the second supports https.""" + + if port is not None: + warnings.warn("Please use the THttpClient('http://host:port/path') syntax", DeprecationWarning, stacklevel=2) + self.host = uri_or_host + self.port = port + assert path + self.path = path + self.scheme = 'http' + else: + parsed = urlparse.urlparse(uri_or_host) + self.scheme = parsed.scheme + assert self.scheme in ('http', 'https') + if self.scheme == 'http': + self.port = parsed.port or httplib.HTTP_PORT + elif self.scheme == 'https': + self.port = parsed.port or httplib.HTTPS_PORT + self.host = parsed.hostname + self.path = parsed.path + if parsed.query: + self.path += '?%s' % parsed.query + self.__wbuf = StringIO() + self.__http = None + self.__timeout = None + + def open(self): + if self.scheme == 'http': + self.__http = httplib.HTTP(self.host, self.port) + else: + self.__http = httplib.HTTPS(self.host, self.port) + + def close(self): + self.__http.close() + self.__http = None + + def isOpen(self): + return self.__http != None + + def setTimeout(self, ms): + if not hasattr(socket, 'getdefaulttimeout'): + raise NotImplementedError + + if ms is None: + self.__timeout = None + else: + self.__timeout = ms/1000.0 + + def read(self, sz): + return self.__http.file.read(sz) + + def write(self, buf): + self.__wbuf.write(buf) + + def __withTimeout(f): + def _f(*args, **kwargs): + orig_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(args[0].__timeout) + result = f(*args, **kwargs) + socket.setdefaulttimeout(orig_timeout) + return result + return _f + + def flush(self): + if self.isOpen(): + self.close() + self.open(); + + # Pull data out of buffer + data = self.__wbuf.getvalue() + self.__wbuf = StringIO() + + # HTTP request + self.__http.putrequest('POST', self.path) + + # Write headers + self.__http.putheader('Host', self.host) + self.__http.putheader('Content-Type', 'application/x-thrift') + self.__http.putheader('Content-Length', str(len(data))) + self.__http.endheaders() + + # Write payload + self.__http.send(data) + + # Get reply to flush the request + self.code, self.message, self.headers = self.__http.getreply() + + # Decorate if we know how to timeout + if hasattr(socket, 'getdefaulttimeout'): + flush = __withTimeout(flush) diff --git a/api/thrift/transport/TSocket.py b/api/thrift/transport/TSocket.py new file mode 100644 index 0000000..d77e358 --- /dev/null +++ b/api/thrift/transport/TSocket.py @@ -0,0 +1,163 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TTransport import * +import os +import errno +import socket +import sys + +class TSocketBase(TTransportBase): + def _resolveAddr(self): + if self._unix_socket is not None: + return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, self._unix_socket)] + else: + return socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE | socket.AI_ADDRCONFIG) + + def close(self): + if self.handle: + self.handle.close() + self.handle = None + +class TSocket(TSocketBase): + """Socket implementation of TTransport base.""" + + def __init__(self, host='localhost', port=9090, unix_socket=None): + """Initialize a TSocket + + @param host(str) The host to connect to. + @param port(int) The (TCP) port to connect to. + @param unix_socket(str) The filename of a unix socket to connect to. + (host and port will be ignored.) + """ + + self.host = host + self.port = port + self.handle = None + self._unix_socket = unix_socket + self._timeout = None + + def setHandle(self, h): + self.handle = h + + def isOpen(self): + return self.handle != None + + def setTimeout(self, ms): + if ms is None: + self._timeout = None + else: + self._timeout = ms/1000.0 + + if (self.handle != None): + self.handle.settimeout(self._timeout) + + def open(self): + try: + res0 = self._resolveAddr() + for res in res0: + self.handle = socket.socket(res[0], res[1]) + self.handle.settimeout(self._timeout) + try: + self.handle.connect(res[4]) + except socket.error, e: + if res is not res0[-1]: + continue + else: + raise e + break + except socket.error, e: + if self._unix_socket: + message = 'Could not connect to socket %s' % self._unix_socket + else: + message = 'Could not connect to %s:%d' % (self.host, self.port) + raise TTransportException(type=TTransportException.NOT_OPEN, message=message) + + def read(self, sz): + try: + buff = self.handle.recv(sz) + except socket.error, e: + if (e.args[0] == errno.ECONNRESET and + (sys.platform == 'darwin' or sys.platform.startswith('freebsd'))): + # freebsd and Mach don't follow POSIX semantic of recv + # and fail with ECONNRESET if peer performed shutdown. + # See corresponding comment and code in TSocket::read() + # in lib/cpp/src/transport/TSocket.cpp. + self.close() + # Trigger the check to raise the END_OF_FILE exception below. + buff = '' + else: + raise + if len(buff) == 0: + raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket read 0 bytes') + return buff + + def write(self, buff): + if not self.handle: + raise TTransportException(type=TTransportException.NOT_OPEN, message='Transport not open') + sent = 0 + have = len(buff) + while sent < have: + plus = self.handle.send(buff) + if plus == 0: + raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket sent 0 bytes') + sent += plus + buff = buff[plus:] + + def flush(self): + pass + +class TServerSocket(TSocketBase, TServerTransportBase): + """Socket implementation of TServerTransport base.""" + + def __init__(self, port=9090, unix_socket=None): + self.host = None + self.port = port + self._unix_socket = unix_socket + self.handle = None + + def listen(self): + res0 = self._resolveAddr() + for res in res0: + if res[0] is socket.AF_INET6 or res is res0[-1]: + break + + # We need remove the old unix socket if the file exists and + # nobody is listening on it. + if self._unix_socket: + tmp = socket.socket(res[0], res[1]) + try: + tmp.connect(res[4]) + except socket.error, err: + eno, message = err.args + if eno == errno.ECONNREFUSED: + os.unlink(res[4]) + + self.handle = socket.socket(res[0], res[1]) + self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(self.handle, 'set_timeout'): + self.handle.set_timeout(None) + self.handle.bind(res[4]) + self.handle.listen(128) + + def accept(self): + client, addr = self.handle.accept() + result = TSocket() + result.setHandle(client) + return result diff --git a/api/thrift/transport/TTransport.py b/api/thrift/transport/TTransport.py new file mode 100644 index 0000000..12e51a9 --- /dev/null +++ b/api/thrift/transport/TTransport.py @@ -0,0 +1,331 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from cStringIO import StringIO +from struct import pack,unpack +from thrift.Thrift import TException + +class TTransportException(TException): + + """Custom Transport Exception class""" + + UNKNOWN = 0 + NOT_OPEN = 1 + ALREADY_OPEN = 2 + TIMED_OUT = 3 + END_OF_FILE = 4 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + +class TTransportBase: + + """Base class for Thrift transport layer.""" + + def isOpen(self): + pass + + def open(self): + pass + + def close(self): + pass + + def read(self, sz): + pass + + def readAll(self, sz): + buff = '' + have = 0 + while (have < sz): + chunk = self.read(sz-have) + have += len(chunk) + buff += chunk + + if len(chunk) == 0: + raise EOFError() + + return buff + + def write(self, buf): + pass + + def flush(self): + pass + +# This class should be thought of as an interface. +class CReadableTransport: + """base class for transports that are readable from C""" + + # TODO(dreiss): Think about changing this interface to allow us to use + # a (Python, not c) StringIO instead, because it allows + # you to write after reading. + + # NOTE: This is a classic class, so properties will NOT work + # correctly for setting. + @property + def cstringio_buf(self): + """A cStringIO buffer that contains the current chunk we are reading.""" + pass + + def cstringio_refill(self, partialread, reqlen): + """Refills cstringio_buf. + + Returns the currently used buffer (which can but need not be the same as + the old cstringio_buf). partialread is what the C code has read from the + buffer, and should be inserted into the buffer before any more reads. The + return value must be a new, not borrowed reference. Something along the + lines of self._buf should be fine. + + If reqlen bytes can't be read, throw EOFError. + """ + pass + +class TServerTransportBase: + + """Base class for Thrift server transports.""" + + def listen(self): + pass + + def accept(self): + pass + + def close(self): + pass + +class TTransportFactoryBase: + + """Base class for a Transport Factory""" + + def getTransport(self, trans): + return trans + +class TBufferedTransportFactory: + + """Factory transport that builds buffered transports""" + + def getTransport(self, trans): + buffered = TBufferedTransport(trans) + return buffered + + +class TBufferedTransport(TTransportBase,CReadableTransport): + + """Class that wraps another transport and buffers its I/O. + + The implementation uses a (configurable) fixed-size read buffer + but buffers all writes until a flush is performed. + """ + + DEFAULT_BUFFER = 4096 + + def __init__(self, trans, rbuf_size = DEFAULT_BUFFER): + self.__trans = trans + self.__wbuf = StringIO() + self.__rbuf = StringIO("") + self.__rbuf_size = rbuf_size + + def isOpen(self): + return self.__trans.isOpen() + + def open(self): + return self.__trans.open() + + def close(self): + return self.__trans.close() + + def read(self, sz): + ret = self.__rbuf.read(sz) + if len(ret) != 0: + return ret + + self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size))) + return self.__rbuf.read(sz) + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + out = self.__wbuf.getvalue() + # reset wbuf before write/flush to preserve state on underlying failure + self.__wbuf = StringIO() + self.__trans.write(out) + self.__trans.flush() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self.__rbuf + + def cstringio_refill(self, partialread, reqlen): + retstring = partialread + if reqlen < self.__rbuf_size: + # try to make a read of as much as we can. + retstring += self.__trans.read(self.__rbuf_size) + + # but make sure we do read reqlen bytes. + if len(retstring) < reqlen: + retstring += self.__trans.readAll(reqlen - len(retstring)) + + self.__rbuf = StringIO(retstring) + return self.__rbuf + +class TMemoryBuffer(TTransportBase, CReadableTransport): + """Wraps a cStringIO object as a TTransport. + + NOTE: Unlike the C++ version of this class, you cannot write to it + then immediately read from it. If you want to read from a + TMemoryBuffer, you must either pass a string to the constructor. + TODO(dreiss): Make this work like the C++ version. + """ + + def __init__(self, value=None): + """value -- a value to read from for stringio + + If value is set, this will be a transport for reading, + otherwise, it is for writing""" + if value is not None: + self._buffer = StringIO(value) + else: + self._buffer = StringIO() + + def isOpen(self): + return not self._buffer.closed + + def open(self): + pass + + def close(self): + self._buffer.close() + + def read(self, sz): + return self._buffer.read(sz) + + def write(self, buf): + self._buffer.write(buf) + + def flush(self): + pass + + def getvalue(self): + return self._buffer.getvalue() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self._buffer + + def cstringio_refill(self, partialread, reqlen): + # only one shot at reading... + raise EOFError() + +class TFramedTransportFactory: + + """Factory transport that builds framed transports""" + + def getTransport(self, trans): + framed = TFramedTransport(trans) + return framed + + +class TFramedTransport(TTransportBase, CReadableTransport): + + """Class that wraps another transport and frames its I/O when writing.""" + + def __init__(self, trans,): + self.__trans = trans + self.__rbuf = StringIO() + self.__wbuf = StringIO() + + def isOpen(self): + return self.__trans.isOpen() + + def open(self): + return self.__trans.open() + + def close(self): + return self.__trans.close() + + def read(self, sz): + ret = self.__rbuf.read(sz) + if len(ret) != 0: + return ret + + self.readFrame() + return self.__rbuf.read(sz) + + def readFrame(self): + buff = self.__trans.readAll(4) + sz, = unpack('!i', buff) + self.__rbuf = StringIO(self.__trans.readAll(sz)) + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + wout = self.__wbuf.getvalue() + wsz = len(wout) + # reset wbuf before write/flush to preserve state on underlying failure + self.__wbuf = StringIO() + # N.B.: Doing this string concatenation is WAY cheaper than making + # two separate calls to the underlying socket object. Socket writes in + # Python turn out to be REALLY expensive, but it seems to do a pretty + # good job of managing string buffer operations without excessive copies + buf = pack("!i", wsz) + wout + self.__trans.write(buf) + self.__trans.flush() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self.__rbuf + + def cstringio_refill(self, prefix, reqlen): + # self.__rbuf will already be empty here because fastbinary doesn't + # ask for a refill until the previous buffer is empty. Therefore, + # we can start reading new frames immediately. + while len(prefix) < reqlen: + self.readFrame() + prefix += self.__rbuf.getvalue() + self.__rbuf = StringIO(prefix) + return self.__rbuf + + +class TFileObjectTransport(TTransportBase): + """Wraps a file-like object to make it work as a Thrift transport.""" + + def __init__(self, fileobj): + self.fileobj = fileobj + + def isOpen(self): + return True + + def close(self): + self.fileobj.close() + + def read(self, sz): + return self.fileobj.read(sz) + + def write(self, buf): + self.fileobj.write(buf) + + def flush(self): + self.fileobj.flush() diff --git a/api/thrift/transport/TTwisted.py b/api/thrift/transport/TTwisted.py new file mode 100644 index 0000000..b6dcb4e --- /dev/null +++ b/api/thrift/transport/TTwisted.py @@ -0,0 +1,219 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from zope.interface import implements, Interface, Attribute +from twisted.internet.protocol import Protocol, ServerFactory, ClientFactory, \ + connectionDone +from twisted.internet import defer +from twisted.protocols import basic +from twisted.python import log +from twisted.web import server, resource, http + +from thrift.transport import TTransport +from cStringIO import StringIO + + +class TMessageSenderTransport(TTransport.TTransportBase): + + def __init__(self): + self.__wbuf = StringIO() + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + msg = self.__wbuf.getvalue() + self.__wbuf = StringIO() + self.sendMessage(msg) + + def sendMessage(self, message): + raise NotImplementedError + + +class TCallbackTransport(TMessageSenderTransport): + + def __init__(self, func): + TMessageSenderTransport.__init__(self) + self.func = func + + def sendMessage(self, message): + self.func(message) + + +class ThriftClientProtocol(basic.Int32StringReceiver): + + MAX_LENGTH = 2 ** 31 - 1 + + def __init__(self, client_class, iprot_factory, oprot_factory=None): + self._client_class = client_class + self._iprot_factory = iprot_factory + if oprot_factory is None: + self._oprot_factory = iprot_factory + else: + self._oprot_factory = oprot_factory + + self.recv_map = {} + self.started = defer.Deferred() + + def dispatch(self, msg): + self.sendString(msg) + + def connectionMade(self): + tmo = TCallbackTransport(self.dispatch) + self.client = self._client_class(tmo, self._oprot_factory) + self.started.callback(self.client) + + def connectionLost(self, reason=connectionDone): + for k,v in self.client._reqs.iteritems(): + tex = TTransport.TTransportException( + type=TTransport.TTransportException.END_OF_FILE, + message='Connection closed') + v.errback(tex) + + def stringReceived(self, frame): + tr = TTransport.TMemoryBuffer(frame) + iprot = self._iprot_factory.getProtocol(tr) + (fname, mtype, rseqid) = iprot.readMessageBegin() + + try: + method = self.recv_map[fname] + except KeyError: + method = getattr(self.client, 'recv_' + fname) + self.recv_map[fname] = method + + method(iprot, mtype, rseqid) + + +class ThriftServerProtocol(basic.Int32StringReceiver): + + MAX_LENGTH = 2 ** 31 - 1 + + def dispatch(self, msg): + self.sendString(msg) + + def processError(self, error): + self.transport.loseConnection() + + def processOk(self, _, tmo): + msg = tmo.getvalue() + + if len(msg) > 0: + self.dispatch(msg) + + def stringReceived(self, frame): + tmi = TTransport.TMemoryBuffer(frame) + tmo = TTransport.TMemoryBuffer() + + iprot = self.factory.iprot_factory.getProtocol(tmi) + oprot = self.factory.oprot_factory.getProtocol(tmo) + + d = self.factory.processor.process(iprot, oprot) + d.addCallbacks(self.processOk, self.processError, + callbackArgs=(tmo,)) + + +class IThriftServerFactory(Interface): + + processor = Attribute("Thrift processor") + + iprot_factory = Attribute("Input protocol factory") + + oprot_factory = Attribute("Output protocol factory") + + +class IThriftClientFactory(Interface): + + client_class = Attribute("Thrift client class") + + iprot_factory = Attribute("Input protocol factory") + + oprot_factory = Attribute("Output protocol factory") + + +class ThriftServerFactory(ServerFactory): + + implements(IThriftServerFactory) + + protocol = ThriftServerProtocol + + def __init__(self, processor, iprot_factory, oprot_factory=None): + self.processor = processor + self.iprot_factory = iprot_factory + if oprot_factory is None: + self.oprot_factory = iprot_factory + else: + self.oprot_factory = oprot_factory + + +class ThriftClientFactory(ClientFactory): + + implements(IThriftClientFactory) + + protocol = ThriftClientProtocol + + def __init__(self, client_class, iprot_factory, oprot_factory=None): + self.client_class = client_class + self.iprot_factory = iprot_factory + if oprot_factory is None: + self.oprot_factory = iprot_factory + else: + self.oprot_factory = oprot_factory + + def buildProtocol(self, addr): + p = self.protocol(self.client_class, self.iprot_factory, + self.oprot_factory) + p.factory = self + return p + + +class ThriftResource(resource.Resource): + + allowedMethods = ('POST',) + + def __init__(self, processor, inputProtocolFactory, + outputProtocolFactory=None): + resource.Resource.__init__(self) + self.inputProtocolFactory = inputProtocolFactory + if outputProtocolFactory is None: + self.outputProtocolFactory = inputProtocolFactory + else: + self.outputProtocolFactory = outputProtocolFactory + self.processor = processor + + def getChild(self, path, request): + return self + + def _cbProcess(self, _, request, tmo): + msg = tmo.getvalue() + request.setResponseCode(http.OK) + request.setHeader("content-type", "application/x-thrift") + request.write(msg) + request.finish() + + def render_POST(self, request): + request.content.seek(0, 0) + data = request.content.read() + tmi = TTransport.TMemoryBuffer(data) + tmo = TTransport.TMemoryBuffer() + + iprot = self.inputProtocolFactory.getProtocol(tmi) + oprot = self.outputProtocolFactory.getProtocol(tmo) + + d = self.processor.process(iprot, oprot) + d.addCallback(self._cbProcess, request, tmo) + return server.NOT_DONE_YET diff --git a/api/thrift/transport/__init__.py b/api/thrift/transport/__init__.py new file mode 100644 index 0000000..02c6048 --- /dev/null +++ b/api/thrift/transport/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TTransport', 'TSocket', 'THttpClient'] diff --git a/api/urls.py b/api/urls.py new file mode 100644 index 0000000..3760419 --- /dev/null +++ b/api/urls.py @@ -0,0 +1,28 @@ +from django.conf.urls.defaults import * #@UnusedWildImport + +# Uncomment the next two lines to enable the admin: +from django.contrib import admin + +from api import restapi + +admin.autodiscover() + +VERSION = r'^v(?P\d+)' +INDEX = r'/indexes/(?P[^/]+)' +FUNCTION = r'/functions/(?P-?\d+)' + +urlpatterns = patterns('', + url('^/?$', 'api.restapi.default', name='default'), + url(VERSION + '/?$', restapi.Version.as_view()), + url(VERSION + '/indexes/?$', restapi.Indexes.as_view()), + url(VERSION + INDEX + '/?$', restapi.Index.as_view()), + url(VERSION + INDEX + '/docs/?$', restapi.Document.as_view()), + url(VERSION + INDEX + '/docs/variables/?$', restapi.Variables.as_view()), + url(VERSION + INDEX + '/docs/categories/?$', restapi.Categories.as_view()), + url(VERSION + INDEX + '/functions/?$', restapi.Functions.as_view()), + url(VERSION + INDEX + FUNCTION + '/?$', restapi.Function.as_view()), + url(VERSION + INDEX + '/search/?$', restapi.Search.as_view()), + url(VERSION + INDEX + '/promote/?$', restapi.Promote.as_view()), + url(VERSION + INDEX + '/autocomplete/?$', restapi.AutoComplete.as_view()), + url(VERSION + INDEX + '/instantlinks/?$', restapi.InstantLinks.as_view()), +) diff --git a/api/wsgi.py b/api/wsgi.py new file mode 100644 index 0000000..6fb3570 --- /dev/null +++ b/api/wsgi.py @@ -0,0 +1,8 @@ +import os +import sys + +sys.path.append(os.path.abspath(os.path.dirname(__file__))) +os.environ['DJANGO_SETTINGS_MODULE'] = 'api.settings' + +import django.core.handlers.wsgi +application = django.core.handlers.wsgi.WSGIHandler() diff --git a/backoffice/.project b/backoffice/.project new file mode 100644 index 0000000..a0d8ab7 --- /dev/null +++ b/backoffice/.project @@ -0,0 +1,18 @@ + + + backoffice + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + org.python.pydev.django.djangoNature + + diff --git a/backoffice/.pydevproject b/backoffice/.pydevproject new file mode 100644 index 0000000..1a746e5 --- /dev/null +++ b/backoffice/.pydevproject @@ -0,0 +1,14 @@ + + + + +Default +python 2.6 + +DJANGO_MANAGE_LOCATION +backoffice/manage.py + + +/backoffice + + diff --git a/backoffice/__init__.py b/backoffice/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backoffice/amazon_credential.py b/backoffice/amazon_credential.py new file mode 100644 index 0000000..c2910f5 --- /dev/null +++ b/backoffice/amazon_credential.py @@ -0,0 +1,2 @@ +AMAZON_USER = "" +AMAZON_PASSWORD = "" diff --git a/backoffice/api_linked_models.py b/backoffice/api_linked_models.py new file mode 100644 index 0000000..0908f60 --- /dev/null +++ b/backoffice/api_linked_models.py @@ -0,0 +1,938 @@ +import hashlib +import random +import binascii + +from lib.indextank.client import ApiClient, IndexAlreadyExists +from lib.authorizenet import AuthorizeNet, BillingException + +from django.db import models +from django.contrib.auth.models import User +from django.utils import simplejson as json +from django.db import IntegrityError +from django.db.models.aggregates import Sum, Count + +from lib import encoder, flaptor_logging + +from django.conf import settings +from datetime import datetime + +logger = flaptor_logging.get_logger('Models') + +# idea taken from https://www.grc.com/passwords.htm +def generate_apikey(id): + key = "2A1A8AE7CAEFAC47D6F74920CE4B0CE46430CDA6CF03D254C1C29402D727E570" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:14] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return unique_part + '-' + random_part + +def generate_onetimepass(id): + key = "CAEFAC47D6F7D727E57024920CE4B0CE46430CDA6CF03D254C1C29402A1A8AE7" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:5] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return unique_part + random_part + +def generate_forgotpass(id): + key = "E57024920CE4B0CE4643CAEFAC47D6F7D7270CDA6CF03D254C1C29402A1A8AE7" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:6] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return random_part + unique_part + + +# StoreFront models +class Account(models.Model): + apikey = models.CharField(max_length=22, unique=True) + creation_time = models.DateTimeField() + package = models.ForeignKey('Package', null=True) + status = models.CharField(max_length=30, null=False) + provisioner = models.ForeignKey('Provisioner', null=True) + + configuration = models.ForeignKey('IndexConfiguration', null=True) + default_analyzer = models.ForeignKey('Analyzer', null=True, related_name="accounts") + + class Statuses: + operational = 'OPERATIONAL' + creating = 'CREATING' + closed = 'CLOSED' + + def __repr__(self): + return 'Account (%s):\n\tuser_email: %s\n\tapikey: %s\n\tcreation_time: %s\n\tstatus: %s\n\tpackage: %s\n\tconfiguration: %s\n' % (self.id, PFUser.objects.filter(account=self)[0].email, str(self.apikey), str(self.creation_time), str(self.status), self.package.name, self.configuration.description) + + def __str__(self): + return '(apikey: %s; creation_time: %s; status: %s)' % (str(self.apikey), str(self.creation_time), str(self.status)) + + def count_indexes(self): + return self.indexes.aggregate(cnt=Count('id'))['cnt'] + + def count_documents(self): + return self.indexes.aggregate(cnt=Sum('current_docs_number'))['cnt'] + + def is_operational(self): + return self.status == Account.Statuses.operational + + def is_heroku(self): + # HACK UNTIL HEROKU IS A PROVISIONER + return self.package.code.startswith('HEROKU_') + #return self.provisioner and self.provisioner.name == 'heroku' + + @classmethod + def create_account(cls, dt, email=None, password=None): + account = Account() + + account.creation_time = datetime.now() + account.status = Account.Statuses.creating + account.save() + + account.apikey = generate_apikey(account.id) + account.save() + + unique_part, random_part = account.apikey.split('-', 1) + if email is None: + email = '%s@indextank.com' % unique_part + + if password is None: + password = random_part + + try: + user = User.objects.create_user(email, '', password) + except IntegrityError, e: + account.delete() + raise e + + try: + pfu = PFUser() + pfu.account = account + pfu.user = user + pfu.email = email + + pfu.save() + except IntegrityError, e: + account.delete() + user.delete() + raise e + + return account, pfu + + def create_index(self, index_name, public_search=None): + index = Index() + + # basic index data + index.populate_for_account(self) + index.name = index_name + index.creation_time = datetime.now() + index.language_code = 'en' + index.status = Index.States.new + if not public_search is None: + index.public_api = public_search + + # test for name uniqueness + # raises IntegrityError if the index name already exists + index.save() + + # define the default function + function = ScoreFunction() + function.index = index + function.name = '0' + function.definition = '-age' + function.save() + + # deduce code from id + index.code = encoder.to_key(index.id) + index.save() + + return index + + def create_demo_index(self): + try: + dataset = DataSet.objects.get(code='DEMO') + except DataSet.DoesNotExist: + logger.exception('DemoIndex dataset not present in database. Aborting demo index creation') + return + + index = self.create_index('DemoIndex') + + index.public_api = True + index.save() + + population = IndexPopulation() + population.index = index + population.status = IndexPopulation.Statuses.created + population.dataset = dataset + population.time = datetime.now() + population.populated_size = 0 + + population.save() + + def close(self): + # Dropping an account implies: + + # - removing the payment information from the account + # - removing the subscriptions from authorize.net + for info in self.payment_informations.all(): + auth = AuthorizeNet() + for subscription in info.subscriptions.all(): + auth.subscription_cancel(subscription.reference_id, subscription.subscription_id) + subscription.delete() + info.delete() + + + # - changing the status to CLOSED + self.status = Account.Statuses.closed + + # - removing and stopping the indexes for the account + for index in self.indexes.all(): + self.drop_index(index) + + # - notify + # send_notification(//close account) + + # - FIXME: handle authorize net errors! + + + self.save() + + def drop_index(self, index): + client = ApiClient(self.get_private_apiurl()) + client.delete_index(index.name) + + def apply_package(self, package): + self.package = package + + self.configuration = package.configuration + + def update_apikey(self): + self.apikey = generate_apikey(self.id) + + def get_private_apikey(self): + return self.apikey.split('-', 1)[1] + + def get_public_apikey(self): + return self.apikey.split('-', 1)[0] + + def get_private_apiurl(self): + return 'http://:%s@%s.api.indextank.com' % (self.get_private_apikey(), self.get_public_apikey()) + + def get_public_apiurl(self): + return 'http://%s.api.indextank.com' % self.get_public_apikey() + + class Meta: + db_table = 'storefront_account' + +class AccountPayingInformation(models.Model): + account = models.ForeignKey('Account', related_name='payment_informations') + + first_name = models.CharField(max_length=50, null=True) + last_name = models.CharField(max_length=50, null=True) + address = models.CharField(max_length=60, null=True) + city = models.CharField(max_length=60, null=True) + state = models.CharField(max_length=2, null=True) + zip_code = models.CharField(max_length=60, null=True) + country = models.CharField(max_length=2, null=True) + + company = models.CharField(max_length=50, null=True) + + credit_card_last_digits = models.CharField(max_length=4, null=True) + contact_email = models.EmailField(max_length=255, null=True) + + #custom subscription + monthly_amount = models.DecimalField(max_digits=8, decimal_places=2, null=True) + subscription_status = models.CharField(max_length=30, null=True) + subscription_type = models.CharField(max_length=30, null=True) + + + class Meta: + db_table = 'storefront_accountpayinginformation' + + +class PaymentSubscription(models.Model): + account = models.ForeignKey('AccountPayingInformation', related_name='subscriptions') + + # authorizenet id + subscription_id = models.CharField(max_length=20, null=False, blank=False) + # indextank id + reference_id = models.CharField(max_length=13, null=False, blank=False) + + amount = models.DecimalField(max_digits=8, decimal_places=2, null=False) + + # Frequency + start_date = models.DateTimeField() + frequency_length = models.IntegerField(null=False) + frequency_unit = models.CharField(max_length=10, null=False, blank=False) + + class Meta: + db_table = 'storefront_paymentsubscription' + + +class EffectivePayment(models.Model): + account = models.ForeignKey('AccountPayingInformation', related_name='payments') + + transaction_date = models.DateTimeField() + + # authorizenet data + transaction_id = models.CharField(max_length=12, null=False, blank=False) + customer_id = models.CharField(max_length=8, null=False, blank=False) + transaction_message = models.CharField(max_length=300, null=True) + subscription_id = models.CharField(max_length=20, null=False, blank=False) + subscription_payment_number = models.IntegerField(null=False) + first_name = models.CharField(max_length=50, null=False, blank=False) + last_name = models.CharField(max_length=50, null=False, blank=False) + address = models.CharField(max_length=60, null=True) + city = models.CharField(max_length=60, null=True) + state = models.CharField(max_length=2, null=True) + zip_code = models.CharField(max_length=60, null=True) + country = models.CharField(max_length=2, null=True) + company = models.CharField(max_length=50, null=True) + + # Inherited data (from account information + credit_card_last_digits = models.CharField(max_length=4, null=False, blank=False) + contact_email = models.EmailField(max_length=255) + + amount = models.DecimalField(max_digits=8, decimal_places=2, null=False) + + class Meta: + db_table = 'storefront_effectivepayment' + +class DataSet(models.Model): + name = models.CharField(null=True, max_length=50, unique=True) + code = models.CharField(null=True, max_length=15, unique=True) + filename = models.CharField(null=True, max_length=100, unique=True) + size = models.IntegerField(default=0) + + class Meta: + db_table = 'storefront_dataset' + +class IndexPopulation(models.Model): + index = models.ForeignKey('Index', related_name='datasets') + dataset = models.ForeignKey('DataSet', related_name='indexes') + time = models.DateTimeField() + populated_size = models.IntegerField(default=0) + + status = models.CharField(max_length=50,null=True) + + class Statuses: + created = 'CREATED' + populating = 'POPULATING' + finished = 'FINISHED' + + class Meta: + db_table = 'storefront_indexpopulation' + + +class Index(models.Model): + account = models.ForeignKey('Account', related_name='indexes') + code = models.CharField(null=True, max_length=22, unique=True) + name = models.CharField(max_length=50) + language_code = models.CharField(max_length=2) + creation_time = models.DateTimeField() + + analyzer_config = models.TextField(null=True) + configuration = models.ForeignKey('IndexConfiguration', null=True) + public_api = models.BooleanField(default=False, null=False) + + status = models.CharField(max_length=50) + + deleted = models.BooleanField(default=False, null=False) + + class States: + new = 'NEW' + live = 'LIVE' + hibernate_requested = 'HIBERNATE_REQUESTED' + hibernated = 'HIBERNATED' + waking_up = 'WAKING_UP' + + def get_json_for_analyzer(self): + if self.analyzer_config is None: + return None + configuration = json.loads(self.analyzer_config) + final_configuration = {} + + if configuration.has_key('per_field'): + per_field_final = {} + per_field = configuration.get('per_field') + for field in per_field.keys(): + per_field_final[field] = Index.get_analyzer(per_field[field]) + final_configuration['perField'] = per_field_final + final_configuration['default'] = Index.get_analyzer(per_field.get('default')) + else: + final_configuration = Index.get_analyzer(configuration) + + return final_configuration + + @classmethod + def get_analyzer(cls, configuration): + analyzer_map = {} + code = configuration.get('code') + if code is None: + raise ValueError('Analyzer configuration has no "code" key') + + try: + analyzer = AnalyzerComponent.objects.get(code=code) + except AnalyzerComponent.DoesNotExist: + raise ValueError('Analyzer configuration "code" key doesn\'t match any analyzers') + + analyzer_map['factory'] = analyzer.factory + analyzer_map['configuration'] = json.loads(analyzer.config) + + if configuration.has_key('filters'): + filters_list = [] + for filter in configuration.get('filters'): + filters_list.append(Index.get_analyzer(filter)) + analyzer_map['configuration']['filters'] = filters_list + + return analyzer_map + +# allows_adds = models.BooleanField(null=False,default=True) +# allows_queries = models.BooleanField(null=False,default=True) + + # index creation data +# allows_snippets = models.BooleanField() +# +# allows_autocomplete = models.BooleanField(default=True) +# autocomplete_type = models.models.CharField(max_length=10, null=True) # NEW +# +# allows_faceting = models.BooleanField() +# facets_bits = models.IntegerField(null=True) # NEW +# +# max_variables = models.IntegerField(null=False) # NEW +# +# max_memory_mb = models.IntegerField(null=False) # NEW +# rti_documents_number = models.IntegerField(null=False) # NEW + + # statistics + current_size = models.FloatField(default=0) + current_docs_number = models.IntegerField(default=0) + queries_per_day = models.FloatField(default=0) + + #demo + base_port = models.IntegerField(null=True) + + def __repr__(self): + return 'Index (%s):\n\tname: %s\n\tcode: %s\n\tcreation_time: %s\n\tconfiguration: %s\n\taccount\'s package: %s\ncurrent deploys: %r' % (self.id, self.name, self.code, self.creation_time, self.configuration.description, self.account.package.name, self.deploys.all()) + + def is_populating(self): + for population in self.datasets.all(): + if not population.status == IndexPopulation.Statuses.finished: + return True + return False + + def is_demo(self): + return self.name == 'DemoIndex' and self.datasets.count() > 0 + + + def is_ready(self): + ''' + Returns True if the end-user can use the index. + (this means for read and write, and it's meant to + be shown in the storefront page). Internally, this + means that at least one deployment for this index + is readable, and at least one is writable. + ''' + return self.is_writable() and self.is_readable() + + def is_hibernated(self): + return self.status in (Index.States.hibernated, Index.States.waking_up) + + def is_writable(self): + ''' + Returns true if there's at least one index that can be written. + ''' + for deploy in self.deploys.all(): + if deploy.is_writable(): + return True + + def is_readable(self): + ''' + Returns true if there's at least one index that can be read. + ''' + for deploy in self.deploys.all(): + if deploy.is_readable(): + return True + + def populate_for_account(self, account): + self.account = account + self.configuration = account.configuration + if account.default_analyzer is not None: + self.analyzer_config = account.default_analyzer.configuration + + def searchable_deploy(self): + '''Returns a single deploy that can be used to search. If no deploy is searcheable + it returns None. Note that if more than one deploy is searcheable, there are no warranties + of wich one will be returned.''' + # TODO : should change once deploy roles are implemented + ds = self.deploys.all() + ds = [d for d in ds if d.is_readable()] + return ds[0] if ds else None + + def indexable_deploys(self): + '''Returns the list of all deploys that should be updated (adds/updates/deletes/etc)''' + # TODO : should change once deploy roles are implemented + ds = self.deploys.all() + ds = [d for d in ds if d.is_writable()] + return ds + + def get_functions_dict(self): + return dict((str(f.name), f.definition) for f in self.scorefunctions.all()) + + def get_debug_info(self): + info = 'Index: %s [%s]\n' % (self.name, self.code) +\ + 'Account: %s\n' % self.account.user.email +\ + 'Deploys:\n' + for d in self.deploys.all(): + info += ' [deploy:%d] %s on [worker:%s] %s:%s' % (d.id, d.status, d.worker.id, d.worker.wan_dns, d.base_port) + return info + + def update_status(self, new_status): + print 'updating status %s for %r' % (new_status, self) + logger.debug('Updating status to %s for idnex %r', new_status, self) + Index.objects.filter(id=self.id).update(status=new_status) + + def mark_deleted(self): + Index.objects.filter(id=self.id).update(deleted=True) + + class AutocompleTypes: + created = 'DOCUMENTS' + initializing = 'QUERIES' + + class Meta: + unique_together = (('code','account'),('name','account')) + db_table = 'storefront_index' + +class Insight(models.Model): + index = models.ForeignKey(Index, related_name='insights') + code = models.CharField(max_length=30, null=False) + data = models.TextField(null=False) + last_update = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = ('index', 'code') + db_table = 'storefront_insight' + +class IndexConfiguration(models.Model): + description = models.TextField(null=False) + creation_date = models.DateField() + json_configuration = models.TextField(null=False) + + def __repr__(self): + j_map = json.loads(self.json_configuration) + mapStr = '{\n' + for m in j_map: + mapStr += '\t\t%s -> %s\n' % (m, j_map[m]) + mapStr += '\t}\n' + return 'IndexConfiguration (%s):\n\tdescription: %s\n\tcreation_date: %s\n\tjson_configuration: %s\n' % (self.id, self.description, str(self.creation_date), mapStr) + + def __str__(self): + return '(description: %s; creation_date: %s; json_configuration: %s)' % (self.description, str(self.creation_date), self.json_configuration) + + def get_data(self): + map = json.loads(self.json_configuration) + data = {} + for k,v in map.items(): + data[str(k)] = v + data['ram'] = data.get('xmx',0) + data.get('bdb_cache',0) + return data + def set_data(self, data): + self.json_configuration = json.dumps(data) + + class Meta: + db_table = 'storefront_indexconfiguration' + +class Analyzer(models.Model): + account = models.ForeignKey('Account', related_name='analyzers') + code = models.CharField(max_length=64) + configuration = models.TextField() + + class Meta: + db_table = 'storefront_analyzer' + +class AnalyzerComponent(models.Model): + code = models.CharField(max_length=15, unique=True) + name = models.CharField(max_length=200) + description = models.CharField(max_length=1000) + config = models.TextField(null=False,blank=False) + factory = models.CharField(max_length=200) + type = models.CharField(max_length=20) + enabled = models.BooleanField() + + class Types: + tokenizer = 'TOKENIZER' + filter = 'FILTER' + + class Meta: + db_table = 'storefront_analyzercomponent' + +def create_analyzer(code, name, config, factory, type, enabled): + analyzer = None + try: + analyzer = AnalyzerComponent.objects.get(code=code) + + analyzer.name = name + analyzer.config = config + analyzer.factory = factory + analyzer.type = type + analyzer.enabled = enabled + + analyzer.save() + except AnalyzerComponent.DoesNotExist: + analyzer = AnalyzerComponent(code=code, name=name, config=config, type=type, enabled=enabled) + analyzer.save() + +class Package(models.Model): + ''' + Packages define what a user have the right to when creating an Account and how does the indexes in that Account + behave. + There are two sections for what the Package configures. A fixed section with the control and limits information + that is used by nebu, storefront or api (base_price, index_max_size, searches_per_day, max_indexes). A dynamic + section that is handled by the IndexConfiguration object. The information of that section is passed to the IndexEngine + as it is and handled by it. + ''' + name = models.CharField(max_length=50) + code = models.CharField(max_length=30) + base_price = models.FloatField() + index_max_size = models.IntegerField() + searches_per_day = models.IntegerField() + max_indexes = models.IntegerField() + + configuration = models.ForeignKey('IndexConfiguration', null=True) + + def __repr__(self): + return 'Package (%s):\n\tname: %s\n\tcode: %s\n\tbase_price: %.2f\n\tindex_max_size: %i\n\tsearches_per_day: %i\n\tmax_indexes: %i\n' % (self.id, self.name, self.code, self.base_price, self.index_max_size, self.searches_per_day, self.max_indexes) + + def __str__(self): + return '(name: %s; code: %s; base_price: %.2f; index_max_size: %i; searches_per_day: %i; max_indexes: %i)' % (self.name, self.code, self.base_price, self.index_max_size, self.searches_per_day, self.max_indexes) + + def max_size_mb(self): + return self.index_max_size * settings.INDEX_SIZE_RATIO + class Meta: + db_table = 'storefront_package' + +class ScoreFunction(models.Model): + index = models.ForeignKey(Index, related_name='scorefunctions') + name = models.IntegerField(null=False) # TODO the java API expects an int. But a String may be nicer for name. + definition = models.CharField(max_length=255, blank=False, null=True) + + class Meta: + db_table = 'storefront_scorefunction' + unique_together = (('index','name')) + + +def create_configuration(description, data, creation_date=None): + configuration = IndexConfiguration() + configuration.description = description + configuration.creation_date = creation_date or datetime.now() + configuration.json_configuration = json.dumps(data) + + configuration.save() + return configuration + +def create_package(code, name, base_price, index_max_size, searches_per_day, max_indexes, configuration_map): +# The configuration_map will only be considered if the package if new or if it didn't already have a configuration + + package = None + try: + package = Package.objects.get(code=code) + + package.name = name + package.base_price = base_price + package.index_max_size = index_max_size + package.searches_per_day = searches_per_day + package.max_indexes = max_indexes + + if not package.configuration: + package.configuration = create_configuration('package:' + code, configuration_map) + + package.save() + except Package.DoesNotExist: + configuration = create_configuration('package:' + code, configuration_map) + package = Package(code=code, base_price=base_price, index_max_size=index_max_size, searches_per_day=searches_per_day, max_indexes=max_indexes, configuration=configuration) + package.save() + +def create_provisioner(name, token, email, plans): + provisioner = None + try: + provisioner = Provisioner.objects.get(name=name) + except Provisioner.DoesNotExist: + provisioner = Provisioner() + provisioner.name = name + provisioner.token = token + provisioner.email = email + provisioner.save() + + provisioner.plans.all().delete() + for plan, code in plans.items(): + pp = ProvisionerPlan() + pp.plan = plan + pp.provisioner = provisioner + pp.package = Package.objects.get(code=code) + pp.save() + + +class AccountMovement(models.Model): + account = models.ForeignKey('Account', related_name='movements') + class Meta: + db_table = 'storefront_accountmovement' + +class ActionLog(models.Model): + account = models.ForeignKey('Account', related_name='actions') + class Meta: + db_table = 'storefront_actionlog' + +class PFUser(models.Model): + user = models.ForeignKey(User, unique=True) + account = models.OneToOneField('Account', related_name='user') + email = models.EmailField(unique=True, max_length=255) + change_password = models.BooleanField(default=False, null=False) + class Meta: + db_table = 'storefront_pfuser' + + + +MAX_USABLE_RAM_PERCENTAGE = 0.9 +# Nebulyzer stuff +class Worker(models.Model): + ''' + Describes an amazon ec2 instance. + ''' + instance_name = models.CharField(max_length=50,null=False,blank=False) + lan_dns = models.CharField(max_length=100,null=False,blank=False) + wan_dns = models.CharField(max_length=100,null=False,blank=False) + status = models.CharField(max_length=30) + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) + #physical memory in MegaBytes + ram = models.IntegerField() + + class States: + created = 'CREATED' + initializing = 'INITIALIZING' + updating = 'UPDATING' + controllable = 'CONTROLLABLE' + decommissioning = 'DECOMMISSIONING' + dying = 'DYING' + dead = 'DEAD' + + class Meta: + db_table = 'storefront_worker' + + def get_usable_ram(self): + '''Return the amount of ram that can be used in this machine for + indexengines. It's calculated as a fixed percentage of the physical + ram. Value returned in MegaBytes''' + return MAX_USABLE_RAM_PERCENTAGE * self.ram + + def get_used_ram(self): + xmx = self.deploys.aggregate(xmx=Sum('effective_xmx'))['xmx'] + bdb = self.deploys.aggregate(bdb=Sum('effective_bdb'))['bdb'] + if xmx == None: + xmx = 0 + if bdb == None: + bdb = 0 + return xmx + bdb + + def is_assignable(self): + return self.status != Worker.States.decommissioning + + def is_ready(self): + return self.status in [Worker.States.controllable, Worker.States.decommissioning] + + def __repr__(self): + return 'Worker (%s):\n\tinstance_name: %s\n\tlan_dns: %s\n\twan_dns: %s\n\tstatus: %s\n\ttimestamp: %s\n\tram: %s\n' %(self.pk, self.instance_name, self.lan_dns, self.wan_dns, self.status, self.timestamp, self.ram) + +class Service(models.Model): + name = models.CharField(max_length=50,null=False,blank=False) + type = models.CharField(max_length=50,null=True,blank=True ) + host = models.CharField(max_length=100,null=False,blank=False) + port = models.IntegerField() + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) + + class Meta: + db_table = 'storefront_service' + + def __repr__(self): + return 'Service (%s):\n\tname: %s\n\ttype: %s\n\thost: %s\n\tport: %s\n\ttimestamp: %s\n' % (self.pk, self.name, self.type, self.host, self.port, self.timestamp) + + +# CPU Stats +class WorkerMountInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="disk_infos") + timestamp = models.DateTimeField() + + mount = models.CharField(max_length=100,null=False,blank=False) + available = models.IntegerField() + used = models.IntegerField() + + class Meta: + db_table = 'storefront_workermountinfo' + + +class WorkerLoadInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="load_infos") + timestamp = models.DateTimeField() + + load_average = models.FloatField() + + class Meta: + db_table = 'storefront_workerloadinfo' + +class WorkerIndexInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="indexes_infos") + timestamp = models.DateTimeField() + + deploy = models.ForeignKey('Deploy', related_name="index_infos") + used_disk = models.IntegerField() + used_mem = models.IntegerField() + + class Meta: + db_table = 'storefront_workerindexinfo' + + +class Deploy(models.Model): + ''' + Describes a deploy of an index on a worker, and it's status. + The idea is that an index can be moving from one worker to another, + so queries and indexing requests have to be mapped to one or more + index engines. + ''' + index = models.ForeignKey(Index, related_name="deploys") + worker = models.ForeignKey(Worker, related_name="deploys") + base_port = models.IntegerField() + status = models.CharField(max_length=30) + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) # Last time we updated this deploy. + parent = models.ForeignKey('self', related_name='children', null=True) # For moving deploys. + effective_xmx = models.IntegerField() + effective_bdb = models.IntegerField() + dying = models.BooleanField(default=False, null=False) + + # TODO add role fields + #searching_role = models.BooleanField() + #indexing_role = models.BooleanField() + + def __repr__(self): + return 'Deploy (%s):\n\tparent deploy: %s\n\tindex code: %s\n\tstatus: %s\n\tworker ip: %s\n\tport: %d\n\teffective_xmx: %d\n\teffective_bdb: %d\n' % (self.id, self.parent_id, self.index.code, self.status, self.worker.lan_dns, self.base_port, self.effective_xmx, self.effective_bdb) + + def __unicode__(self): + return "Deploy: %s on %s:%d" % (self.status, self.worker.lan_dns, self.base_port) + + def is_readable(self): + '''Returns true if a search can be performed on this deployment, and + the returned data is up to date''' + return self.status == Deploy.States.controllable or \ + self.status == Deploy.States.move_requested or \ + self.status == Deploy.States.moving or \ + (self.status == Deploy.States.recovering and not self.parent) + + def is_writable(self): + '''Returns True if new data has to be written to this deployment.''' + return self.status == Deploy.States.controllable or \ + self.status == Deploy.States.recovering or \ + self.status == Deploy.States.move_requested or \ + self.status == Deploy.States.moving + + def total_ram(self): + return self.effective_xmx + self.effective_bdb + + def update_status(self, new_status): + print 'updating status %s for %r' % (new_status, self) + logger.debug('Updating status to %s for deploy %r', new_status, self) + Deploy.objects.filter(id=self.id).update(status=new_status, timestamp=datetime.now()) + + def update_parent(self, new_parent): + logger.debug('Updating parent to %s for deploy %r', new_parent, self) + Deploy.objects.filter(id=self.id).update(parent=new_parent) + + class States: + created = 'CREATED' + initializing = 'INITIALIZING' + recovering = 'RECOVERING' + resurrecting = 'RESURRECTING' + controllable = 'CONTROLLABLE' + move_requested = 'MOVE_REQUESTED' + moving = 'MOVING' + decommissioning = 'DECOMMISSIONING' + + class Meta: + db_table = 'storefront_deploy' + +class BetaTestRequest(models.Model): + email = models.EmailField(unique=True, max_length=255) + site_url = models.CharField(max_length=200,null=False,blank=False) + summary = models.TextField(null=False,blank=False) + + request_date = models.DateTimeField(default=datetime.now) + status = models.CharField(max_length=50,null=True) + + class Meta: + db_table = 'storefront_betatestrequest' + +class BetaInvitation(models.Model): + password = models.CharField(max_length=20, null=True) + account = models.ForeignKey('Account', null=True) + assigned_customer = models.CharField(max_length=50, null=True) + beta_requester = models.ForeignKey('BetaTestRequest', null=True, related_name="invitation") + + invitation_date = models.DateTimeField(default=datetime.now) + forced_package = models.ForeignKey('Package', null=False) + + class Meta: + db_table = 'storefront_signupotp' + +class ContactInfo(models.Model): + name = models.CharField(max_length=64) + email = models.EmailField(unique=True, max_length=255) + request_date = models.DateTimeField(default=datetime.now) + source = models.CharField(max_length=64, null=True) + + class Meta: + db_table = 'storefront_contactinfo' + + + +class Provisioner(models.Model): + name = models.CharField(max_length=64) + token = models.CharField(max_length=64, null=False, blank=False) + email = models.EmailField(max_length=255) # contact info for the provisioner + + class Meta: + db_table = "storefront_provisioner" + +class ProvisionerPlan(models.Model): + plan = models.CharField(max_length=50) + provisioner = models.ForeignKey('Provisioner', related_name='plans') + package = models.ForeignKey('Package') + + class Meta: + db_table = "storefront_provisionerplan" + +class BlogPostInfo(models.Model): + title = models.CharField(max_length=200) + url = models.CharField(max_length=1024) + date = models.DateTimeField() + author = models.CharField(max_length=64) + + class Meta: + db_table = 'storefront_blogpost' + diff --git a/backoffice/api_linked_rpc.py b/backoffice/api_linked_rpc.py new file mode 100644 index 0000000..2fd7649 --- /dev/null +++ b/backoffice/api_linked_rpc.py @@ -0,0 +1,169 @@ +from flaptor.indextank.rpc import Indexer, Searcher, Suggestor, Storage, LogWriter, WorkerManager,\ + DeployManager, Controller, FrontendManager + +from flaptor.indextank.rpc.ttypes import NebuException, IndextankException + +''' =========================== + THRIFT STUFF + =========================== ''' +from thrift.transport import TSocket, TTransport +from thrift.protocol import TBinaryProtocol +from lib import flaptor_logging, exceptions +from thrift.transport.TTransport import TTransportException +from socket import socket +from socket import error as SocketError + + +logger = flaptor_logging.get_logger('RPC') + +# Missing a way to close transport +def getThriftControllerClient(host, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host,19010, timeout_ms) + client = Controller.Client(protocol) + transport.open() + return client + +# Missing a way to close transport +def getThriftIndexerClient(host, base_port, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host, base_port + 1, timeout_ms) + client = Indexer.Client(protocol) + transport.open() + return client + +def getThriftSearcherClient(host, base_port, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host, base_port + 2, timeout_ms) + client = Searcher.Client(protocol) + transport.open() + return client + +def getThriftSuggestorClient(host, base_port): + protocol, transport = __getThriftProtocolTransport(host, base_port + 3) + client = Suggestor.Client(protocol) + transport.open() + return client + +storage_port = 10000 +def getThriftStorageClient(): + protocol, transport = __getThriftProtocolTransport('storage',storage_port) + client = Storage.Client(protocol) + transport.open() + return client + +def getThriftLogWriterClient(host, port, timeout_ms=500): + protocol, transport = __getThriftProtocolTransport(host,port,timeout_ms) + client = LogWriter.Client(protocol) + transport.open() + return client + +def getThriftLogReaderClient(host, port, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host,port,timeout_ms) + client = LogWriter.Client(protocol) + transport.open() + return client + +class ReconnectingClient: + def __init__(self, factory): + self.factory = factory + self.delegate = None #factory() + + def __getattr__(self, name): + import types + if self.delegate is None: + self.delegate = self.factory() + att = getattr(self.delegate, name) + if type(att) is types.MethodType: + def wrap(*args, **kwargs): + try: + return att(*args, **kwargs) + except (NebuException, IndextankException): + logger.warn('raising catcheable exception') + raise + except (TTransportException, IOError, SocketError): + logger.warn('failed to run %s, reconnecting once', name) + self.delegate = self.factory() + att2 = getattr(self.delegate, name) + return att2(*args, **kwargs) + except Exception: + logger.exception('Unexpected failure to run %s, reconnecting once', name) + self.delegate = self.factory() + att2 = getattr(self.delegate, name) + return att2(*args, **kwargs) + + return wrap + else: + return att + +def getReconnectingStorageClient(): + return ReconnectingClient(getThriftStorageClient) + +def getReconnectingLogWriterClient(host, port): + return ReconnectingClient(lambda: getThriftLogWriterClient(host, port)) + +worker_manager_port = 8799 +def getThriftWorkerManagerClient(host): + protocol, transport = __getThriftProtocolTransport(host,worker_manager_port) + client = WorkerManager.Client(protocol) + transport.open() + return client + +deploymanager_port = 8899 +def get_deploy_manager(): + protocol, transport = __getThriftProtocolTransport('deploymanager',deploymanager_port) + client = DeployManager.Client(protocol) + transport.open() + return client + + +def __getThriftProtocolTransport(host, port=0, timeout_ms=None): + ''' returns protocol,transport''' + # Make socket + transport = TSocket.TSocket(host, port) + + if timeout_ms is not None: + transport.setTimeout(timeout_ms) + + # Buffering is critical. Raw sockets are very slow + transport = TTransport.TBufferedTransport(transport) + + # Wrap in a protocol + protocol = TBinaryProtocol.TBinaryProtocol(transport) + return protocol, transport + + +def get_searcher_client(index, timeout_ms=None): + ''' + This method returns a single searcherclient, or None + ''' + deploy = index.searchable_deploy() + if deploy: + return getThriftSearcherClient(deploy.worker.lan_dns, int(deploy.base_port), timeout_ms) + else: + return None + +def get_worker_controller(worker, timeout_ms=None): + return getThriftControllerClient(worker. lan_dns) + +def get_suggestor_client(index): + ''' + This method returns a single suggestorclient, or None + ''' + deploy = index.searchable_deploy() + if deploy: + return getThriftSuggestorClient(deploy.worker.lan_dns, int(deploy.base_port)) + else: + return None + +def get_indexer_clients(index, timeout_ms=1000): + ''' + This method returns the list of all indexerclients that should be updated + on add,delete,update, and category updates. + @raise exceptions.NoIndexerException if this index has no writable deploy. + ''' + deploys = index.indexable_deploys() + retval = [] + for d in deploys: + retval.append(getThriftIndexerClient(d.worker.lan_dns, int(d.base_port), timeout_ms)) + if retval: + return retval + else: + raise exceptions.NoIndexerException() diff --git a/backoffice/cloud/amazon_credential.py b/backoffice/cloud/amazon_credential.py new file mode 100644 index 0000000..c2910f5 --- /dev/null +++ b/backoffice/cloud/amazon_credential.py @@ -0,0 +1,2 @@ +AMAZON_USER = "" +AMAZON_PASSWORD = "" diff --git a/backoffice/cloud/config_dbslave.py b/backoffice/cloud/config_dbslave.py new file mode 100644 index 0000000..fc527a3 --- /dev/null +++ b/backoffice/cloud/config_dbslave.py @@ -0,0 +1,201 @@ +import sys, time +import subprocess, shlex +import MySQLdb + +from optparse import OptionParser + +def config_dbslave(master_host, slave_host, user, password): + + print 'Configuring DB slave' + print 'MASTER:%s' % master_host + print 'SLAVE:%s' % slave_host + + # ASSIGN SERVER-ID AND CONFIGURE MY.CNF FILE + slave_id = int(time.time()) + print 'SLAVE: Configuring mysql slave with id: %s' % slave_id + + set_up_mysql_config = ( + "sudo service mysql stop;" + + "sudo sed -i 's/server-id = 0/server-id = %s/' /etc/mysql/my.cnf;" % slave_id + + "sudo service mysql start" + ) + + retry = 0 + ssh_done = False + + while not ssh_done: + ret_code = subprocess.call(['ssh','-oUserKnownHostsFile=/dev/null','-oStrictHostKeyChecking=no', 'flaptor@%s' % slave_host, set_up_mysql_config]) + + if ret_code: + if retry < 3: + print 'WARNING: Couldn\'t configure and start mysql service at slave %s. Retrying' % slave_host + retry += 1 + time.sleep(1) + else: + print 'ERROR:Couldn\'t configure and start mysql service at slave %s' % slave_host + sys.exit(1) + else: + ssh_done = True + + ############################################################################################# + # This script requires a machine running with mysql, '****' user, and 'indextank' database + # 1.- Promote master_host to master (if applies) + # 2.- Dump data from master + # 3.- Import data in slave + # 4.- turn on replication + # 5.- check tables are synced + ############################################################################################# + + #################### + # Open connections # + #################### + + dbmaster = MySQLdb.connect(host=master_host,user=user,passwd=password,db='indextank') + dbslave = MySQLdb.connect(host=slave_host,user=user,passwd=password,db='indextank') + + ########################################## + # 1.- Promote master to slave (if applies) + ########################################## + + print 'MASTER: Checking master_host is master (and promoting to master if not)' + + mc = dbmaster.cursor() + is_slave = mc.execute('SHOW SLAVE STATUS') + if is_slave: + mc.execute('STOP SLAVE') + mc.execute('RESET MASTER') + mc.execute("CHANGE MASTER TO MASTER_HOST=''") + + is_slave = mc.execute('SHOW SLAVE STATUS') + if is_slave: + print 'Warning: MASTER (%s) seems to still work as slave' + + ############################ + # 2.- dump data from master + ############################ + print 'MASTER: Locking tables' + start_lock_time = time.time() + mc.execute('FLUSH TABLES WITH READ LOCK') + + print 'MASTER: Dumping data' + command_line = 'mysqldump --tables -h%s -u%s -p%s indextank' % (master_host, user, password) + dump_data_process = subprocess.Popen(shlex.split(command_line), stdout=subprocess.PIPE) + + ########################## + # 3.- import data in slave + ########################## + + # 3.1.- Stop slave + print 'SLAVE: Stopping slave process' + sc = dbslave.cursor() + sc.execute('stop slave') + + # 2.2.- import data + print 'SLAVE: Importing master dump' + ret_code = subprocess.call(['mysql', '-h%s' % slave_host, '-u%s' % user, '-p%s' % password, 'indextank'], stdin=dump_data_process.stdout) + dump_data_process.stdout.close() + + if ret_code: + print 'ERROR SLAVE: Import data failed on (%s)' % slave_host + sys.exit(1) + + ######################### + # 4.- turn on replication + ######################### + + # 4.1 Request master status + print 'MASTER: request master status' + mc.execute('SHOW MASTER STATUS') + master_status = mc.fetchone() + if not master_status: + print "ERROR: Couldn't turn replication on" + sys.exit(1) + + binlog_filename = master_status[0] + binlog_position = master_status[1] + + # 4.2 config master in slave and start slave + print 'SLAVE: config master in slave' + + sc.execute("CHANGE MASTER TO MASTER_HOST=%s, MASTER_USER=%s, MASTER_PASSWORD=%s, MASTER_LOG_FILE=%s, MASTER_LOG_POS=%s", [master_host, user, password, binlog_filename, binlog_position]) + + sc.execute('START SLAVE') + + is_slave = sc.execute('SHOW SLAVE STATUS') + + if not is_slave: + print 'WARNING SLAVE: Setting up slave failed' + + # 4.3 unlock master tables + print 'MASTER: unlock tables' + mc.execute('UNLOCK TABLES') + end_lock_time = time.time() + + print 'Locking time: %s sec.' % (end_lock_time - start_lock_time) + + ##################### + # Close connections # + ##################### + mc.close() + dbmaster.close() + + sc.close() + dbslave.close() + + ################################## + # 5. check tables are synchronized + ################################## + print 'Checking tables are synchronized' + chescksum_process = subprocess.Popen(['mk-table-checksum', '--databases', 'indextank', '-u'+user, '-p'+password, master_host, slave_host], stdout=subprocess.PIPE) + + checksum_output = chescksum_process.communicate()[0] + + checksum_lines = checksum_output.splitlines() + + found_error = False + for i in range(0,len(checksum_lines)/2): + table_check1 = checksum_lines[2*i+1].split() + table_check2 = checksum_lines[2*i+2].split() + is_synchronized = table_check1[0] == table_check2[0] and table_check1[1] == table_check2[1] and table_check1[4] == table_check2[4] and table_check1[6] == table_check2[6] + if not is_synchronized: + found_error = True + print 'ERROR: table %s.%s out of sync \n%s \n%s' % (table_check1[0], table_check1[1], table_check1, table_check2) + + if found_error: + print 'Some errors were found in synchronization. Please check master-slave status.' + else: + print 'Synchronization succesful.' + +# SET AND READ PARAMETERS +if __name__ == '__main__': + parser = OptionParser(usage="usage: %prog -m master_host -s slave_host -u db_user -p db_pass") + parser.add_option("-m", "--master_host", dest="master", help="Master host") + parser.add_option("-s", "--slave_host", dest="slave", help="Slave host") + parser.add_option("-u", "--user", dest="user", default=True, help="Database user") + parser.add_option("-p", "--passwd", dest="password", help="Database user") + + options, _ = parser.parse_args() + + if not options.master: + print 'master_host option is required' + sys.exit(1) + master_host = options.master + + if not options.slave: + print 'master_slave option is required' + sys.exit(1) + slave_host = options.slave + + if not options.user: + print 'user option is required' + sys.exit(1) + user = options.user + + if options.password: + password = options.password + else: + print 'Enter DB password:' + password = sys.stdin.readline() + + config_dbslave(master_host, slave_host, user, password) + diff --git a/backoffice/cloud/create_dbslave.py b/backoffice/cloud/create_dbslave.py new file mode 100644 index 0000000..1ac441a --- /dev/null +++ b/backoffice/cloud/create_dbslave.py @@ -0,0 +1,22 @@ +import subprocess, sys, json +from config_dbslave import config_dbslave +from launch_ami import launch_ami + +def create_dbslave(master_host): + print 'Creating DB SLAVE from MASTER %s' % master_host + instance_data = launch_ami('db') + + slave_host = instance_data['public_dns'] + config_dbslave(master_host, slave_host, '****', '****') + + print 'DB Slave succesfully started in: %s' % slave_host + +if __name__ == '__main__': + if len(sys.argv) < 2: + print "Usage: %s " % sys.argv[0] + sys.exit(1) + + master_host = sys.argv[1] + create_dbslave(master_host) + + diff --git a/backoffice/cloud/launch_ami.py b/backoffice/cloud/launch_ami.py new file mode 100644 index 0000000..e9e320f --- /dev/null +++ b/backoffice/cloud/launch_ami.py @@ -0,0 +1,42 @@ +import boto, sys, json, re + +from amazon_credential import AMAZON_USER, AMAZON_PASSWORD + +from replicate_instance import replicate_instance + +def ec2_connection(): + return boto.connect_ec2(AMAZON_USER, AMAZON_PASSWORD) + +def launch_ami(ami_type): + print 'Launching %s instance' % ami_type + print 'Finding %s ami' % ami_type + + conn = ec2_connection() + amis = conn.get_all_images() + + sort_function = lambda x: x.location + + selected_amis = [] + for ami in amis: + if re.search('indextank-%s' % ami_type, ami.location): + selected_amis.append(ami) + + if len(selected_amis) == 0: + print 'ERROR: no ami for type %s' % ami_type + sys.exit(1) + + selected_amis.sort(key=sort_function) + + ami = selected_amis[-1] + + print 'AMI found: %s %s' % (ami.id, ami.location) + return replicate_instance(ami_type, ami.id, logging=True) + +if __name__ == '__main__': + if len(sys.argv) < 2: + print "Usage: %s [fend|api|db]" % sys.argv[0] + sys.exit(1) + + ami_type = sys.argv[1] + print json.dumps(launch_ami(ami_type)) + diff --git a/backoffice/cloud/launch_ami.sh b/backoffice/cloud/launch_ami.sh new file mode 100755 index 0000000..213ea4d --- /dev/null +++ b/backoffice/cloud/launch_ami.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +show_help() { echo; echo "Usage: $0 [fend|api|db|log]"; echo; exit; } + +if [ $# -lt 1 ]; then + show_help +fi + +ami_type=$1 + +ami_id=`ec2-describe-images | awk '/^IMAGE/ {if($3~/indextank-'${ami_type}'-/) {print $2,$3}}' | sed -r 's/(ami-[0-9a-f]+) .*indextank-.*-([0-9.]+).*/\2 \1/' | sort -nr | head -1 | awk '{print $2}'` + +echo python replicate_instance.py $ami_type $ami_id +python replicate_instance.py $ami_type $ami_id diff --git a/backoffice/cloud/replicate_instance.py b/backoffice/cloud/replicate_instance.py new file mode 100644 index 0000000..79301b4 --- /dev/null +++ b/backoffice/cloud/replicate_instance.py @@ -0,0 +1,68 @@ +import boto, time +import socket, sys, json + +from amazon_credential import AMAZON_USER, AMAZON_PASSWORD + +def ec2_connection(): + return boto.connect_ec2(AMAZON_USER, AMAZON_PASSWORD) + +def replicate_instance(ami_type, ami_id, zone='us-east-1a', security_group=None, logging=False): + INSTANCE_TYPES = { + 'fend': 'c1.medium', + 'api': 'm1.xlarge', + 'db': 'm1.large', + 'log': 'm1.large' + } + + SECURITY_GROUPS = { + 'fend': 'indextank-main', + 'api': 'indextank-main', + 'db': 'indextank-db', + 'log': 'indextank-logstorage' + } + + if not ami_type in INSTANCE_TYPES or (not security_group and not ami_type in SECURITY_GROUPS): + raise Exception("Invalid instance type") + + if not security_group: + security_group = SECURITY_GROUPS[ami_type] + + conn = ec2_connection() + res = conn.run_instances(image_id=ami_id, security_groups=[security_group], instance_type=INSTANCE_TYPES[ami_type], placement=zone) + + if len(res.instances) == 0: + raise Exception("Replicated instance creation failed") + + instance = res.instances[0] + instance_name = instance.id + + if logging: + print "Successfully created replica (Instance: " + instance_name + ")" + + time.sleep(5) + + reservations = conn.get_all_instances([instance_name]) + instance = reservations[0].instances[0] + + while not instance.state == 'running': + if logging: + print "Waiting for the instance to start..." + time.sleep(10) + + reservations = conn.get_all_instances([instance_name]) + instance = reservations[0].instances[0] + + time.sleep(2) + + return {'id': instance_name, 'public_dns': instance.public_dns_name, 'private_dns': instance.private_dns_name} + +if __name__ == "__main__": + if len(sys.argv) < 3: + print 'replicate_instance receives at least two arguments' + sys.exit(1) + + if len(sys.argv) == 4: + print json.dumps(replicate_instance(sys.argv[1], sys.argv[2], sys.argv[3])) + else: + print json.dumps(replicate_instance(sys.argv[1], sys.argv[2])) + diff --git a/backoffice/forms.py b/backoffice/forms.py new file mode 100644 index 0000000..244d2aa --- /dev/null +++ b/backoffice/forms.py @@ -0,0 +1,9 @@ +from django import forms +from django.forms.widgets import HiddenInput + +class LoginForm(forms.Form): + email = forms.EmailField(required=True, label='E-Mail') + password = forms.CharField(widget=forms.PasswordInput, label='Password') + +class InvitationForm(forms.Form): + requesting_customer = forms.CharField(max_length=50, required=True) \ No newline at end of file diff --git a/backoffice/kill_webapp.sh b/backoffice/kill_webapp.sh new file mode 100755 index 0000000..35e822c --- /dev/null +++ b/backoffice/kill_webapp.sh @@ -0,0 +1,2 @@ +#!/bin/bash +kill `cat pid` diff --git a/backoffice/lib/__init__.py b/backoffice/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backoffice/lib/authorizenet.py b/backoffice/lib/authorizenet.py new file mode 100644 index 0000000..00bcd6c --- /dev/null +++ b/backoffice/lib/authorizenet.py @@ -0,0 +1,153 @@ +from xml.dom.minidom import Document, parseString +import httplib +import urlparse + + +class AuthorizeNet: + """ + Basic client for Authorize.net's Automated Recurring Billing (ARB) service + """ + + def __init__(self): + from django.conf import settings + f = open("authorize.settings.prod") if not settings.DEBUG else open("authorize.settings.debug") + for line in f: + line = line.strip() + if len(line) > 0 and not line.startswith('#'): + parts = line.split('=',1) + var = parts[0].strip() + val = parts[1].strip() + if var in ['host_url','api_login_id','transaction_key']: + cmd = 'self.%s = %s' % (var,val) + exec(cmd) + + def subscription_create(self, refId, name, length, unit, startDate, totalOccurrences, trialOccurrences, + amount, trialAmount, cardNumber, expirationDate, firstName, lastName, company, + address, city, state, zip, country): + doc,root = self._new_doc("ARBCreateSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + subscription = self._add_node(doc, root, 'subscription') + self._add_text_node(doc, subscription, 'name', name) + paymentSchedule = self._add_node(doc, subscription, 'paymentSchedule') + interval = self._add_node(doc, paymentSchedule, 'interval') + self._add_text_node(doc, interval, 'length', length) + self._add_text_node(doc, interval, 'unit', unit) + self._add_text_node(doc, paymentSchedule, 'startDate', startDate) + self._add_text_node(doc, paymentSchedule, 'totalOccurrences', totalOccurrences) + self._add_text_node(doc, paymentSchedule, 'trialOccurrences', trialOccurrences) + self._add_text_node(doc, subscription, 'amount', amount) + self._add_text_node(doc, subscription, 'trialAmount', trialAmount) + payment = self._add_node(doc, subscription, 'payment') + creditcard = self._add_node(doc, payment, 'creditCard') + self._add_text_node(doc, creditcard, 'cardNumber', cardNumber) + self._add_text_node(doc, creditcard, 'expirationDate', expirationDate) + billto = self._add_node(doc, subscription, 'billTo') + self._add_text_node(doc, billto, 'firstName', firstName) + self._add_text_node(doc, billto, 'lastName', lastName) + self._add_text_node(doc, billto, 'company', company) + self._add_text_node(doc, billto, 'address', address) + self._add_text_node(doc, billto, 'city', city) + self._add_text_node(doc, billto, 'state', state) + self._add_text_node(doc, billto, 'zip', zip) + self._add_text_node(doc, billto, 'country', country) + res = self._send_xml(doc.toxml()) + subscriptionId = res.getElementsByTagName('subscriptionId')[0].childNodes[0].nodeValue + return subscriptionId + + + def subscription_update(self, refId, subscriptionId, name, startDate, totalOccurrences, trialOccurrences, + amount, trialAmount, cardNumber, expirationDate, firstName, lastName, company, + address, city, state, zip, country): + doc,root = self._new_doc("ARBUpdateSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + self._add_text_node(doc, root, 'subscriptionId', subscriptionId) + subscription = self._add_node(doc, root, 'subscription') + if name: + self._add_text_node(doc, subscription, 'name', name) + if startDate or totalOccurrences or trialOccurrences: + paymentSchedule = self._add_node(doc, subscription, 'paymentSchedule') + if startDate: + self._add_text_node(doc, paymentSchedule, 'startDate', startDate) + if totalOccurrences: + self._add_text_node(doc, paymentSchedule, 'totalOccurrences', totalOccurrences) + if trialOccurrences: + self._add_text_node(doc, paymentSchedule, 'trialOccurrences', trialOccurrences) + if amount: + self._add_text_node(doc, subscription, 'amount', amount) + if trialAmount: + self._add_text_node(doc, subscription, 'trialAmount', trialAmount) + if cardNumber and expirationDate: + payment = self._add_node(doc, subscription, 'payment') + creditcard = self._add_node(doc, payment, 'creditCard') + self._add_text_node(doc, creditcard, 'cardNumber', cardNumber) + self._add_text_node(doc, creditcard, 'expirationDate', expirationDate) + if firstName and lastName: + billto = self._add_node(doc, subscription, 'billTo') + self._add_text_node(doc, billto, 'firstName', firstName) + self._add_text_node(doc, billto, 'lastName', lastName) + if company: + self._add_text_node(doc, billto, 'company', company) + if address and city and state and zip and country: + self._add_text_node(doc, billto, 'address', address) + self._add_text_node(doc, billto, 'city', city) + self._add_text_node(doc, billto, 'state', state) + self._add_text_node(doc, billto, 'zip', zip) + self._add_text_node(doc, billto, 'country', country) + self._send_xml(doc.toxml()) + + + def subscription_cancel(self, refId, subscriptionId): + doc,root = self._new_doc("ARBCancelSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + self._add_text_node(doc, root, 'subscriptionId', subscriptionId) + self._send_xml(doc.toxml()) + + + def _add_node(self, doc, node, name): + elem = doc.createElement(name) + node.appendChild(elem) + return elem + + def _add_text_node(self, doc, node, name, text): + elem = self._add_node(doc, node, name) + text_node = doc.createTextNode(text) + elem.appendChild(text_node) + return elem + + def _new_doc(self, operation): + doc = Document() + root = doc.createElement(operation) + root.setAttribute('xmlns','AnetApi/xml/v1/schema/AnetApiSchema.xsd') + doc.appendChild(root) + auth = self._add_node(doc, root, 'merchantAuthentication') + self._add_text_node(doc, auth, 'name', self.api_login_id) + self._add_text_node(doc, auth, 'transactionKey', self.transaction_key) + return doc, root + + def _send_xml(self, xml): + splits = urlparse.urlsplit(self.host_url) + print "connection.request('POST', "+self.host_url+", xml, {'Content-Type':'text/xml'})" + print "xml: "+xml + connection = httplib.HTTPSConnection(splits.hostname) + connection.request('POST', self.host_url, xml, {'Content-Type':'text/xml'}) + response = connection.getresponse() + response.body = response.read() + connection.close() + print "resp: "+response.body + res = parseString(response.body) + ok = res.getElementsByTagName('resultCode')[0].childNodes[0].nodeValue == "Ok" + if not ok: + code = res.getElementsByTagName('message')[0].childNodes[0].childNodes[0].nodeValue + msg = res.getElementsByTagName('message')[0].childNodes[1].childNodes[0].nodeValue + " (%s)"%code + raise BillingException(msg,code) + return res + + +class BillingException(Exception): + def __init__(self, msg, code): + self.msg = msg + self.code = code + def __str__(self): + return repr(self.msg) + + diff --git a/backoffice/lib/encoder.py b/backoffice/lib/encoder.py new file mode 100644 index 0000000..f6bb4dd --- /dev/null +++ b/backoffice/lib/encoder.py @@ -0,0 +1,74 @@ +# Short URL Generator + +#DEFAULT_ALPHABET = 'JedR8LNFY2j6MrhkBSADUyfP5amuH9xQCX4VqbgpsGtnW7vc3TwKE' +DEFAULT_ALPHABET = 'ed82j6rhkyf5amu9x4qbgpstn7vc3w1ioz' +DEFAULT_BLOCK_SIZE = 22 + +class Encoder(object): + def __init__(self, alphabet=DEFAULT_ALPHABET, block_size=DEFAULT_BLOCK_SIZE): + self.alphabet = alphabet + self.block_size = block_size + self.mask = (1 << block_size) - 1 + self.mapping = range(block_size) + self.mapping.reverse() + def encode_url(self, n, min_length=0): + return self.enbase(self.encode(n), min_length) + def decode_url(self, n): + return self.decode(self.debase(n)) + def encode(self, n): + return (n & ~self.mask) | self._encode(n & self.mask) + def _encode(self, n): + result = 0 + for i, b in enumerate(self.mapping): + if n & (1 << i): + result |= (1 << b) + return result + def decode(self, n): + return (n & ~self.mask) | self._decode(n & self.mask) + def _decode(self, n): + result = 0 + for i, b in enumerate(self.mapping): + if n & (1 << b): + result |= (1 << i) + return result + def enbase(self, x, min_length=0): + result = self._enbase(x) + padding = self.alphabet[0] * (min_length - len(result)) + return '%s%s' % (padding, result) + def _enbase(self, x): + n = len(self.alphabet) + if x < n: + return self.alphabet[x] + return self.enbase(x/n) + self.alphabet[x%n] + def debase(self, x): + n = len(self.alphabet) + result = 0 + for i, c in enumerate(reversed(x)): + result += self.alphabet.index(c) * (n**i) + return result + +DEFAULT_ENCODER = Encoder() + +def encode(n): + return DEFAULT_ENCODER.encode(n) + +def decode(n): + return DEFAULT_ENCODER.decode(n) + +def enbase(n, min_length=0): + return DEFAULT_ENCODER.enbase(n, min_length) + +def debase(n): + return DEFAULT_ENCODER.debase(n) + +def encode_url(n, min_length=0): + return DEFAULT_ENCODER.encode_url(n, min_length) + +def decode_url(n): + return DEFAULT_ENCODER.decode_url(n) + +def to_key(n): + return enbase(encode(n)) + +def from_key(n): + return decode(debase(n)) diff --git a/backoffice/lib/error_logging.py b/backoffice/lib/error_logging.py new file mode 100644 index 0000000..dbf927b --- /dev/null +++ b/backoffice/lib/error_logging.py @@ -0,0 +1,13 @@ +import traceback +from lib import flaptor_logging +from django.http import HttpResponse + +logger = flaptor_logging.get_logger('error_logging') + +class ViewErrorLoggingMiddleware: + + def process_view(self, request, view_func, view_args, view_kwargs): + self.view_name = view_func.__name__ + def process_exception(self, request, exception): + logger.error('UNEXPECTED EXCEPTION in view "%s". Exception is: %s', self.view_name, repr(traceback.print_exc())) + return HttpResponse('{"status":"ERROR", "message":"Unexpected error."}') diff --git a/backoffice/lib/exceptions.py b/backoffice/lib/exceptions.py new file mode 100644 index 0000000..c6ff0a7 --- /dev/null +++ b/backoffice/lib/exceptions.py @@ -0,0 +1,8 @@ + + +class CloudException(Exception): + pass + +class NoIndexerException(CloudException): + pass + diff --git a/backoffice/lib/flaptor_logging.py b/backoffice/lib/flaptor_logging.py new file mode 100644 index 0000000..1af893c --- /dev/null +++ b/backoffice/lib/flaptor_logging.py @@ -0,0 +1,100 @@ +import logging as pylogging +from logging import config +import os + +usingNativeLogger = True + +__loggers = {} + + + +def get_logger(name, force_new=False): + '''Get the Logger instance for a given name''' + global __loggers + if __loggers is None: + __loggers = {} + if force_new: + return pylogging.getLogger(name) + if not __loggers.has_key(name): + __loggers[name] = pylogging.getLogger(name) + return __loggers[name] + +class SpecialFormatter(pylogging.Formatter): + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) + RESET_SEQ = "\033[0m" + COLOR_SEQ = "\033[37;4%dm" + PIDCOLOR_SEQ = "\033[1;3%dm" + BOLD_SEQ = "\033[1m" + COLORS = { + 'WARN': YELLOW, + 'INFO': GREEN, + 'DEBU': BLUE, + 'CRIT': RED, + 'ERRO': RED + } + + def __init__(self, *args, **kwargs): + pylogging.Formatter.__init__(self, *args, **kwargs) + def format(self, record): + if not hasattr(record, 'prefix'): record.prefix = '' + if not hasattr(record, 'suffix'): record.suffix = '' + if not hasattr(record, 'compname'): record.compname = '' + record.pid = os.getpid() + + record.levelname = record.levelname[:4] + + r = pylogging.Formatter.format(self, record) + if record.levelname in SpecialFormatter.COLORS: + levelcolor = SpecialFormatter.COLOR_SEQ % (SpecialFormatter.COLORS[record.levelname]) + r = r.replace('$LEVELCOLOR', levelcolor) + r = r.replace('$RESET', SpecialFormatter.RESET_SEQ) + else: + r = r.replace('$COLOR', '') + r = r.replace('$RESET', '') + pidcolor = SpecialFormatter.COLOR_SEQ % (1 + (record.pid % 5)) + r = r.replace('$PIDCOLOR', pidcolor) + r = r.replace('$BOLD', SpecialFormatter.BOLD_SEQ) + return r + +pylogging.SpecialFormatter = SpecialFormatter + +if usingNativeLogger: + try: + config.fileConfig('logging.conf') + except Exception, e: + print e + +#class NativePythonLogger: +# def __init__(self, name): +# '''Creates a new Logger for the given name. +# Do not call this method directly, instead use +# get_logger(name) to get the appropriate instance''' +# self.name = name +# self.__logger = pylogging.getLogger(name) +# #self.updateLevel(5) +# +# def updateLevel(self, level): +# self.__level = level +# if level == 1: +# self.__logger.setLevel(pylogging.CRITICAL) +# elif level == 2: +# self.__logger.setLevel(pylogging.INFO) +# elif level == 3: +# self.__logger.setLevel(pylogging.WARNING) +# elif level == 4: +# self.__logger.setLevel(pylogging.INFO) +# elif level == 5: +# self.__logger.setLevel(pylogging.DEBUG) +# +# def debug(self, format_str, *values): +# self.__logger.debug(format_str, *values) +# def info(self, format_str, *values): +# self.__logger.info(format_str, *values) +# def warn(self, format_str, *values): +# self.__logger.warn(format_str, *values) +# def error(self, format_str, *values): +# self.__logger.error(format_str, *values) +# def exception(self, format_str, *values): +# self.__logger.exception(format_str, *values) +# def fatal(self, format_str, *values): +# self.__logger.critical(format_str, *values) diff --git a/backoffice/lib/mail.py b/backoffice/lib/mail.py new file mode 100644 index 0000000..b1d3a08 --- /dev/null +++ b/backoffice/lib/mail.py @@ -0,0 +1,101 @@ +from django.core.mail import send_mail + +try: + ENV=open('/data/env.name').readline().strip() +except Exception: + ENV='PROD+EXC' + +def _no_fail(method, *args, **kwargs): + def decorated(*args, **kwargs): + try: + return method(*args, **kwargs) + except Exception, e: + print e + return + return decorated + + + +@_no_fail +def report_payment_data(account): + activity_report = 'An Account has entered payment data\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + account.package.name + '\n' + activity_report += 'Email: ' + account.user.email + '\n' + activity_report += 'API KEY: ' + account.apikey + '' + + report_activity('Payment Data for ' + account.package.name + ' Account (' + account.user.email + ')', activity_report) + +@_no_fail +def report_new_account(account): + activity_report = 'A new Account was created\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + account.package.name + '\n' + activity_report += 'Email: ' + account.user.email + '\n' + activity_report += 'API KEY: ' + account.apikey + '' + + report_activity('New ' + account.package.name + ' Account (' + account.user.email + ')', activity_report) + +@_no_fail +def report_new_index(index): + activity_report = 'A new Index was created\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + index.account.package.name + '\n' + activity_report += 'User Email: ' + index.account.user.email + '\n' + activity_report += 'Index Name: ' + index.name + '' + + report_activity('Index activity (' + index.code + ')', activity_report, 'l') + +@_no_fail +def report_new_deploy(deploy): + activity_report = 'A new Deploy is now controllable\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + deploy.index.account.package.name + '\n' + activity_report += 'User Email: ' + deploy.index.account.user.email + '\n' + activity_report += 'Index Name: ' + deploy.index.name + '\n' + activity_report += 'Worker: #' + str(deploy.worker.id) + '\n' + activity_report += ('Deploy: %r' % deploy) + '\n' + activity_report += ('Container Index: %r' % deploy.index) + '\n' + + report_activity('Index activity (' + deploy.index.code + ')', activity_report, 'l') + +@_no_fail +def report_delete_index(index): + activity_report = 'An Index has been deleted\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + index.account.package.name + '\n' + activity_report += 'User Email: ' + index.account.user.email + '\n' + activity_report += 'Index Name: ' + index.name + '\n' + + report_activity('Index activity (' + index.code + ')', activity_report, 'l') + +@_no_fail +def report_new_worker(worker): + activity_report = 'A new Worker was created\n' + activity_report += '---------------------------\n' + activity_report += repr(worker) + + report_activity('New Worker (%d)' % (worker.pk), activity_report, 't') + +@_no_fail +def report_automatic_redeploy(deploy, initial_xmx, new_xmx): + activity_report = 'Automatic redeploy.\n' + activity_report += '---------------------------\n' + activity_report += 'initial xmx value: %d\n' % (initial_xmx) + activity_report += 'new xmx value: %d\n' % (new_xmx) + activity_report += repr(deploy) + + report_activity('Automatic redeploy', activity_report, 't') + +@_no_fail +def report_activity(subject, body, type='b'): + if type == 'b': + mail_to = 'activity@indextank.com' + elif type == 't': + mail_to = 'activitytech@indextank.com' + elif type == 'l': + mail_to = 'lowactivity@indextank.com' + else: + raise Exception('Wrong report type') + + send_mail(ENV + ' - ' + subject, body, 'IndexTank Activity ', [mail_to], fail_silently=False) diff --git a/backoffice/lib/monitor.py b/backoffice/lib/monitor.py new file mode 100644 index 0000000..3fe9818 --- /dev/null +++ b/backoffice/lib/monitor.py @@ -0,0 +1,148 @@ + +from threading import Thread +from traceback import format_tb +import time, datetime +import sys +import shelve + +from django.core.mail import send_mail + +from lib import flaptor_logging + +try: + ENV=open('/data/env.name').readline().strip() +except Exception: + ENV='PROD+EXC' + +#helper functions +def is_prod(): + return ENV == 'PROD' or ENV == 'QoS_Monitor' + +def env_name(): + if ENV == 'PROD': + return 'PRODUCTION' + elif ENV == 'QoS_Monitor': + return 'QoS_Monitor' + else: + return ENV + +class Monitor(Thread): + def __init__(self, pagerduty_email='api-monitor@flaptor.pagerduty.com'): + super(Monitor, self).__init__() + self.name = self.__class__.__name__ + self.statuses = shelve.open('/data/monitor-%s.shelf' % self.name) + self.logger = flaptor_logging.get_logger(self.name) + self.failure_threshold = 1 + self.fatal_failure_threshold = 0 + self.severity = 'WARNING' + self.title_template = '%s::%s: [%s] %s' + self.pagerduty_email = pagerduty_email + + def iterable(self): + return [None] + + def run(self): + self.step = 1 + while True: + starttime = int(time.time()) + try: + self.logger.info("running cycle %d", self.step) + for object in self.iterable(): + self._monitor(object) + self.report_ok("unexpected error in monitor cycle") + self.clean() + except Exception: + self.logger.exception("Unexpected error while executing cycle") + self.report_bad("unexpected error in monitor cycle", 1, 0, 'UNEXPECTED ERROR IN THE CYCLE OF %s\n\n%s' % (self.name, self.describe_error())) + self.step += 1 + self.statuses.sync() + time.sleep(max(0, self.period - (int(time.time()) - starttime))) + + def clean(self): + for title, status in self.statuses.items(): + if not status['working']: + if status['last_update'] != self.step: + self.report_ok(title) + else: + del self.statuses[title] + + + def _monitor(self, object): + try: + if self.monitor(object): + self.report_ok(str(self.alert_title(object))) + else: + self.report_bad(str(self.alert_title(object)), self.failure_threshold, self.fatal_failure_threshold, self.alert_msg(object)) + self.report_ok("unexpected error in monitor") + except Exception, e: + self.logger.exception("Unexpected error while executing monitor. Exception is: %s" % (e)) + message = 'UNEXPECTED ERROR IN THE MONITORING OF %s FOR TITLE: %s\n\n%s' % (self.name, self.alert_title(object), self.describe_error()) + self.report_bad("unexpected error in monitor", 1, 'WARNING', message) + + def describe_error(self): + exc_type, exc_value, exc_traceback = sys.exc_info() + return 'EXCEPTION: %s : %s\ntraceback:\n%s' % (exc_type, exc_value, ''.join(format_tb(exc_traceback))) + + def update_status(self, key, **kwargs): + self.statuses[key] = kwargs + + def send_alert(self, title, message, severity): + try: + if is_prod(): + if severity == 'FATAL': + name = 'FATAL ALERT (%s)' % env_name() + else: + name = 'ALERT (%s)' % env_name() + else: + name = '%s test alert' % ENV + + title = self.title_template % (ENV, self.name, severity, title) + message += '\n\n--------SENT AT ' + str(datetime.datetime.now()) + to = ['alerts@indextank.com'] + if severity == 'FATAL' and is_prod(): + to.append('alerts+fatal@indextank.com') + to.append(self.pagerduty_email) + send_mail(title, message, '"%s" ' % name, to, fail_silently=False) + self.logger.info('Sending alert for title: %s\n============\n%s', title, message) + except Exception, e: + self.logger.exception("Unexpected error while sending alerts. Exception is: %s" % (e)) + + def report_ok(self, title): + if title in self.statuses and not self.statuses[title]['working'] and (self.statuses[title]['alerted'] or self.statuses[title]['alerted_fatal']): + # it has just been resolved + self.send_alert(title, 'The problem is no longer reported. The last message was:\n %s' % (self.statuses[title]['message']), self.severity) + if title in self.statuses: + del self.statuses[title] + + def report_bad(self, title, threshold, fatal_threshold, message): + if title in self.statuses and not self.statuses[title]['working']: + # this object had already failed, let's grab the first step in which it failed + first_failure = self.statuses[title]['first_failure'] + has_alerted = self.statuses[title]['alerted'] + has_alerted_fatal = self.statuses[title]['alerted_fatal'] + else: + # this object was fine, first failure is now + first_failure = self.step + has_alerted = False + has_alerted_fatal = False + + + should_alert = self.step - first_failure + 1 >= threshold + should_alert_fatal = fatal_threshold > 0 and self.step - first_failure + 1 >= fatal_threshold + + if should_alert_fatal: + if not has_alerted_fatal: + has_alerted_fatal = True + if is_prod(): + self.send_alert(title, 'A new problem on IndexTank has been detected:\n %s' % (message), 'FATAL') + else: + self.logger.info('Fatal error was found but alert has already been sent') + elif should_alert: + if not has_alerted: + has_alerted = True + self.send_alert(title, 'A new problem on IndexTank has been detected:\n %s' % (message), self.severity) + else: + self.logger.info('Error was found but alert has already been sent') + + # save current state of the object (is_failed, message, first_failure, last_update) + self.update_status(title, working=False, last_update=self.step, message=message, first_failure=first_failure, alerted=has_alerted, alerted_fatal=has_alerted_fatal) diff --git a/backoffice/logging.conf b/backoffice/logging.conf new file mode 100644 index 0000000..c09a3b7 --- /dev/null +++ b/backoffice/logging.conf @@ -0,0 +1,35 @@ +[loggers] +keys=root,rpc,boto + +[handlers] +keys=consoleHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler + +[logger_rpc] +level=INFO +handlers=consoleHandler +propagate=0 +qualname=RPC + +[logger_boto] +level=INFO +handlers=consoleHandler +propagate=0 +qualname=boto + +[handler_consoleHandler] +class=StreamHandler +formatter=simpleFormatter +args=(sys.stdout,) + +[formatter_simpleFormatter] +format=%(pid)+5s %(asctime)s %(name)+8.8s:%(levelname)s%(prefix)s %(message)-90s %(suffix)s@%(filename)s:%(lineno)s +datefmt=%d/%m-%H.%M.%S +class=logging.SpecialFormatter + diff --git a/backoffice/manage.py b/backoffice/manage.py new file mode 100644 index 0000000..5e78ea9 --- /dev/null +++ b/backoffice/manage.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +from django.core.management import execute_manager +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) + sys.exit(1) + +if __name__ == "__main__": + execute_manager(settings) diff --git a/backoffice/models.py b/backoffice/models.py new file mode 100644 index 0000000..9c32fb1 --- /dev/null +++ b/backoffice/models.py @@ -0,0 +1 @@ +from api_linked_models import * \ No newline at end of file diff --git a/backoffice/rpc.py b/backoffice/rpc.py new file mode 100644 index 0000000..e03a661 --- /dev/null +++ b/backoffice/rpc.py @@ -0,0 +1 @@ +from api_linked_rpc import * #@UnusedWildImport diff --git a/backoffice/settings.py b/backoffice/settings.py new file mode 100644 index 0000000..05ce244 --- /dev/null +++ b/backoffice/settings.py @@ -0,0 +1,86 @@ +# Django settings for RTS project. + +DEBUG = True +TEMPLATE_DEBUG = DEBUG + +ADMINS = ( + # ('Your Name', 'your_email@domain.com'), +) + +MANAGERS = ADMINS + +DATABASE_ENGINE = 'mysql' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'ado_mssql'. +DATABASE_NAME = 'indextank' # Or path to database file if using sqlite3. +DATABASE_USER = '****' # Not used with sqlite3. +DATABASE_PASSWORD = '****' # Not used with sqlite3. +DATABASE_HOST = 'database' # Set to empty string for localhost. Not used with sqlite3. +DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'Etc/GMT+0' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.load_template_source', + 'django.template.loaders.app_directories.load_template_source', +# 'django.template.loaders.eggs.load_template_source', +) + +MIDDLEWARE_CLASSES = ( + 'django.middleware.gzip.GZipMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.locale.LocaleMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', +) + +ROOT_URLCONF = 'backoffice.urls' + +TEMPLATE_DIRS = ( + 'templates' + # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.admin', + 'django.contrib.humanize', + 'backoffice', +) + +AUTH_PROFILE_MODULE = 'backoffice.PFUser' +STATIC_URLS = [ '/_static' ] +ALLOWED_INCLUDE_ROOTS = ('static') +LOGIN_URL = '/login' + +#PROCS_DIR = '../indexengine/' + +USER_COOKIE_NAME = "pf_user" +COMMON_DOMAIN = 'localhost' +#SESSION_COOKIE_DOMAIN = COMMON_DOMAIN +FORCE_SCRIPT_NAME = '' +USE_MULTITHREADED_SERVER = True +LOGGER_CONFIG_FILE='logging.conf' + +# 0.001 MB per doc +INDEX_SIZE_RATIO = 0.002 + +EMAIL_HOST='localhost' +EMAIL_PORT=25 +EMAIL_HOST_USER='user%localhost' +EMAIL_HOST_PASSWORD='****' diff --git a/backoffice/start_webapp.sh b/backoffice/start_webapp.sh new file mode 100755 index 0000000..93209ff --- /dev/null +++ b/backoffice/start_webapp.sh @@ -0,0 +1,3 @@ +#!/bin/bash +LOGFILE='/data/logs/backoffice.log' +nohup python manage.py runfcgi method=prefork maxchildren=30 host=127.0.0.1 port=4200 pidfile=pid workdir="$PWD" outlog="$LOGFILE" errlog="$LOGFILE" >> $LOGFILE 2>&1 diff --git a/backoffice/static/biz_stats.css b/backoffice/static/biz_stats.css new file mode 100644 index 0000000..da6fce7 --- /dev/null +++ b/backoffice/static/biz_stats.css @@ -0,0 +1,30 @@ +body { + margin: 0; + padding-top: 2em; + padding-left: 2em; +} +div.h { + visibility: hidden; + position: absolute; + top: 3em; + left: 2em; +} +.ss8pt { +} +.hss8pt { + font-family: sans-serif; + font-size: 8pt; +} +.tss8pt { + font-family: sans-serif; + font-size: 8pt; + border-left: 1px solid white; + border-right: 1px solid silver; +} + +.bs_header { +} + +.bs_right { + text-align: right; +} diff --git a/backoffice/static/biz_stats.js b/backoffice/static/biz_stats.js new file mode 100644 index 0000000..4eeef91 --- /dev/null +++ b/backoffice/static/biz_stats.js @@ -0,0 +1,1001 @@ +google.load("jquery", "1.4.4"); +google.load("visualization", "1", {packages: ["table", "annotatedtimeline", "corechart"]}); + +const MS = ["01","02","03","04","05","06","07","08","09","10"]; + +function dateFormat(d) { + var dd = d.getUTCDate(); + var mm = d.getUTCMonth(); + var ms = (mm < 10)? MS[mm]:mm; + var yy = d.getUTCFullYear(); + if(dd < 10) dd = MS[dd-1]; + return yy+"-"+ms+"-"+dd; +} + +var rawJsonData = null; + +var activityTable = null; +var activityData = null; +var activityDataReset = null; +var activitySetup = { + allowHtml: true, + showRowNumber: true, + width: 1000, + sortColumn: 0, + sortAscending: false, + cssClassNames: {tableCell: "tss8pt", headerCell: "hss8pt"} +}; + +const activityDataColumnIndexPackage = 4; +const activityDataColumnIndexCount = 5; + +const historyDataColumnIndexPackage = 3; + +var historyTable = null; +var historyData = null; +var historyDataReset = null; +var historySetup = { + allowHtml: true, + showRowNumber: true, + width: 1000, + sortColumn: 7, + sortAscending: false, + cssClassNames: {tableCell: "tss8pt", headerCell: "hss8pt"} +}; +//,cssClassNames: {tableCell: "ss8pt", headerCell: "ss8pt", headerRow: "bs_header"} + +function historyTableHideDataColumns(columns) { +// historyTable.hideDataColumns(0); +// historyData.removeColumn(0); + +// historyTable.draw(historyData, historySetup); +} + +function historyTablePackageFilter(packageNames) { + var rows = historyData.getNumberOfRows(); + var toBeRemoved = []; + for(var i = 0; i < rows; i++) { + var value = historyData.getValue(i, historyDataColumnIndexPackage); + for(var j = 0; j < packageNames.length; j++) { + var packageName = packageNames[j]; + if(packageName == value) { + toBeRemoved.push(i); + } + } + } + for(var i = toBeRemoved.length-1; i >= 0; i--) { + historyData.removeRow(toBeRemoved[i]); + } + historyTable.draw(historyData, historySetup); +} + +function historyTablePackageFilterOnly(packageNames) { + var rows = historyData.getNumberOfRows(); + var toBeRemoved = []; + for(var i = 0; i < rows; i++) { + var value = historyData.getValue(i, historyDataColumnIndexPackage); + for(var j = 0; j < packageNames.length; j++) { + var packageName = packageNames[j]; + if(packageName != value) { + toBeRemoved.push(i); + } + } + } + for(var i = toBeRemoved.length-1; i >= 0; i--) { + historyData.removeRow(toBeRemoved[i]); + } + historyTable.draw(historyData, historySetup); +} + +function activityTableCountFilter() { + var rows = activityData.getNumberOfRows(); + var toBeRemoved = []; + for(var i = 0; i < rows; i++) { + var value = activityData.getValue(i, activityDataColumnIndexCount); + if(value <= 0) { + toBeRemoved.push(i); + } + } + for(var i = toBeRemoved.length-1; i >= 0; i--) { + activityData.removeRow(toBeRemoved[i]); + } + activityTable.draw(activityData, activitySetup); +} + +function activityTablePackageFilter(packageNames) { + var rows = activityData.getNumberOfRows(); + var toBeRemoved = []; + for(var i = 0; i < rows; i++) { + var value = activityData.getValue(i, activityDataColumnIndexPackage); + for(var j = 0; j < packageNames.length; j++) { + var packageName = packageNames[j]; + if(packageName == value) { + toBeRemoved.push(i); + } + } + } + for(var i = toBeRemoved.length-1; i >= 0; i--) { + activityData.removeRow(toBeRemoved[i]); + } + activityTable.draw(activityData, activitySetup); +} + +function activityTablePackageFilterOnly(packageNames) { + var rows = activityData.getNumberOfRows(); + var toBeRemoved = []; + for(var i = 0; i < rows; i++) { + var value = activityData.getValue(i, activityDataColumnIndexPackage); + for(var j = 0; j < packageNames.length; j++) { + var packageName = packageNames[j]; + if(packageName != value) { + toBeRemoved.push(i); + } + } + } + for(var i = toBeRemoved.length-1; i >= 0; i--) { + activityData.removeRow(toBeRemoved[i]); + } + activityTable.draw(activityData, activitySetup); +} + +function activityTableReset() { + activityData = activityDataReset; + activityDataReset = activityData.clone(); + activityTable.draw(activityData, activitySetup); +} + +google.setOnLoadCallback(function() { + $(function() { + + function descriptiveLogStats(logStats) { + return "
"
+        + "Mean:   " + logStats.mean + "ms\n"
+        + "Median: " + logStats.median + "ms\n"
+        + "Min:    " + logStats.min + "ms\n"
+        + "Max:    " + logStats.max + "ms\n"
+        + "95% (" + logStats.Q95K + ") are less than " + logStats.Q95V + "ms"
+        + "
"; + } + + function activity(data) { + + var s = data.getHistory.length; + //console.log(data.getHistory); + + return "" + //+ "
API elapsed time per operation (per account indexes, all accounts)
" + //+ descriptiveLogStats(data.logStats) + + "
API elapsed time per Get hit (per account indexes, all accounts)
" + + descriptiveLogStats(data.logGetStats) + //+ "
API Put elapsed time per operation (per account indexes, all accounts)
" + //+ descriptiveLogStats(data.logPutStats) + //+ "\n" + ; + } + + function revenue(data) { + //TODO: change this to use package.base_price + //TODO: account.package.code.startswith('HEROKU_') + var herokuProfit = .7; + var accountList = data.accountPaymentList; + var herokuAmountTotal = 0; + var amountTotal = 0; + var typeMap = {}; + for(var i = 0; i < accountList.length; i++) { + if(accountList[i].package.substring(0,"HEROKU".length) === "HEROKU") + herokuAmountTotal += parseInt(accountList[i].packagePrice); + else { + var gatewayPrice = parseInt(accountList[i].gatewayPrice); + amountTotal += gatewayPrice; + var prev = typeMap[accountList[i].subscription]; + if(prev == undefined) + typeMap[accountList[i].subscription] = gatewayPrice; + else + typeMap[accountList[i].subscription] = prev + gatewayPrice; + } + } + + var herokuNetTotal = Math.round(herokuProfit * herokuAmountTotal); + + var total = herokuNetTotal + amountTotal; + var gateways = ""; + $.each(typeMap, function(k,v) { + gateways += k + "\t\t: " + v + " usd\n"; + }); + + return "" + + "
Monthly subscriptions
" + + "
"
+        + gateways
+        + "Heroku addon    : " + herokuNetTotal + " usd (" + herokuAmountTotal + " gross)\n"
+        + "Total           : " + total + " usd\n"
+        + "
" + ; + } + + function renderSubscriptionList(accountSubscriptionList) { + var data = new google.visualization.DataTable(); + data.addColumn("number", "Id"); + data.addColumn("string", "First Name"); + data.addColumn("string", "Last Name"); + data.addColumn("string", "Country"); + data.addColumn("string", "Start Date"); + data.addColumn("number", "Amount"); + data.addColumn("number", "AmountDue"); + data.addRows(accountSubscriptionList.length); + for(var i = 0; i < accountSubscriptionList.length; i++) { + + data.setCell(i, 0, parseInt(accountSubscriptionList[i].id)); + data.setCell(i, 1, accountSubscriptionList[i].fistName); + data.setCell(i, 2, accountSubscriptionList[i].lastName); + data.setCell(i, 3, accountSubscriptionList[i].country); + data.setCell(i, 4, accountSubscriptionList[i].startDate); + data.setCell(i, 5, parseFloat(accountSubscriptionList[i].amount)); + data.setCell(i, 6, parseFloat(accountSubscriptionList[i].amountDue)); + } + var table = new google.visualization.Table(document.getElementById("subscriptionsList")); + + var setup = { + allowHtml: true, + showRowNumber: true, + sortColumn: 0, + sortAscending: false, + cssClassNames: {tableCell: "ss8pt", headerCell: "ss8pt"} + }; + + var sortInfoColumn = 0; + var sortInfoAscending = false; + google.visualization.events.addListener(table, "sort", function(e) { + sortInfoAscending = (sortInfoColumn == e.column)? !sortInfoAscending : false; + sortInfoColumn = e.column; + setup.sortColumn = sortInfoColumn; + setup.sortAscending = sortInfoAscending; + table.draw(data, setup); + }); + + table.draw(data, setup); + + } + + function renderPaymentList(list) { + var data = new google.visualization.DataTable(); + data.addColumn("number", "Id"); + data.addColumn("string", "Email"); + data.addColumn("string", "Creation"); + data.addColumn("string", "Package"); + data.addColumn("number", "Amount"); + data.addColumn("number", "Total"); + data.addRows(list.length); + for(var i = 0; i < list.length; i++) { + + var amount; + var packagePrice = parseInt(list[i].packagePrice); + var gatewayPrice = parseInt(list[i].gatewayPrice); + var monthsBetween = parseInt(list[i].monthsBetween); + if(gatewayPrice == 0) { + amount = packagePrice; + } else { + amount = gatewayPrice; + } + var amountTotal = amount * monthsBetween; + j = 0; + data.setCell(i, j++, parseInt(list[i].id)); + data.setCell(i, j++, list[i].email); + data.setCell(i, j++, list[i].creation); + data.setCell(i, j++, list[i]['package']); + data.setCell(i, j++, amount); + data.setCell(i, j++, amountTotal); + } + var table = new google.visualization.Table(document.getElementById("subscriptionsList")); + + var setup = { + allowHtml: true, + showRowNumber: true, + sortColumn: 0, + sortAscending: false, + cssClassNames: {tableCell: "ss8pt", headerCell: "ss8pt"} + }; + + var sortInfoColumn = 0; + var sortInfoAscending = false; + google.visualization.events.addListener(table, "sort", function(e) { + sortInfoAscending = (sortInfoColumn == e.column)? !sortInfoAscending : false; + sortInfoColumn = e.column; + setup.sortColumn = sortInfoColumn; + setup.sortAscending = sortInfoAscending; + table.draw(data, setup); + }); + + table.draw(data, setup); + + } + function renderTwitterHistory(accountHistory) { + var data = new google.visualization.DataTable(); + data.addColumn("date", "Date"); + data.addColumn("number", "Tweets"); + + data.addRows(accountHistory.length); + + for(var i = 0; i < accountHistory.length; i++) { + data.setCell(i, 0, new Date(accountHistory[i].date)); + data.setCell(i, 1, parseInt(accountHistory[i].count)); + } + + var annotatedtimeline = new google.visualization.AnnotatedTimeLine(document.getElementById("visualization")); + annotatedtimeline.draw(data, { + "displayAnnotations": false, + "displayRangeSelector": true, + "displayZoomButtons": false, + "fill": 50, + "colors": ["black"] + }); + } + + function renderActivationAARRR(hist) { + var data = new google.visualization.DataTable(); + data.addColumn("date", "Date"); + data.addColumn("number", "% Created an index"); + data.addColumn("number", "% Used an index"); + + var keys = Object.keys(hist); + data.addRows(keys.length); + + for(var i = 0; i < keys.length; i++) { + var k = keys[i]; + var v = hist[k]; + var j = 0; + var t = parseInt(v[0]); + data.setCell(i, j++, new Date(k)); + data.setCell(i, j++, Math.round(100*parseInt(v[1])/t), v[1]); + data.setCell(i, j++, Math.round(100*parseInt(v[2])/t), v[2]); + } + + var annotatedtimeline = new google.visualization.AnnotatedTimeLine(document.getElementById("activationAARRRVisualization")); + annotatedtimeline.draw(data, { + "displayAnnotations": false, + "displayRangeSelector": false, + "displayZoomButtons": false, + "fill": 50, + "colors": ["silver", "gray", "black"] + }); + } + + function renderRetentionAARRR(hist) { + var data = new google.visualization.DataTable(); + data.addColumn("date", "Date"); + data.addColumn("number", "% GET"); + data.addColumn("number", "% PUT"); + data.addColumn("number", "% GET/PUT"); + + var keys = Object.keys(hist); + data.addRows(keys.length); + + for(var i = 0; i < keys.length; i++) { + var k = keys[i]; + var v = hist[k]; + var j = 0; + var t = parseInt(v[0]); + data.setCell(i, j++, new Date(k)); + data.setCell(i, j++, Math.round(100*parseInt(v[1])/t), v[1]); + data.setCell(i, j++, Math.round(100*parseInt(v[2])/t), v[2]); + data.setCell(i, j++, Math.round(100*parseInt(v[3])/t), v[3]); + } + + var annotatedtimeline = new google.visualization.AnnotatedTimeLine(document.getElementById("retentionAARRRVisualization")); + annotatedtimeline.draw(data, { + "displayAnnotations": false, + "displayRangeSelector": false, + "displayZoomButtons": false, + "fill": 50, + "colors": ["silver", "gray", "black"] + }); + } + + function renderAccountHistory(accountHistory) { + var data = new google.visualization.DataTable(); + data.addColumn("date", "Date"); + data.addColumn("number", "Total"); + data.addColumn("number", "New"); + + data.addRows(accountHistory.length); + + for(var i = 0; i < accountHistory.length; i++) { + data.setCell(i, 0, new Date(accountHistory[i].date)); + data.setCell(i, 1, parseInt(accountHistory[i].total)); + data.setCell(i, 2, parseInt(accountHistory[i].count)); + } + + var annotatedtimeline = new google.visualization.AnnotatedTimeLine(document.getElementById("accountsHistoryVisualization")); + annotatedtimeline.draw(data, { + "displayAnnotations": false, + "displayRangeSelector": true, + "displayZoomButtons": false, + "fill": 50, + "colors": ["black"] + }); + } + + function renderAdoptionAARRR1(accountHistory, element) { + var data = new google.visualization.DataTable(); + data.addColumn("string", "Date"); + data.addColumn("number", "New Accounts"); + + var length = Object.keys(accountHistory).length; + data.addRows(length); + + var i = 0; + $.each(accountHistory, function(k,v) { + data.setValue(i, 0, k); + data.setValue(i, 1, v); + i++; + }); + var vis = new google.visualization.LineChart(element); + vis.draw(data, {}); + } + + function renderAdoptionAARRR2(accountHistory, element) { + var data = new google.visualization.DataTable(); + data.addColumn("date", "Date"); + data.addColumn("number", "New Accounts"); + + var length = Object.keys(accountHistory).length; + data.addRows(length); + + var i = 0; + $.each(accountHistory, function(k,v) { + data.setValue(i, 0, new Date(v.date)); + data.setValue(i, 1, parseInt(v.count)); + i++; + }); + var vis = new google.visualization.LineChart(element); + vis.draw(data, {}); + } + + function renderAdoptionAARRR(json) { + renderAdoptionAARRR1(json.monthAccountHistory, document.getElementById("adoptionAARRRVisualization")); + renderAdoptionAARRR1(json.weekAccountHistory, document.getElementById("adoptionAARRRVisualization1")); + renderAdoptionAARRR2(json.accountHistory, document.getElementById("adoptionAARRRVisualization2")); + } + + function renderAARRR(data) { + var d = new Date(); + // prev week + var w = new Date(d.getTime() - 604800000); + // first ms prev week + var lastWeekMillis = new Date(w.getFullYear(), w.getMonth(), w.getDate()); + + var accountCreatedPerWeek = {}; + var accountIndexCreatedPerWeek = {}; + var accountIndexUsedPerWeek = {}; + + for(var i = 0; i < data.accountList.length; i++) { + var e = data.accountList[i]; + + if(new Date(e.creation).getTime() >= lastWeekMillis) { + accountCreatedPerWeek[e.name] = e.name; + } + } + + for(var i = 0; i < data.indexList.length; i++) { + var j = 0; + var e = data.indexList[i]; + var documentCount = parseInt(e.documentCount); + + if(e.indexName == "DemoIndex") { + continue; + } + + if(new Date(e.accountCreation).getTime() >= lastWeekMillis) { + if(new Date(e.creation).getTime() >= lastWeekMillis) { + accountIndexCreatedPerWeek[e.name] = e.name; + if(documentCount > 0) { + accountIndexUsedPerWeek[e.name] = e.name; + } + } + } + + } + + var accountCreatedPerWeekTotal = Object.keys(accountCreatedPerWeek).length; + var accountIndexCreatedPerWeekTotal = Object.keys(accountIndexCreatedPerWeek).length; + var accountIndexUsedPerWeekTotal = Object.keys(accountIndexUsedPerWeek).length; + + var accountIndexCreatedPerWeekPercent = accountIndexCreatedPerWeekTotal / accountCreatedPerWeekTotal; + var accountIndexUsedPerWeekPercent = accountIndexUsedPerWeekTotal / accountCreatedPerWeekTotal; + + + return "" + + "
Weekly account activations
" + + "
"
+        + "Total new accounts : " + accountCreatedPerWeekTotal + " (last 7 days)\n"
+        + "Created an index   : " + Math.round(100*accountIndexCreatedPerWeekPercent) + "% ("+ accountIndexCreatedPerWeekTotal +")\n"
+        + "Used an index      : " + Math.round(100*accountIndexUsedPerWeekPercent) + "% ("+ accountIndexUsedPerWeekTotal +") \n"   
+        + "
" + ; + } + + function renderGlobalHistory(gets, puts) { + var data = new google.visualization.DataTable(); + data.addColumn("date", "Date"); + data.addColumn("number", "Get hits"); + data.addColumn("number", "Put hits"); + data.addColumn("number", "Total hits"); + + var keys = Object.keys(gets); + data.addRows(keys.length); + + for(var i = 0; i < keys.length; i++) { + data.setCell(i, 0, new Date(keys[i])); + var g = parseInt(gets[keys[i]]); + var p = parseInt(puts[keys[i]]); + var t = p + g; + data.setCell(i, 1, g); + data.setCell(i, 2, p); + data.setCell(i, 3, t); + } + + var annotatedtimeline = new google.visualization.AnnotatedTimeLine(document.getElementById("globalHistoryVisualization")); + annotatedtimeline.draw(data, { + "displayAnnotations": false, + "displayRangeSelector": false, + "displayZoomButtons": false, + "fill": 50, + "colors": ["black", "gray", "silver"] + }); + } + + function renderIndexList(indexList) { + var data = new google.visualization.DataTable(); + data.addColumn("string", "Account"); + data.addColumn("string", "Email"); + data.addColumn("string", "Index"); + data.addColumn("number", "Documents"); + data.addColumn("number", "Id"); + data.addColumn("string", "Code"); + data.addColumn("string", "Creation"); + + data.addRows(indexList.length); + + for(var i = 0; i < indexList.length; i++) { + var j = 0; + var e = indexList[i]; + var documentCount = parseInt(indexList[i].documentCount); + + data.setCell(i, j++, indexList[i].name); + data.setCell(i, j++, indexList[i].email); + data.setCell(i, j++, indexList[i].indexName); + data.setCell(i, j++, documentCount); + data.setCell(i, j++, parseInt(indexList[i].indexId)); + data.setCell(i, j++, indexList[i].indexCode); + data.setCell(i, j++, e.creation); + } + + var table = new google.visualization.Table(document.getElementById("indexesList")); + + var setup = { + allowHtml: true, + showRowNumber: true, + sortColumn: 6, + sortAscending: false, + cssClassNames: {tableCell: "ss8pt", headerCell: "ss8pt"} + }; + + var sortInfoColumn = 6; + var sortInfoAscending = false; + google.visualization.events.addListener(table, "sort", function(e) { + sortInfoAscending = (sortInfoColumn == e.column)? !sortInfoAscending : false; + sortInfoColumn = e.column; + setup.sortColumn = sortInfoColumn; + setup.sortAscending = sortInfoAscending; + table.draw(data, setup); + }); + + table.draw(data, setup); + } + + function renderQosList(customMap, accountEmail) { + var data = new google.visualization.DataTable(); + data.addColumn("string", "
"+"Account"+"
"); + data.addColumn("string", "
"+"Email"+"
"); + data.addColumn("string", "
"+"HTTP"+"
"); + for(var i = 0; i < 24; i++) { + data.addColumn("number", "
"+i+"
"); + } + + // if we're showing per account rows: + //data.addRows(Object.keys(customMap).length); + + var df = dateFormat(new Date()); + var i = 0; + $.each(customMap, function(k,v) { + // if we're showing all http codes: + //data.addRows(Object.keys(v).length); + $.each(v, function(k1, v1) { + // return only http code 200 + if(k1 != 200) return; + data.addRows(1); + var j = 0; + data.setCell(i, j++, k); + if(accountEmail != undefined && k in accountEmail) + data.setCell(i, j, accountEmail[k]); + j++; + data.setCell(i, j++, k1); + $.each(v1, function(k2, v2) { + var gmtHour = k2.split(" ")[1]; + var d = df+" "+gmtHour; + var vv2 = v1[d]; + if(vv2 != undefined) { + data.setCell(i, parseInt(gmtHour)+3, Math.round(v2[1]/v2[0])); + } + }); + i++; + }); + }); + + var table = new google.visualization.Table(document.getElementById("qosList")); + + var setup = { + allowHtml: true, + showRowNumber: true, + sortColumn: 0, + sortAscending: false, + cssClassNames: {tableCell: "tss8pt", headerCell: "tss8pt"} + }; + + var sortInfoColumn = 0; + var sortInfoAscending = false; + google.visualization.events.addListener(table, "sort", function(e) { + sortInfoAscending = (sortInfoColumn == e.column)? !sortInfoAscending : false; + sortInfoColumn = e.column; + setup.sortColumn = sortInfoColumn; + setup.sortAscending = sortInfoAscending; + table.draw(data, setup); + }); + + table.draw(data, setup); + } + + + function renderQosErrorList(customMap, accountEmail) { + var data = new google.visualization.DataTable(); + + var d = new Date(); + var ds = new Array(); + + //generate prev 5 days keys + const days = 5; + for(var k = days-1; k >= 0; k--) { + ds[k] = dateFormat(d); + d = new Date( d.getTime() - 86400000 ); + } + + data.addColumn("string", "
"+"Account"+"
"); + data.addColumn("string", "
"+"Email"+"
"); + for(var i = 0; i < days; i++) { + data.addColumn("number", "
"+ds[i]+"
"); + } + + data.addRows(Object.keys(customMap).length); + + var i = 0; + $.each(customMap, function(k,v) { + var j = 0; + // account code + data.setCell(i, j++, k); + // account email + if(accountEmail != undefined && k in accountEmail) + data.setCell(i, j, accountEmail[k]); + j++; + for(var l=0; l < days; l++) { + var vv = v[ds[l]]; + if(vv != undefined) { + data.setCell(i, j+l, parseInt(vv)); + } + } + i++; + }); + + var table = new google.visualization.Table(document.getElementById("qosErrorList")); + + var setup = { + allowHtml: true, + showRowNumber: true, + sortColumn: 6, + sortAscending: false, + cssClassNames: {tableCell: "tss8pt", headerCell: "tss8pt"} + }; + + var sortInfoColumn = 6; + var sortInfoAscending = false; + google.visualization.events.addListener(table, "sort", function(e) { + sortInfoAscending = (sortInfoColumn == e.column)? !sortInfoAscending : false; + sortInfoColumn = e.column; + setup.sortColumn = sortInfoColumn; + setup.sortAscending = sortInfoAscending; + table.draw(data, setup); + }); + + table.draw(data, setup); + } + + + function renderQos95List(customMap, accountEmail) { + var data = new google.visualization.DataTable(); + + data.addColumn("string", "
"+"Account"+"
"); + data.addColumn("string", "
"+"Email"+"
"); + data.addColumn("number", "
Q95
"); + + data.addRows(Object.keys(customMap).length); + + var i = 0; + $.each(customMap, function(k,v) { + var j = 0; + // account code + // account email + if(accountEmail != undefined && k in accountEmail) { + data.setCell(i, j++, k); + data.setCell(i, j++, accountEmail[k]); + data.setCell(i, j++, Math.round(v*1000)); + } + i++; + }); + + var table = new google.visualization.Table(document.getElementById("qos95List")); + + var setup = { + allowHtml: true, + showRowNumber: true, + sortColumn: 2, + sortAscending: false, + cssClassNames: {tableCell: "tss8pt", headerCell: "tss8pt"} + }; + + var sortInfoColumn = 2; + var sortInfoAscending = false; + google.visualization.events.addListener(table, "sort", function(e) { + sortInfoAscending = (sortInfoColumn == e.column)? !sortInfoAscending : false; + sortInfoColumn = e.column; + setup.sortColumn = sortInfoColumn; + setup.sortAscending = sortInfoAscending; + table.draw(data, setup); + }); + + table.draw(data, setup); + } + + + function renderAccountHistoryList(accountList) { + var d = new Date(); + var ds = new Array(); + + //generate prev 5 days keys + for(var k = 0; k < 5; k++) { + var dd = d.getUTCDate(); + var mm = d.getUTCMonth(); + var ms = (mm < 10)? MS[mm]:mm; + var yy = d.getUTCFullYear(); + if(dd < 10) dd = MS[dd-1]; + ds[k] = yy+"-"+ms+"-"+dd; + d = new Date( d.getTime() - 86400000 ); + } + + var data = new google.visualization.DataTable(); + data.addColumn("number", "
"+"Id"+"
"); + data.addColumn("string", "
"+"Email"+"
"); + data.addColumn("string", "
"+"Name"+"
"); + data.addColumn("string", "
"+"Package"+"
"); + data.addColumn("number", "
"+"Documents"+"
"); + data.addColumn("number", "
"+ds[4]+"
"); + data.addColumn("number", "
"+ds[3]+"
"); + data.addColumn("number", "
"+ds[2]+"
"); + data.addColumn("number", "
"+ds[1]+"
"); + data.addColumn("number", "
"+ds[0]+"
"); + data.addColumn("number", "
"+"Average"+"
"); + + data.addRows(accountList.length); + + for(var i = 0; i < accountList.length; i++) { + var j = 0; + var count = parseInt(accountList[i].count); + var email = accountList[i].email; + if(email.length > 35) + email = email.substring(0, 35) + "..."; + + data.setCell(i, j++, parseInt(accountList[i].id)); + data.setCell(i, j++, email); + data.setCell(i, j++, accountList[i].name); + data.setCell(i, j++, accountList[i].package); + var documentCount = parseInt(accountList[i].indexCount); + if(documentCount > 0) data.setCell(i, j, documentCount); j++; + + if(accountList[i].getHistory != undefined) { + var tqs = 0; + for(var k = 0; k < ds.length; k++) { + var qs = accountList[i].getHistory[ds[4-k]]; + if(qs != undefined && k < 4) + tqs += qs; + data.setCell(i, j++, qs); + } + var average = Math.round(tqs/4); + if(average > 0) data.setCell(i, j++, average); + } + + } + + var table = new google.visualization.Table(document.getElementById("historyList")); + + var sortInfoColumn = 7; + var sortInfoAscending = false; + google.visualization.events.addListener(table, "sort", function(e) { + sortInfoAscending = (sortInfoColumn == e.column)? !sortInfoAscending : false; + sortInfoColumn = e.column; + historySetup.sortColumn = sortInfoColumn; + historySetup.sortAscending = sortInfoAscending; + table.draw(data, historySetup); + }); + + table.draw(data, historySetup); + + historyTable = table; + historyData = data; + historyDataReset = data.clone(); + + } + + + function renderAccountList(accountList) { + var data = new google.visualization.DataTable(); + + data.addColumn("number", "
"+"Id"+"
"); + data.addColumn("string", "
"+"Email"+"
"); + data.addColumn("string", "
"+"Name"+"
"); + data.addColumn("string", "
"+"Creation"+"
"); + data.addColumn("string", "
"+"Package"+"
"); + //data.addColumn("number", "
"+"Count"+"
"); + //data.addColumn("number", "
"+"AvgSecs"+"
"); + data.addColumn("string", "
"+"LastGET"+"
"); + data.addColumn("string", "
"+"LastPUT"+"
"); + data.addColumn("number", "
"+"GET"+"
"); + data.addColumn("number", "
"+"PUT"+"
"); + data.addColumn("number", "
"+"Documents"+"
"); + data.addColumn("number", "
"+"Indexes"+"
"); + + //data.addColumn("number", "Usage Ratio (GET/PUT)"); + + //data.addColumn("string", "Country"); + //data.addColumn("string", "Start Date"); + //data.addColumn("string", "Amount"); + //data.addColumn("string", "Amount Billed"); + + data.addRows(accountList.length); + + for(var i = 0; i < accountList.length; i++) { + var j = 0; + var count = parseInt(accountList[i].count); + var avg = parseFloat(accountList[i].avg); + var ratio = parseFloat(accountList[i].ratio); + var getCount = parseInt(accountList[i].getCount); + var putCount = parseInt(accountList[i].putCount); + var indexes = parseInt(accountList[i].indexCount); + var documents = parseInt(accountList[i].documentCount); + var email = accountList[i].email; + if(email.length > 35) + email = email.substring(0, 35) + "..."; + + data.setCell(i, j++, parseInt(accountList[i].id)); + data.setCell(i, j++, email); + data.setCell(i, j++, accountList[i].name); + data.setCell(i, j++, accountList[i].creation); + data.setCell(i, j++, accountList[i].package); + //if(count > 0) data.setCell(i, j, count); j++; + //if(avg > 0) data.setCell(i, j, avg); j++; + data.setCell(i, j++, accountList[i].get); + data.setCell(i, j++, accountList[i].put); + //if(ratio > 0) data.setCell(i, j, ratio); j++; + if(getCount > 0) data.setCell(i, j, getCount); j++; + if(putCount > 0) data.setCell(i, j, putCount); j++; + //data.setCell(i, j++, accountList[i].country); + //data.setCell(i, j++, accountList[i].startDate); + //data.setCell(i, j++, accountList[i].amount); + //data.setCell(i, j++, accountList[i].amountDue); + if(indexes > 0) data.setCell(i, j, indexes); j++; + if(documents > 0) data.setCell(i, j, documents); j++; + } + + var table = new google.visualization.Table(document.getElementById("accountsList")); + + var sortInfoColumn = 0; + var sortInfoAscending = false; + google.visualization.events.addListener(table, "sort", function(e) { + sortInfoAscending = (sortInfoColumn == e.column)? !sortInfoAscending : false; + sortInfoColumn = e.column; + activitySetup.sortColumn = sortInfoColumn; + activitySetup.sortAscending = sortInfoAscending; + table.draw(data, activitySetup); + }); + + table.draw(data, activitySetup); + + activityTable = table; + activityData = data; + activityDataReset = data.clone(); + + } + + $.getJSON("/biz_stats.json.daily", {}, function(data) { + renderActivationAARRR(data.activations); + renderRetentionAARRR(data.retentions); + + $.getJSON("/biz_stats.json", {}, function(data) { + rawJsonData = data; + $("div#activityStats") + .append(revenue(data)) + .append(renderAARRR(data)) + .append(activity(data)) + ; + renderAccountList(data.accountList); + renderIndexList(data.indexList); + renderAccountHistoryList(data.accountList); + // + renderAdoptionAARRR(data); + renderAccountHistory(data.accountHistory); + renderGlobalHistory(data.getHistory, data.putHistory); + //renderSubscriptionList(data.accountSubscriptionList); + renderPaymentList(data.accountPaymentList); + renderQosList(data.accountGetHistoryHourly, data.accountEmail); + renderQos95List(data.account95HistoryMonthly, data.accountEmail); + renderQosErrorList(data.accountErrorHistoryDaily, data.accountEmail); + $("div#revenue").append(revenue(data)); + hide("activity"); + hide("accounts"); + hide("accountsHistory"); + hide("subscriptions"); + hide("history"); + hide("globalHistory"); + hide("indexes"); + hide("qos"); + hide("activationAARRR"); + show("activity"); + }); + + }); + + $.ajax({ + url: "http://tweets-cron.appspot.com/json", + dataType: "jsonp", + success: function(data) { + renderTwitterHistory(data.tweets); + hide("tweets"); + } + }); + + }); // $ fun +}); // g callback + +function show(id) { + document.getElementById(id).style.display="block"; +} + +function hide(id) { + document.getElementById(id).style.display="none"; + document.getElementById(id).style.visibility="visible"; +} + +function hideAll() { + hide("activationAARRR"); + hide("tweets"); + hide("subscriptions"); + hide("accountsHistory"); + hide("accounts"); + hide("activity"); + hide("history"); + hide("qos"); + hide("globalHistory"); + hide("indexes"); +} + +function hideAllAndShow(id) { + hideAll(); + show(id); +} diff --git a/backoffice/static/default.css b/backoffice/static/default.css new file mode 100644 index 0000000..df01a7f --- /dev/null +++ b/backoffice/static/default.css @@ -0,0 +1,193 @@ +.btn { display: block; position: relative; background: #ef6a31; padding: 5px; color: #fff; text-decoration: none; cursor: pointer; text-align: center; line-height: normal;} +.btn i,.btn span { font-style: normal; background-image: url(images/btn3.png); background-repeat: no-repeat; display: block; position: relative; } +.btn i { background-position: top left; position: absolute; margin-bottom: -5px; top: 0; left: 0; width: 5px; height: 5px; } +.btn span { background-position: bottom left; left: -5px; padding: 0 0 5px 10px; margin-bottom: -5px; } +.btn span i { background-position: bottom right; margin-bottom: 0; position: absolute; left: 100%; width: 10px; height: 100%; top: 0; } +.btn span span { background-position: top right; position: absolute; right: -10px; margin-left: 10px; top: -5px; height: 0; } +.btn.blue { background: #2ae; } +.btn.green { background: #9d4; } +.btn.disabled { background-color: silver; cursor: default; } +.btn.disabled:hover { background-color: silver; } +.btn.green2 { background-color: #88bf1c; } +.btn.pink { background: #e1a; } +.btn.black { background: #333; } +.btn.red { background: #a00; } +.btn.red:hover { background-color: #aaa; } +.btn.yellow { background-color: #fa0; } +.btn.white { background-color: #fff; color: #777; } +.btn:hover { background-color: #a00; text-decoration: none; } +.btn:active { background-color: #333; } +.btn[class] { background-image: url(images/shade.png); background-position: bottom; } + +.superbox { + border: solid 2px #ccc; + padding: 30px; + background: white; + -webkit-border-radius: 25px; + -webkit-box-shadow: 0 0 10px silver; + -moz-border-radius: 25px; + -moz-box-shadow: 0 0 10px silver; +} +.superbox.red { + background: #FFF8F4; + border-color: white; + -webkit-box-shadow: 0 0 5px #CA9; + -moz-box-shadow: 0 0 5px #CA9; +} +.superbox.green { + background: #F4FFF4; + border-color: white; + -webkit-box-shadow: 0 0 5px #9C9; + -moz-box-shadow: 0 0 5px #9C9; +} +.superbox h1 { + font-weight: bold !important; + font-size: 30px; + text-align: center; + text-shadow: silver 2px 2px 2px; + color: #333; +} +.superbox h1 .highlight { + color: #ef6a31; +} + +.roundbox { + line-height: normal; + border: solid 1px silver; + padding: 8px 14px; + -moz-border-radius: 20px; + -webkit-border-radius: 20px; + background-color: white; + color: gray; +} + +.roundbox.orange { + border: none; + color: white; + background-color: #ef6a31; +} +.roundbox.black { + border: none; + color: white; + background-color: black; +} +.roundbox.gold { + border: none; + color: white; + background-color: #DA0; +} +.roundbox.green { + border: none; + color: white; + background-color: green; +} +.roundbox.gray { + border: none; + color: white; + background-color: gray; +} +.roundbox.silver { + border: none; + color: white; + background-color: silver; +} +.roundbox.topright { + margin-right: -14px; margin-top: -8px; + float: right; + -webkit-border-top-left-radius: 0px; + -webkit-border-bottom-right-radius: 0px; + -moz-border-radius-topleft: 0px; + -moz-border-radius-bottomright: 0px; + padding-top: 6px; +} +.roundbox.topleft { + margin-left: -14px; margin-top: -8px; + float: left; + -webkit-border-top-right-radius: 0px; + -webkit-border-bottom-left-radius: 0px; + -moz-border-radius-topright: 0px; + -moz-border-radius-bottomleft: 0px; + padding-top: 6px; +} +.roundbox big { + font-weight: bold; +} + +.apitable { + border: solid 2px gray; + border-collapse: collapse; + line-height: normal; +} +.apitable td, .apitable th { + border: solid 1px gray; + border-collapse: collapse; + padding: 5px; + font-size: 12px; + vertical-align: top; +} +.apitable th { + background: #ef6a31; + border: none; + border-bottom: solid 2px gray; + color: white; +} + +.form .field { +} +.form .field .label { + font-size: 20px; + padding: 3px 10px 10px 0px; + width: 180px; + text-align: right; + vertical-align: top; +} +.form .field .input { + margin-top: 15px; + margin-left: 150px; + width: 200px; + vertical-align: top; +} + +.form .field .input input[type=text], .form .field .input input[type=password] { + width: 180px; + border: solid 1px silver; + -moz-border-radius-bottomleft: 3px; + -moz-border-radius-bottomright: 3px; + -moz-border-radius-topleft: 3px; + -moz-border-radius-topright: 3px; + -webkit-border-radius: 3px; + padding: 5px; + font-size: 16px; +} +.form .field .input textarea { + width: 180px; + height: 60px; +} +.form .field .input.large280 { + width: 280px; +} +.form .field .input.functionname { + width: 40px; + margin-left: 10px; +} + +.form .field .input.small40 input[type=text] { + width: 30px; +} + +.form .field .input.small400 { + margin-left: 20px; + width: 280px; +} + +.form .field .input.small400 input[type=text] { + width: 270px; +} +.form .field .input.large280 input[type=text], .form .field .input.large280 input[type=password], .form .field .input.large280 textarea { + width: 260px; +} + +.form .errors { + color: red; + vertical-align: top; +} diff --git a/backoffice/static/images/accept.png b/backoffice/static/images/accept.png new file mode 100644 index 0000000..4723b92 Binary files /dev/null and b/backoffice/static/images/accept.png differ diff --git a/backoffice/static/images/btn3.png b/backoffice/static/images/btn3.png new file mode 100644 index 0000000..3647454 Binary files /dev/null and b/backoffice/static/images/btn3.png differ diff --git a/backoffice/static/images/clarin.gif b/backoffice/static/images/clarin.gif new file mode 100644 index 0000000..23ed3fb Binary files /dev/null and b/backoffice/static/images/clarin.gif differ diff --git a/backoffice/static/images/clarin.png b/backoffice/static/images/clarin.png new file mode 100644 index 0000000..6957f6a Binary files /dev/null and b/backoffice/static/images/clarin.png differ diff --git a/backoffice/static/images/clock.png b/backoffice/static/images/clock.png new file mode 100644 index 0000000..1e087e6 Binary files /dev/null and b/backoffice/static/images/clock.png differ diff --git a/backoffice/static/images/clock_2.png b/backoffice/static/images/clock_2.png new file mode 100644 index 0000000..055abf8 Binary files /dev/null and b/backoffice/static/images/clock_2.png differ diff --git a/backoffice/static/images/database_add.png b/backoffice/static/images/database_add.png new file mode 100644 index 0000000..878699b Binary files /dev/null and b/backoffice/static/images/database_add.png differ diff --git a/backoffice/static/images/database_add_2.png b/backoffice/static/images/database_add_2.png new file mode 100644 index 0000000..65000f1 Binary files /dev/null and b/backoffice/static/images/database_add_2.png differ diff --git a/backoffice/static/images/database_search.png b/backoffice/static/images/database_search.png new file mode 100644 index 0000000..26c1f2f Binary files /dev/null and b/backoffice/static/images/database_search.png differ diff --git a/backoffice/static/images/database_search_2.png b/backoffice/static/images/database_search_2.png new file mode 100644 index 0000000..54b114a Binary files /dev/null and b/backoffice/static/images/database_search_2.png differ diff --git a/backoffice/static/images/dollar_currency_sign.png b/backoffice/static/images/dollar_currency_sign.png new file mode 100644 index 0000000..85cbcab Binary files /dev/null and b/backoffice/static/images/dollar_currency_sign.png differ diff --git a/backoffice/static/images/fastwait.gif b/backoffice/static/images/fastwait.gif new file mode 100644 index 0000000..e89f2bb Binary files /dev/null and b/backoffice/static/images/fastwait.gif differ diff --git a/backoffice/static/images/graph_up.png b/backoffice/static/images/graph_up.png new file mode 100644 index 0000000..8bd08b4 Binary files /dev/null and b/backoffice/static/images/graph_up.png differ diff --git a/backoffice/static/images/howitworks.png b/backoffice/static/images/howitworks.png new file mode 100644 index 0000000..33736fa Binary files /dev/null and b/backoffice/static/images/howitworks.png differ diff --git a/backoffice/static/images/img03.gif b/backoffice/static/images/img03.gif new file mode 100644 index 0000000..2bc7a72 Binary files /dev/null and b/backoffice/static/images/img03.gif differ diff --git a/backoffice/static/images/indextank copy.png b/backoffice/static/images/indextank copy.png new file mode 100644 index 0000000..687e8d2 Binary files /dev/null and b/backoffice/static/images/indextank copy.png differ diff --git a/backoffice/static/images/indextank.png b/backoffice/static/images/indextank.png new file mode 100644 index 0000000..4abddfc Binary files /dev/null and b/backoffice/static/images/indextank.png differ diff --git a/backoffice/static/images/loading.gif b/backoffice/static/images/loading.gif new file mode 100644 index 0000000..c40378c Binary files /dev/null and b/backoffice/static/images/loading.gif differ diff --git a/backoffice/static/images/logo.png b/backoffice/static/images/logo.png new file mode 100644 index 0000000..5aafc83 Binary files /dev/null and b/backoffice/static/images/logo.png differ diff --git a/backoffice/static/images/male_female_users.png b/backoffice/static/images/male_female_users.png new file mode 100644 index 0000000..2b1f9cf Binary files /dev/null and b/backoffice/static/images/male_female_users.png differ diff --git a/backoffice/static/images/mylife.gif b/backoffice/static/images/mylife.gif new file mode 100644 index 0000000..c98fc0f Binary files /dev/null and b/backoffice/static/images/mylife.gif differ diff --git a/backoffice/static/images/mylife.png b/backoffice/static/images/mylife.png new file mode 100644 index 0000000..2c18588 Binary files /dev/null and b/backoffice/static/images/mylife.png differ diff --git a/backoffice/static/images/process.png b/backoffice/static/images/process.png new file mode 100644 index 0000000..72acb87 Binary files /dev/null and b/backoffice/static/images/process.png differ diff --git a/backoffice/static/images/process_2.png b/backoffice/static/images/process_2.png new file mode 100644 index 0000000..2b902fd Binary files /dev/null and b/backoffice/static/images/process_2.png differ diff --git a/backoffice/static/images/refresh.png b/backoffice/static/images/refresh.png new file mode 100644 index 0000000..b57cf87 Binary files /dev/null and b/backoffice/static/images/refresh.png differ diff --git a/backoffice/static/images/refreshing.gif b/backoffice/static/images/refreshing.gif new file mode 100644 index 0000000..5e03b9f Binary files /dev/null and b/backoffice/static/images/refreshing.gif differ diff --git a/backoffice/static/images/security.png b/backoffice/static/images/security.png new file mode 100644 index 0000000..ffc9316 Binary files /dev/null and b/backoffice/static/images/security.png differ diff --git a/backoffice/static/images/tableheader.png b/backoffice/static/images/tableheader.png new file mode 100644 index 0000000..3a66423 Binary files /dev/null and b/backoffice/static/images/tableheader.png differ diff --git a/backoffice/static/images/target.png b/backoffice/static/images/target.png new file mode 100644 index 0000000..5304f62 Binary files /dev/null and b/backoffice/static/images/target.png differ diff --git a/backoffice/static/images/tools.png b/backoffice/static/images/tools.png new file mode 100644 index 0000000..51c1201 Binary files /dev/null and b/backoffice/static/images/tools.png differ diff --git a/backoffice/static/images/user_add.png b/backoffice/static/images/user_add.png new file mode 100644 index 0000000..a14ab2f Binary files /dev/null and b/backoffice/static/images/user_add.png differ diff --git a/backoffice/static/images/user_add_2.png b/backoffice/static/images/user_add_2.png new file mode 100644 index 0000000..98fec97 Binary files /dev/null and b/backoffice/static/images/user_add_2.png differ diff --git a/backoffice/static/images/users.png b/backoffice/static/images/users.png new file mode 100644 index 0000000..cfc89c8 Binary files /dev/null and b/backoffice/static/images/users.png differ diff --git a/backoffice/static/images/users_2.png b/backoffice/static/images/users_2.png new file mode 100644 index 0000000..e9fe1b0 Binary files /dev/null and b/backoffice/static/images/users_2.png differ diff --git a/backoffice/static/images/wordpress.png b/backoffice/static/images/wordpress.png new file mode 100644 index 0000000..652f308 Binary files /dev/null and b/backoffice/static/images/wordpress.png differ diff --git a/backoffice/static/images/zip_file_download.png b/backoffice/static/images/zip_file_download.png new file mode 100644 index 0000000..2821075 Binary files /dev/null and b/backoffice/static/images/zip_file_download.png differ diff --git a/backoffice/static/images/zip_file_download_32.png b/backoffice/static/images/zip_file_download_32.png new file mode 100644 index 0000000..f2fa108 Binary files /dev/null and b/backoffice/static/images/zip_file_download_32.png differ diff --git a/backoffice/static/img/icons/go.png b/backoffice/static/img/icons/go.png new file mode 100644 index 0000000..8b80114 Binary files /dev/null and b/backoffice/static/img/icons/go.png differ diff --git a/backoffice/static/img/icons/x_clock.png b/backoffice/static/img/icons/x_clock.png new file mode 100644 index 0000000..95b80b2 Binary files /dev/null and b/backoffice/static/img/icons/x_clock.png differ diff --git a/backoffice/static/img/icons/x_instant.png b/backoffice/static/img/icons/x_instant.png new file mode 100644 index 0000000..618fd6a Binary files /dev/null and b/backoffice/static/img/icons/x_instant.png differ diff --git a/backoffice/static/img/icons/x_key.png b/backoffice/static/img/icons/x_key.png new file mode 100644 index 0000000..b668c0f Binary files /dev/null and b/backoffice/static/img/icons/x_key.png differ diff --git a/backoffice/static/img/icons/x_ok.png b/backoffice/static/img/icons/x_ok.png new file mode 100644 index 0000000..0b01755 Binary files /dev/null and b/backoffice/static/img/icons/x_ok.png differ diff --git a/backoffice/static/img/icons/x_speed.png b/backoffice/static/img/icons/x_speed.png new file mode 100644 index 0000000..9d56797 Binary files /dev/null and b/backoffice/static/img/icons/x_speed.png differ diff --git a/backoffice/static/img/pictures/checkbox_switch.png b/backoffice/static/img/pictures/checkbox_switch.png new file mode 100644 index 0000000..0bf6eec Binary files /dev/null and b/backoffice/static/img/pictures/checkbox_switch.png differ diff --git a/backoffice/static/img/slide_show/slide_01.jpg b/backoffice/static/img/slide_show/slide_01.jpg new file mode 100644 index 0000000..20178cc Binary files /dev/null and b/backoffice/static/img/slide_show/slide_01.jpg differ diff --git a/backoffice/static/img/slide_show/slide_02.jpg b/backoffice/static/img/slide_show/slide_02.jpg new file mode 100644 index 0000000..652ce2a Binary files /dev/null and b/backoffice/static/img/slide_show/slide_02.jpg differ diff --git a/backoffice/static/img/slide_show/slide_03.jpg b/backoffice/static/img/slide_show/slide_03.jpg new file mode 100644 index 0000000..ce0d4e5 Binary files /dev/null and b/backoffice/static/img/slide_show/slide_03.jpg differ diff --git a/backoffice/static/img/slide_show/slide_1.psd b/backoffice/static/img/slide_show/slide_1.psd new file mode 100644 index 0000000..718429c Binary files /dev/null and b/backoffice/static/img/slide_show/slide_1.psd differ diff --git a/backoffice/static/img/slide_show/slide_2.psd b/backoffice/static/img/slide_show/slide_2.psd new file mode 100644 index 0000000..241d698 Binary files /dev/null and b/backoffice/static/img/slide_show/slide_2.psd differ diff --git a/backoffice/static/img/slide_show/slide_3.psd b/backoffice/static/img/slide_show/slide_3.psd new file mode 100644 index 0000000..b396e02 Binary files /dev/null and b/backoffice/static/img/slide_show/slide_3.psd differ diff --git a/backoffice/static/img/slide_show/slide_x.psd b/backoffice/static/img/slide_show/slide_x.psd new file mode 100644 index 0000000..820209f Binary files /dev/null and b/backoffice/static/img/slide_show/slide_x.psd differ diff --git a/backoffice/static/img/structure/ballon.gif b/backoffice/static/img/structure/ballon.gif new file mode 100644 index 0000000..044f47d Binary files /dev/null and b/backoffice/static/img/structure/ballon.gif differ diff --git a/backoffice/static/img/structure/bg.jpg b/backoffice/static/img/structure/bg.jpg new file mode 100644 index 0000000..73694a0 Binary files /dev/null and b/backoffice/static/img/structure/bg.jpg differ diff --git a/backoffice/static/img/structure/bg.jpg.b b/backoffice/static/img/structure/bg.jpg.b new file mode 100644 index 0000000..20b7825 Binary files /dev/null and b/backoffice/static/img/structure/bg.jpg.b differ diff --git a/backoffice/static/img/structure/bg.png b/backoffice/static/img/structure/bg.png new file mode 100644 index 0000000..5963485 Binary files /dev/null and b/backoffice/static/img/structure/bg.png differ diff --git a/backoffice/static/img/structure/bg_footer.gif b/backoffice/static/img/structure/bg_footer.gif new file mode 100644 index 0000000..d81a04c Binary files /dev/null and b/backoffice/static/img/structure/bg_footer.gif differ diff --git a/backoffice/static/img/structure/bg_footer.png b/backoffice/static/img/structure/bg_footer.png new file mode 100644 index 0000000..c6c54d5 Binary files /dev/null and b/backoffice/static/img/structure/bg_footer.png differ diff --git a/backoffice/static/img/structure/footer.png b/backoffice/static/img/structure/footer.png new file mode 100644 index 0000000..2286fc3 Binary files /dev/null and b/backoffice/static/img/structure/footer.png differ diff --git a/backoffice/static/img/structure/header.png b/backoffice/static/img/structure/header.png new file mode 100644 index 0000000..5d2d236 Binary files /dev/null and b/backoffice/static/img/structure/header.png differ diff --git a/backoffice/static/img/structure/logo.png b/backoffice/static/img/structure/logo.png new file mode 100644 index 0000000..53a4b96 Binary files /dev/null and b/backoffice/static/img/structure/logo.png differ diff --git a/backoffice/static/img/structure/none.png b/backoffice/static/img/structure/none.png new file mode 100644 index 0000000..36ece78 Binary files /dev/null and b/backoffice/static/img/structure/none.png differ diff --git a/backoffice/static/img/structure/ok.png b/backoffice/static/img/structure/ok.png new file mode 100644 index 0000000..67d2751 Binary files /dev/null and b/backoffice/static/img/structure/ok.png differ diff --git a/backoffice/static/img/structure/shadow-bottom.png b/backoffice/static/img/structure/shadow-bottom.png new file mode 100644 index 0000000..050e4df Binary files /dev/null and b/backoffice/static/img/structure/shadow-bottom.png differ diff --git a/backoffice/static/img/structure/shadow.png b/backoffice/static/img/structure/shadow.png new file mode 100644 index 0000000..b05603d Binary files /dev/null and b/backoffice/static/img/structure/shadow.png differ diff --git a/backoffice/static/img/structure/try_it.gif b/backoffice/static/img/structure/try_it.gif new file mode 100644 index 0000000..86847c1 Binary files /dev/null and b/backoffice/static/img/structure/try_it.gif differ diff --git a/backoffice/static/img/testimonials/user.jpg b/backoffice/static/img/testimonials/user.jpg new file mode 100644 index 0000000..dfc7731 Binary files /dev/null and b/backoffice/static/img/testimonials/user.jpg differ diff --git a/backoffice/static/img/testimonials/user.png b/backoffice/static/img/testimonials/user.png new file mode 100644 index 0000000..e89b35c Binary files /dev/null and b/backoffice/static/img/testimonials/user.png differ diff --git a/backoffice/static/img/trustedby/clarin.png b/backoffice/static/img/trustedby/clarin.png new file mode 100644 index 0000000..0533b8c Binary files /dev/null and b/backoffice/static/img/trustedby/clarin.png differ diff --git a/backoffice/static/img/trustedby/mylife.png b/backoffice/static/img/trustedby/mylife.png new file mode 100644 index 0000000..cd23998 Binary files /dev/null and b/backoffice/static/img/trustedby/mylife.png differ diff --git a/backoffice/static/img/trustedby/wordpress.png b/backoffice/static/img/trustedby/wordpress.png new file mode 100644 index 0000000..6b44554 Binary files /dev/null and b/backoffice/static/img/trustedby/wordpress.png differ diff --git a/backoffice/static/indextank-java-client.tar.gz b/backoffice/static/indextank-java-client.tar.gz new file mode 100644 index 0000000..75e09f4 Binary files /dev/null and b/backoffice/static/indextank-java-client.tar.gz differ diff --git a/backoffice/static/indextank-java-client.zip b/backoffice/static/indextank-java-client.zip new file mode 100644 index 0000000..24c747d Binary files /dev/null and b/backoffice/static/indextank-java-client.zip differ diff --git a/backoffice/static/indextank-php-client.gz b/backoffice/static/indextank-php-client.gz new file mode 100644 index 0000000..1aba20b Binary files /dev/null and b/backoffice/static/indextank-php-client.gz differ diff --git a/backoffice/static/indextank-php-client.zip b/backoffice/static/indextank-php-client.zip new file mode 100644 index 0000000..277d119 Binary files /dev/null and b/backoffice/static/indextank-php-client.zip differ diff --git a/backoffice/static/indextank-python-client.gz b/backoffice/static/indextank-python-client.gz new file mode 100644 index 0000000..4599300 Binary files /dev/null and b/backoffice/static/indextank-python-client.gz differ diff --git a/backoffice/static/indextank-python-client.zip b/backoffice/static/indextank-python-client.zip new file mode 100644 index 0000000..460a80e Binary files /dev/null and b/backoffice/static/indextank-python-client.zip differ diff --git a/backoffice/static/indextank-ruby-client.gz b/backoffice/static/indextank-ruby-client.gz new file mode 100644 index 0000000..8a415e8 Binary files /dev/null and b/backoffice/static/indextank-ruby-client.gz differ diff --git a/backoffice/static/indextank-ruby-client.zip b/backoffice/static/indextank-ruby-client.zip new file mode 100644 index 0000000..2f6fb0a Binary files /dev/null and b/backoffice/static/indextank-ruby-client.zip differ diff --git a/backoffice/static/indextank.py.gz b/backoffice/static/indextank.py.gz new file mode 100644 index 0000000..047ea06 Binary files /dev/null and b/backoffice/static/indextank.py.gz differ diff --git a/backoffice/static/indextank.rb.gz b/backoffice/static/indextank.rb.gz new file mode 100644 index 0000000..4420624 Binary files /dev/null and b/backoffice/static/indextank.rb.gz differ diff --git a/backoffice/static/indextank.wordpress.tar.gz b/backoffice/static/indextank.wordpress.tar.gz new file mode 100644 index 0000000..937775f Binary files /dev/null and b/backoffice/static/indextank.wordpress.tar.gz differ diff --git a/backoffice/static/jquery.autocomplete.js b/backoffice/static/jquery.autocomplete.js new file mode 100644 index 0000000..e0a8826 --- /dev/null +++ b/backoffice/static/jquery.autocomplete.js @@ -0,0 +1,398 @@ +/** +* Ajax Autocomplete for jQuery, version 1.1.3 +* (c) 2010 Tomas Kirda +* +* Ajax Autocomplete for jQuery is freely distributable under the terms of an MIT-style license. +* For details, see the web site: http://www.devbridge.com/projects/autocomplete/jquery/ +* +* Last Review: 04/19/2010 +*/ + +/*jslint onevar: true, evil: true, nomen: true, eqeqeq: true, bitwise: true, regexp: true, newcap: true, immed: true */ +/*global window: true, document: true, clearInterval: true, setInterval: true, jQuery: true */ + +(function($) { + + var reEscape = new RegExp('(\\' + ['/', '.', '*', '+', '?', '|', '(', ')', '[', ']', '{', '}', '\\'].join('|\\') + ')', 'g'); + + function fnFormatResult(value, data, currentValue) { + var pattern = '(' + currentValue.replace(reEscape, '\\$1') + ')'; + return value.replace(new RegExp(pattern, 'gi'), '$1<\/strong>'); + } + + function Autocomplete(el, options) { + this.el = $(el); + this.el.attr('autocomplete', 'off'); + this.suggestions = []; + this.data = []; + this.badQueries = []; + this.selectedIndex = -1; + this.currentValue = this.el.val(); + this.intervalId = 0; + this.cachedResponse = []; + this.onChangeInterval = null; + this.ignoreValueChange = false; + this.serviceUrl = options.serviceUrl; + this.isLocal = false; + this.options = { + autoSubmit: false, + minChars: 1, + maxHeight: 300, + deferRequestBy: 0, + width: 0, + highlight: true, + params: {}, + fnFormatResult: fnFormatResult, + delimiter: null, + zIndex: 9999 + }; + this.initialize(); + this.setOptions(options); + } + + $.fn.autocomplete = function(options) { + return new Autocomplete(this.get(0)||$(''), options); + }; + + + Autocomplete.prototype = { + + killerFn: null, + + initialize: function() { + + var me, uid, autocompleteElId; + me = this; + uid = Math.floor(Math.random()*0x100000).toString(16); + autocompleteElId = 'Autocomplete_' + uid; + + this.killerFn = function(e) { + if ($(e.target).parents('.autocomplete').size() === 0) { + me.killSuggestions(); + me.disableKillerFn(); + } + }; + + if (!this.options.width) { this.options.width = this.el.width(); } + this.mainContainerId = 'AutocompleteContainter_' + uid; + + $('
').appendTo('body'); + + this.container = $('#' + autocompleteElId); + this.fixPosition(); + if (window.opera) { + this.el.keypress(function(e) { me.onKeyPress(e); }); + } else { + this.el.keydown(function(e) { me.onKeyPress(e); }); + } + this.el.keyup(function(e) { me.onKeyUp(e); }); + this.el.blur(function() { me.enableKillerFn(); }); + this.el.focus(function() { me.fixPosition(); }); + }, + + setOptions: function(options){ + var o = this.options; + $.extend(o, options); + if(o.lookup){ + this.isLocal = true; + if($.isArray(o.lookup)){ o.lookup = { suggestions:o.lookup, data:[] }; } + } + $('#'+this.mainContainerId).css({ zIndex:o.zIndex }); + this.container.css({ maxHeight: o.maxHeight + 'px', width:o.width }); + }, + + clearCache: function(){ + this.cachedResponse = []; + this.badQueries = []; + }, + + disable: function(){ + this.disabled = true; + }, + + enable: function(){ + this.disabled = false; + }, + + fixPosition: function() { + var offset = this.el.offset(); + $('#' + this.mainContainerId).css({ top: (offset.top + this.el.innerHeight()) + 'px', left: offset.left + 'px' }); + }, + + enableKillerFn: function() { + var me = this; + $(document).bind('click', me.killerFn); + }, + + disableKillerFn: function() { + var me = this; + $(document).unbind('click', me.killerFn); + }, + + killSuggestions: function() { + var me = this; + this.stopKillSuggestions(); + this.intervalId = window.setInterval(function() { me.hide(); me.stopKillSuggestions(); }, 300); + }, + + stopKillSuggestions: function() { + window.clearInterval(this.intervalId); + }, + + onKeyPress: function(e) { + if (this.disabled || !this.enabled) { return; } + // return will exit the function + // and event will not be prevented + switch (e.keyCode) { + case 27: //KEY_ESC: + this.el.val(this.currentValue); + this.hide(); + break; + case 9: //KEY_TAB: + case 13: //KEY_RETURN: + if (this.selectedIndex === -1) { + this.hide(); + return; + } + this.select(this.selectedIndex); + if(e.keyCode === 9){ return; } + if(e.keyCode === 13){ this.el.parents("form").submit(); } // ADDED BY DBUTHAY + break; + case 38: //KEY_UP: + this.moveUp(); + break; + case 40: //KEY_DOWN: + this.moveDown(); + break; + default: + return; + } + e.stopImmediatePropagation(); + e.preventDefault(); + }, + + onKeyUp: function(e) { + if(this.disabled){ return; } + switch (e.keyCode) { + case 38: //KEY_UP: + case 40: //KEY_DOWN: + return; + } + clearInterval(this.onChangeInterval); + if (this.currentValue !== this.el.val()) { + if (this.options.deferRequestBy > 0) { + // Defer lookup in case when value changes very quickly: + var me = this; + this.onChangeInterval = setInterval(function() { me.onValueChange(); }, this.options.deferRequestBy); + } else { + this.onValueChange(); + } + } + }, + + onValueChange: function() { + clearInterval(this.onChangeInterval); + this.currentValue = this.el.val(); + var q = this.getQuery(this.currentValue); + this.selectedIndex = -1; + if (this.ignoreValueChange) { + this.ignoreValueChange = false; + return; + } + if (q === '' || q.length < this.options.minChars) { + this.hide(); + } else { + this.getSuggestions(q); + } + }, + + getQuery: function(val) { + var d, arr; + d = this.options.delimiter; + if (!d) { return $.trim(val); } + arr = val.split(d); + return $.trim(arr[arr.length - 1]); + }, + + getSuggestionsLocal: function(q) { + var ret, arr, len, val, i; + arr = this.options.lookup; + len = arr.suggestions.length; + ret = { suggestions:[], data:[] }; + q = q.toLowerCase(); + for(i=0; i< len; i++){ + val = arr.suggestions[i]; + if(val.toLowerCase().indexOf(q) === 0){ + ret.suggestions.push(val); + ret.data.push(arr.data[i]); + } + } + return ret; + }, + + getSuggestions: function(q) { + var cr, me; + cr = this.isLocal ? this.getSuggestionsLocal(q) : this.cachedResponse[q]; + if (cr && $.isArray(cr.suggestions)) { + this.suggestions = cr.suggestions; + this.data = cr.data; + this.suggest(); + } else if (!this.isBadQuery(q)) { + me = this; + me.options.params.query = q; + $.ajax( { + url: this.serviceUrl, + data: me.options.params, + success: function(txt) { me.processResponse(txt); }, + dataType: 'jsonp' + + } ); + } + }, + + isBadQuery: function(q) { + var i = this.badQueries.length; + while (i--) { + if (q.indexOf(this.badQueries[i]) === 0) { return true; } + } + return false; + }, + + hide: function() { + this.enabled = false; + this.selectedIndex = -1; + this.container.hide(); + }, + + suggest: function() { + if (this.suggestions.length === 0) { + this.hide(); + return; + } + + var me, len, div, f, v, i, s, mOver, mClick; + me = this; + len = this.suggestions.length; + f = this.options.fnFormatResult; + v = this.getQuery(this.currentValue); + mOver = function(xi) { return function() { me.activate(xi); }; }; + mClick = function(xi) { return function() { me.select(xi); }; }; + this.container.hide().empty(); + for (i = 0; i < len; i++) { + s = this.suggestions[i]; + div = $((me.selectedIndex === i ? '
' + f(s, this.data[i], v) + '
'); + div.mouseover(mOver(i)); + div.click(mClick(i)); + this.container.append(div); + } + this.enabled = true; + this.container.show(); + }, + + processResponse: function(text) { + var response; + try { +// response = eval('(' + text + ')'); + response = text; // HACK + } catch (err) { return; } + if (!$.isArray(response.data)) { response.data = []; } + if(!this.options.noCache){ + this.cachedResponse[response.query] = response; + if (response.suggestions.length === 0) { this.badQueries.push(response.query); } + } + if (response.query === this.getQuery(this.currentValue)) { + this.suggestions = response.suggestions; + this.data = response.data; + this.suggest(); + } + }, + + activate: function(index) { + var divs, activeItem; + divs = this.container.children(); + // Clear previous selection: + if (this.selectedIndex !== -1 && divs.length > this.selectedIndex) { + $(divs.get(this.selectedIndex)).removeClass(); + } + this.selectedIndex = index; + if (this.selectedIndex !== -1 && divs.length > this.selectedIndex) { + activeItem = divs.get(this.selectedIndex); + $(activeItem).addClass('selected'); + } + return activeItem; + }, + + deactivate: function(div, index) { + div.className = ''; + if (this.selectedIndex === index) { this.selectedIndex = -1; } + }, + + select: function(i) { + var selectedValue, f; + selectedValue = this.suggestions[i]; + if (selectedValue) { + this.el.val(selectedValue); + if (this.options.autoSubmit) { + f = this.el.parents('form'); + if (f.length > 0) { f.get(0).submit(); } + } + this.ignoreValueChange = true; + this.hide(); + this.onSelect(i); + } + }, + + moveUp: function() { + if (this.selectedIndex === -1) { return; } + if (this.selectedIndex === 0) { + this.container.children().get(0).className = ''; + this.selectedIndex = -1; + this.el.val(this.currentValue); + return; + } + this.adjustScroll(this.selectedIndex - 1); + }, + + moveDown: function() { + if (this.selectedIndex === (this.suggestions.length - 1)) { return; } + this.adjustScroll(this.selectedIndex + 1); + }, + + adjustScroll: function(i) { + var activeItem, offsetTop, upperBound, lowerBound; + activeItem = this.activate(i); + offsetTop = activeItem.offsetTop; + upperBound = this.container.scrollTop(); + lowerBound = upperBound + this.options.maxHeight - 25; + if (offsetTop < upperBound) { + this.container.scrollTop(offsetTop); + } else if (offsetTop > lowerBound) { + this.container.scrollTop(offsetTop - this.options.maxHeight + 25); + } + this.el.val(this.getValue(this.suggestions[i])); + }, + + onSelect: function(i) { + var me, fn, s, d; + me = this; + fn = me.options.onSelect; + s = me.suggestions[i]; + d = me.data[i]; + me.el.val(me.getValue(s)); + if ($.isFunction(fn)) { fn(s, d, me.el); } + }, + + getValue: function(value){ + var del, currVal, arr, me; + me = this; + del = me.options.delimiter; + if (!del) { return value; } + currVal = me.currentValue; + arr = currVal.split(del); + if (arr.length === 1) { return value; } + return currVal.substr(0, currVal.length - arr[arr.length - 1].length) + value; + } + + }; + +}(jQuery)); diff --git a/backoffice/static/js/jquery-1.3.2.min.js b/backoffice/static/js/jquery-1.3.2.min.js new file mode 100644 index 0000000..b1ae21d --- /dev/null +++ b/backoffice/static/js/jquery-1.3.2.min.js @@ -0,0 +1,19 @@ +/* + * jQuery JavaScript Library v1.3.2 + * http://jquery.com/ + * + * Copyright (c) 2009 John Resig + * Dual licensed under the MIT and GPL licenses. + * http://docs.jquery.com/License + * + * Date: 2009-02-19 17:34:21 -0500 (Thu, 19 Feb 2009) + * Revision: 6246 + */ +(function(){var l=this,g,y=l.jQuery,p=l.$,o=l.jQuery=l.$=function(E,F){return new o.fn.init(E,F)},D=/^[^<]*(<(.|\s)+>)[^>]*$|^#([\w-]+)$/,f=/^.[^:#\[\.,]*$/;o.fn=o.prototype={init:function(E,H){E=E||document;if(E.nodeType){this[0]=E;this.length=1;this.context=E;return this}if(typeof E==="string"){var G=D.exec(E);if(G&&(G[1]||!H)){if(G[1]){E=o.clean([G[1]],H)}else{var I=document.getElementById(G[3]);if(I&&I.id!=G[3]){return o().find(E)}var F=o(I||[]);F.context=document;F.selector=E;return F}}else{return o(H).find(E)}}else{if(o.isFunction(E)){return o(document).ready(E)}}if(E.selector&&E.context){this.selector=E.selector;this.context=E.context}return this.setArray(o.isArray(E)?E:o.makeArray(E))},selector:"",jquery:"1.3.2",size:function(){return this.length},get:function(E){return E===g?Array.prototype.slice.call(this):this[E]},pushStack:function(F,H,E){var G=o(F);G.prevObject=this;G.context=this.context;if(H==="find"){G.selector=this.selector+(this.selector?" ":"")+E}else{if(H){G.selector=this.selector+"."+H+"("+E+")"}}return G},setArray:function(E){this.length=0;Array.prototype.push.apply(this,E);return this},each:function(F,E){return o.each(this,F,E)},index:function(E){return o.inArray(E&&E.jquery?E[0]:E,this)},attr:function(F,H,G){var E=F;if(typeof F==="string"){if(H===g){return this[0]&&o[G||"attr"](this[0],F)}else{E={};E[F]=H}}return this.each(function(I){for(F in E){o.attr(G?this.style:this,F,o.prop(this,E[F],G,I,F))}})},css:function(E,F){if((E=="width"||E=="height")&&parseFloat(F)<0){F=g}return this.attr(E,F,"curCSS")},text:function(F){if(typeof F!=="object"&&F!=null){return this.empty().append((this[0]&&this[0].ownerDocument||document).createTextNode(F))}var E="";o.each(F||this,function(){o.each(this.childNodes,function(){if(this.nodeType!=8){E+=this.nodeType!=1?this.nodeValue:o.fn.text([this])}})});return E},wrapAll:function(E){if(this[0]){var F=o(E,this[0].ownerDocument).clone();if(this[0].parentNode){F.insertBefore(this[0])}F.map(function(){var G=this;while(G.firstChild){G=G.firstChild}return G}).append(this)}return this},wrapInner:function(E){return this.each(function(){o(this).contents().wrapAll(E)})},wrap:function(E){return this.each(function(){o(this).wrapAll(E)})},append:function(){return this.domManip(arguments,true,function(E){if(this.nodeType==1){this.appendChild(E)}})},prepend:function(){return this.domManip(arguments,true,function(E){if(this.nodeType==1){this.insertBefore(E,this.firstChild)}})},before:function(){return this.domManip(arguments,false,function(E){this.parentNode.insertBefore(E,this)})},after:function(){return this.domManip(arguments,false,function(E){this.parentNode.insertBefore(E,this.nextSibling)})},end:function(){return this.prevObject||o([])},push:[].push,sort:[].sort,splice:[].splice,find:function(E){if(this.length===1){var F=this.pushStack([],"find",E);F.length=0;o.find(E,this[0],F);return F}else{return this.pushStack(o.unique(o.map(this,function(G){return o.find(E,G)})),"find",E)}},clone:function(G){var E=this.map(function(){if(!o.support.noCloneEvent&&!o.isXMLDoc(this)){var I=this.outerHTML;if(!I){var J=this.ownerDocument.createElement("div");J.appendChild(this.cloneNode(true));I=J.innerHTML}return o.clean([I.replace(/ jQuery\d+="(?:\d+|null)"/g,"").replace(/^\s*/,"")])[0]}else{return this.cloneNode(true)}});if(G===true){var H=this.find("*").andSelf(),F=0;E.find("*").andSelf().each(function(){if(this.nodeName!==H[F].nodeName){return}var I=o.data(H[F],"events");for(var K in I){for(var J in I[K]){o.event.add(this,K,I[K][J],I[K][J].data)}}F++})}return E},filter:function(E){return this.pushStack(o.isFunction(E)&&o.grep(this,function(G,F){return E.call(G,F)})||o.multiFilter(E,o.grep(this,function(F){return F.nodeType===1})),"filter",E)},closest:function(E){var G=o.expr.match.POS.test(E)?o(E):null,F=0;return this.map(function(){var H=this;while(H&&H.ownerDocument){if(G?G.index(H)>-1:o(H).is(E)){o.data(H,"closest",F);return H}H=H.parentNode;F++}})},not:function(E){if(typeof E==="string"){if(f.test(E)){return this.pushStack(o.multiFilter(E,this,true),"not",E)}else{E=o.multiFilter(E,this)}}var F=E.length&&E[E.length-1]!==g&&!E.nodeType;return this.filter(function(){return F?o.inArray(this,E)<0:this!=E})},add:function(E){return this.pushStack(o.unique(o.merge(this.get(),typeof E==="string"?o(E):o.makeArray(E))))},is:function(E){return !!E&&o.multiFilter(E,this).length>0},hasClass:function(E){return !!E&&this.is("."+E)},val:function(K){if(K===g){var E=this[0];if(E){if(o.nodeName(E,"option")){return(E.attributes.value||{}).specified?E.value:E.text}if(o.nodeName(E,"select")){var I=E.selectedIndex,L=[],M=E.options,H=E.type=="select-one";if(I<0){return null}for(var F=H?I:0,J=H?I+1:M.length;F=0||o.inArray(this.name,K)>=0)}else{if(o.nodeName(this,"select")){var N=o.makeArray(K);o("option",this).each(function(){this.selected=(o.inArray(this.value,N)>=0||o.inArray(this.text,N)>=0)});if(!N.length){this.selectedIndex=-1}}else{this.value=K}}})},html:function(E){return E===g?(this[0]?this[0].innerHTML.replace(/ jQuery\d+="(?:\d+|null)"/g,""):null):this.empty().append(E)},replaceWith:function(E){return this.after(E).remove()},eq:function(E){return this.slice(E,+E+1)},slice:function(){return this.pushStack(Array.prototype.slice.apply(this,arguments),"slice",Array.prototype.slice.call(arguments).join(","))},map:function(E){return this.pushStack(o.map(this,function(G,F){return E.call(G,F,G)}))},andSelf:function(){return this.add(this.prevObject)},domManip:function(J,M,L){if(this[0]){var I=(this[0].ownerDocument||this[0]).createDocumentFragment(),F=o.clean(J,(this[0].ownerDocument||this[0]),I),H=I.firstChild;if(H){for(var G=0,E=this.length;G1||G>0?I.cloneNode(true):I)}}if(F){o.each(F,z)}}return this;function K(N,O){return M&&o.nodeName(N,"table")&&o.nodeName(O,"tr")?(N.getElementsByTagName("tbody")[0]||N.appendChild(N.ownerDocument.createElement("tbody"))):N}}};o.fn.init.prototype=o.fn;function z(E,F){if(F.src){o.ajax({url:F.src,async:false,dataType:"script"})}else{o.globalEval(F.text||F.textContent||F.innerHTML||"")}if(F.parentNode){F.parentNode.removeChild(F)}}function e(){return +new Date}o.extend=o.fn.extend=function(){var J=arguments[0]||{},H=1,I=arguments.length,E=false,G;if(typeof J==="boolean"){E=J;J=arguments[1]||{};H=2}if(typeof J!=="object"&&!o.isFunction(J)){J={}}if(I==H){J=this;--H}for(;H-1}},swap:function(H,G,I){var E={};for(var F in G){E[F]=H.style[F];H.style[F]=G[F]}I.call(H);for(var F in G){H.style[F]=E[F]}},css:function(H,F,J,E){if(F=="width"||F=="height"){var L,G={position:"absolute",visibility:"hidden",display:"block"},K=F=="width"?["Left","Right"]:["Top","Bottom"];function I(){L=F=="width"?H.offsetWidth:H.offsetHeight;if(E==="border"){return}o.each(K,function(){if(!E){L-=parseFloat(o.curCSS(H,"padding"+this,true))||0}if(E==="margin"){L+=parseFloat(o.curCSS(H,"margin"+this,true))||0}else{L-=parseFloat(o.curCSS(H,"border"+this+"Width",true))||0}})}if(H.offsetWidth!==0){I()}else{o.swap(H,G,I)}return Math.max(0,Math.round(L))}return o.curCSS(H,F,J)},curCSS:function(I,F,G){var L,E=I.style;if(F=="opacity"&&!o.support.opacity){L=o.attr(E,"opacity");return L==""?"1":L}if(F.match(/float/i)){F=w}if(!G&&E&&E[F]){L=E[F]}else{if(q.getComputedStyle){if(F.match(/float/i)){F="float"}F=F.replace(/([A-Z])/g,"-$1").toLowerCase();var M=q.getComputedStyle(I,null);if(M){L=M.getPropertyValue(F)}if(F=="opacity"&&L==""){L="1"}}else{if(I.currentStyle){var J=F.replace(/\-(\w)/g,function(N,O){return O.toUpperCase()});L=I.currentStyle[F]||I.currentStyle[J];if(!/^\d+(px)?$/i.test(L)&&/^\d/.test(L)){var H=E.left,K=I.runtimeStyle.left;I.runtimeStyle.left=I.currentStyle.left;E.left=L||0;L=E.pixelLeft+"px";E.left=H;I.runtimeStyle.left=K}}}}return L},clean:function(F,K,I){K=K||document;if(typeof K.createElement==="undefined"){K=K.ownerDocument||K[0]&&K[0].ownerDocument||document}if(!I&&F.length===1&&typeof F[0]==="string"){var H=/^<(\w+)\s*\/?>$/.exec(F[0]);if(H){return[K.createElement(H[1])]}}var G=[],E=[],L=K.createElement("div");o.each(F,function(P,S){if(typeof S==="number"){S+=""}if(!S){return}if(typeof S==="string"){S=S.replace(/(<(\w+)[^>]*?)\/>/g,function(U,V,T){return T.match(/^(abbr|br|col|img|input|link|meta|param|hr|area|embed)$/i)?U:V+">"});var O=S.replace(/^\s+/,"").substring(0,10).toLowerCase();var Q=!O.indexOf("",""]||!O.indexOf("",""]||O.match(/^<(thead|tbody|tfoot|colg|cap)/)&&[1,"","
"]||!O.indexOf("",""]||(!O.indexOf("",""]||!O.indexOf("",""]||!o.support.htmlSerialize&&[1,"div
","
"]||[0,"",""];L.innerHTML=Q[1]+S+Q[2];while(Q[0]--){L=L.lastChild}if(!o.support.tbody){var R=/"&&!R?L.childNodes:[];for(var M=N.length-1;M>=0;--M){if(o.nodeName(N[M],"tbody")&&!N[M].childNodes.length){N[M].parentNode.removeChild(N[M])}}}if(!o.support.leadingWhitespace&&/^\s/.test(S)){L.insertBefore(K.createTextNode(S.match(/^\s*/)[0]),L.firstChild)}S=o.makeArray(L.childNodes)}if(S.nodeType){G.push(S)}else{G=o.merge(G,S)}});if(I){for(var J=0;G[J];J++){if(o.nodeName(G[J],"script")&&(!G[J].type||G[J].type.toLowerCase()==="text/javascript")){E.push(G[J].parentNode?G[J].parentNode.removeChild(G[J]):G[J])}else{if(G[J].nodeType===1){G.splice.apply(G,[J+1,0].concat(o.makeArray(G[J].getElementsByTagName("script"))))}I.appendChild(G[J])}}return E}return G},attr:function(J,G,K){if(!J||J.nodeType==3||J.nodeType==8){return g}var H=!o.isXMLDoc(J),L=K!==g;G=H&&o.props[G]||G;if(J.tagName){var F=/href|src|style/.test(G);if(G=="selected"&&J.parentNode){J.parentNode.selectedIndex}if(G in J&&H&&!F){if(L){if(G=="type"&&o.nodeName(J,"input")&&J.parentNode){throw"type property can't be changed"}J[G]=K}if(o.nodeName(J,"form")&&J.getAttributeNode(G)){return J.getAttributeNode(G).nodeValue}if(G=="tabIndex"){var I=J.getAttributeNode("tabIndex");return I&&I.specified?I.value:J.nodeName.match(/(button|input|object|select|textarea)/i)?0:J.nodeName.match(/^(a|area)$/i)&&J.href?0:g}return J[G]}if(!o.support.style&&H&&G=="style"){return o.attr(J.style,"cssText",K)}if(L){J.setAttribute(G,""+K)}var E=!o.support.hrefNormalized&&H&&F?J.getAttribute(G,2):J.getAttribute(G);return E===null?g:E}if(!o.support.opacity&&G=="opacity"){if(L){J.zoom=1;J.filter=(J.filter||"").replace(/alpha\([^)]*\)/,"")+(parseInt(K)+""=="NaN"?"":"alpha(opacity="+K*100+")")}return J.filter&&J.filter.indexOf("opacity=")>=0?(parseFloat(J.filter.match(/opacity=([^)]*)/)[1])/100)+"":""}G=G.replace(/-([a-z])/ig,function(M,N){return N.toUpperCase()});if(L){J[G]=K}return J[G]},trim:function(E){return(E||"").replace(/^\s+|\s+$/g,"")},makeArray:function(G){var E=[];if(G!=null){var F=G.length;if(F==null||typeof G==="string"||o.isFunction(G)||G.setInterval){E[0]=G}else{while(F){E[--F]=G[F]}}}return E},inArray:function(G,H){for(var E=0,F=H.length;E0?this.clone(true):this).get();o.fn[F].apply(o(L[K]),I);J=J.concat(I)}return this.pushStack(J,E,G)}});o.each({removeAttr:function(E){o.attr(this,E,"");if(this.nodeType==1){this.removeAttribute(E)}},addClass:function(E){o.className.add(this,E)},removeClass:function(E){o.className.remove(this,E)},toggleClass:function(F,E){if(typeof E!=="boolean"){E=!o.className.has(this,F)}o.className[E?"add":"remove"](this,F)},remove:function(E){if(!E||o.filter(E,[this]).length){o("*",this).add([this]).each(function(){o.event.remove(this);o.removeData(this)});if(this.parentNode){this.parentNode.removeChild(this)}}},empty:function(){o(this).children().remove();while(this.firstChild){this.removeChild(this.firstChild)}}},function(E,F){o.fn[E]=function(){return this.each(F,arguments)}});function j(E,F){return E[0]&&parseInt(o.curCSS(E[0],F,true),10)||0}var h="jQuery"+e(),v=0,A={};o.extend({cache:{},data:function(F,E,G){F=F==l?A:F;var H=F[h];if(!H){H=F[h]=++v}if(E&&!o.cache[H]){o.cache[H]={}}if(G!==g){o.cache[H][E]=G}return E?o.cache[H][E]:H},removeData:function(F,E){F=F==l?A:F;var H=F[h];if(E){if(o.cache[H]){delete o.cache[H][E];E="";for(E in o.cache[H]){break}if(!E){o.removeData(F)}}}else{try{delete F[h]}catch(G){if(F.removeAttribute){F.removeAttribute(h)}}delete o.cache[H]}},queue:function(F,E,H){if(F){E=(E||"fx")+"queue";var G=o.data(F,E);if(!G||o.isArray(H)){G=o.data(F,E,o.makeArray(H))}else{if(H){G.push(H)}}}return G},dequeue:function(H,G){var E=o.queue(H,G),F=E.shift();if(!G||G==="fx"){F=E[0]}if(F!==g){F.call(H)}}});o.fn.extend({data:function(E,G){var H=E.split(".");H[1]=H[1]?"."+H[1]:"";if(G===g){var F=this.triggerHandler("getData"+H[1]+"!",[H[0]]);if(F===g&&this.length){F=o.data(this[0],E)}return F===g&&H[1]?this.data(H[0]):F}else{return this.trigger("setData"+H[1]+"!",[H[0],G]).each(function(){o.data(this,E,G)})}},removeData:function(E){return this.each(function(){o.removeData(this,E)})},queue:function(E,F){if(typeof E!=="string"){F=E;E="fx"}if(F===g){return o.queue(this[0],E)}return this.each(function(){var G=o.queue(this,E,F);if(E=="fx"&&G.length==1){G[0].call(this)}})},dequeue:function(E){return this.each(function(){o.dequeue(this,E)})}}); +/* + * Sizzle CSS Selector Engine - v0.9.3 + * Copyright 2009, The Dojo Foundation + * Released under the MIT, BSD, and GPL Licenses. + * More information: http://sizzlejs.com/ + */ +(function(){var R=/((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^[\]]*\]|['"][^'"]*['"]|[^[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?/g,L=0,H=Object.prototype.toString;var F=function(Y,U,ab,ac){ab=ab||[];U=U||document;if(U.nodeType!==1&&U.nodeType!==9){return[]}if(!Y||typeof Y!=="string"){return ab}var Z=[],W,af,ai,T,ad,V,X=true;R.lastIndex=0;while((W=R.exec(Y))!==null){Z.push(W[1]);if(W[2]){V=RegExp.rightContext;break}}if(Z.length>1&&M.exec(Y)){if(Z.length===2&&I.relative[Z[0]]){af=J(Z[0]+Z[1],U)}else{af=I.relative[Z[0]]?[U]:F(Z.shift(),U);while(Z.length){Y=Z.shift();if(I.relative[Y]){Y+=Z.shift()}af=J(Y,af)}}}else{var ae=ac?{expr:Z.pop(),set:E(ac)}:F.find(Z.pop(),Z.length===1&&U.parentNode?U.parentNode:U,Q(U));af=F.filter(ae.expr,ae.set);if(Z.length>0){ai=E(af)}else{X=false}while(Z.length){var ah=Z.pop(),ag=ah;if(!I.relative[ah]){ah=""}else{ag=Z.pop()}if(ag==null){ag=U}I.relative[ah](ai,ag,Q(U))}}if(!ai){ai=af}if(!ai){throw"Syntax error, unrecognized expression: "+(ah||Y)}if(H.call(ai)==="[object Array]"){if(!X){ab.push.apply(ab,ai)}else{if(U.nodeType===1){for(var aa=0;ai[aa]!=null;aa++){if(ai[aa]&&(ai[aa]===true||ai[aa].nodeType===1&&K(U,ai[aa]))){ab.push(af[aa])}}}else{for(var aa=0;ai[aa]!=null;aa++){if(ai[aa]&&ai[aa].nodeType===1){ab.push(af[aa])}}}}}else{E(ai,ab)}if(V){F(V,U,ab,ac);if(G){hasDuplicate=false;ab.sort(G);if(hasDuplicate){for(var aa=1;aa":function(Z,U,aa){var X=typeof U==="string";if(X&&!/\W/.test(U)){U=aa?U:U.toUpperCase();for(var V=0,T=Z.length;V=0)){if(!V){T.push(Y)}}else{if(V){U[X]=false}}}}return false},ID:function(T){return T[1].replace(/\\/g,"")},TAG:function(U,T){for(var V=0;T[V]===false;V++){}return T[V]&&Q(T[V])?U[1]:U[1].toUpperCase()},CHILD:function(T){if(T[1]=="nth"){var U=/(-?)(\d*)n((?:\+|-)?\d*)/.exec(T[2]=="even"&&"2n"||T[2]=="odd"&&"2n+1"||!/\D/.test(T[2])&&"0n+"+T[2]||T[2]);T[2]=(U[1]+(U[2]||1))-0;T[3]=U[3]-0}T[0]=L++;return T},ATTR:function(X,U,V,T,Y,Z){var W=X[1].replace(/\\/g,"");if(!Z&&I.attrMap[W]){X[1]=I.attrMap[W]}if(X[2]==="~="){X[4]=" "+X[4]+" "}return X},PSEUDO:function(X,U,V,T,Y){if(X[1]==="not"){if(X[3].match(R).length>1||/^\w/.test(X[3])){X[3]=F(X[3],null,null,U)}else{var W=F.filter(X[3],U,V,true^Y);if(!V){T.push.apply(T,W)}return false}}else{if(I.match.POS.test(X[0])||I.match.CHILD.test(X[0])){return true}}return X},POS:function(T){T.unshift(true);return T}},filters:{enabled:function(T){return T.disabled===false&&T.type!=="hidden"},disabled:function(T){return T.disabled===true},checked:function(T){return T.checked===true},selected:function(T){T.parentNode.selectedIndex;return T.selected===true},parent:function(T){return !!T.firstChild},empty:function(T){return !T.firstChild},has:function(V,U,T){return !!F(T[3],V).length},header:function(T){return/h\d/i.test(T.nodeName)},text:function(T){return"text"===T.type},radio:function(T){return"radio"===T.type},checkbox:function(T){return"checkbox"===T.type},file:function(T){return"file"===T.type},password:function(T){return"password"===T.type},submit:function(T){return"submit"===T.type},image:function(T){return"image"===T.type},reset:function(T){return"reset"===T.type},button:function(T){return"button"===T.type||T.nodeName.toUpperCase()==="BUTTON"},input:function(T){return/input|select|textarea|button/i.test(T.nodeName)}},setFilters:{first:function(U,T){return T===0},last:function(V,U,T,W){return U===W.length-1},even:function(U,T){return T%2===0},odd:function(U,T){return T%2===1},lt:function(V,U,T){return UT[3]-0},nth:function(V,U,T){return T[3]-0==U},eq:function(V,U,T){return T[3]-0==U}},filter:{PSEUDO:function(Z,V,W,aa){var U=V[1],X=I.filters[U];if(X){return X(Z,W,V,aa)}else{if(U==="contains"){return(Z.textContent||Z.innerText||"").indexOf(V[3])>=0}else{if(U==="not"){var Y=V[3];for(var W=0,T=Y.length;W=0)}}},ID:function(U,T){return U.nodeType===1&&U.getAttribute("id")===T},TAG:function(U,T){return(T==="*"&&U.nodeType===1)||U.nodeName===T},CLASS:function(U,T){return(" "+(U.className||U.getAttribute("class"))+" ").indexOf(T)>-1},ATTR:function(Y,W){var V=W[1],T=I.attrHandle[V]?I.attrHandle[V](Y):Y[V]!=null?Y[V]:Y.getAttribute(V),Z=T+"",X=W[2],U=W[4];return T==null?X==="!=":X==="="?Z===U:X==="*="?Z.indexOf(U)>=0:X==="~="?(" "+Z+" ").indexOf(U)>=0:!U?Z&&T!==false:X==="!="?Z!=U:X==="^="?Z.indexOf(U)===0:X==="$="?Z.substr(Z.length-U.length)===U:X==="|="?Z===U||Z.substr(0,U.length+1)===U+"-":false},POS:function(X,U,V,Y){var T=U[2],W=I.setFilters[T];if(W){return W(X,V,U,Y)}}}};var M=I.match.POS;for(var O in I.match){I.match[O]=RegExp(I.match[O].source+/(?![^\[]*\])(?![^\(]*\))/.source)}var E=function(U,T){U=Array.prototype.slice.call(U);if(T){T.push.apply(T,U);return T}return U};try{Array.prototype.slice.call(document.documentElement.childNodes)}catch(N){E=function(X,W){var U=W||[];if(H.call(X)==="[object Array]"){Array.prototype.push.apply(U,X)}else{if(typeof X.length==="number"){for(var V=0,T=X.length;V";var T=document.documentElement;T.insertBefore(U,T.firstChild);if(!!document.getElementById(V)){I.find.ID=function(X,Y,Z){if(typeof Y.getElementById!=="undefined"&&!Z){var W=Y.getElementById(X[1]);return W?W.id===X[1]||typeof W.getAttributeNode!=="undefined"&&W.getAttributeNode("id").nodeValue===X[1]?[W]:g:[]}};I.filter.ID=function(Y,W){var X=typeof Y.getAttributeNode!=="undefined"&&Y.getAttributeNode("id");return Y.nodeType===1&&X&&X.nodeValue===W}}T.removeChild(U)})();(function(){var T=document.createElement("div");T.appendChild(document.createComment(""));if(T.getElementsByTagName("*").length>0){I.find.TAG=function(U,Y){var X=Y.getElementsByTagName(U[1]);if(U[1]==="*"){var W=[];for(var V=0;X[V];V++){if(X[V].nodeType===1){W.push(X[V])}}X=W}return X}}T.innerHTML="";if(T.firstChild&&typeof T.firstChild.getAttribute!=="undefined"&&T.firstChild.getAttribute("href")!=="#"){I.attrHandle.href=function(U){return U.getAttribute("href",2)}}})();if(document.querySelectorAll){(function(){var T=F,U=document.createElement("div");U.innerHTML="

";if(U.querySelectorAll&&U.querySelectorAll(".TEST").length===0){return}F=function(Y,X,V,W){X=X||document;if(!W&&X.nodeType===9&&!Q(X)){try{return E(X.querySelectorAll(Y),V)}catch(Z){}}return T(Y,X,V,W)};F.find=T.find;F.filter=T.filter;F.selectors=T.selectors;F.matches=T.matches})()}if(document.getElementsByClassName&&document.documentElement.getElementsByClassName){(function(){var T=document.createElement("div");T.innerHTML="
";if(T.getElementsByClassName("e").length===0){return}T.lastChild.className="e";if(T.getElementsByClassName("e").length===1){return}I.order.splice(1,0,"CLASS");I.find.CLASS=function(U,V,W){if(typeof V.getElementsByClassName!=="undefined"&&!W){return V.getElementsByClassName(U[1])}}})()}function P(U,Z,Y,ad,aa,ac){var ab=U=="previousSibling"&&!ac;for(var W=0,V=ad.length;W0){X=T;break}}}T=T[U]}ad[W]=X}}}var K=document.compareDocumentPosition?function(U,T){return U.compareDocumentPosition(T)&16}:function(U,T){return U!==T&&(U.contains?U.contains(T):true)};var Q=function(T){return T.nodeType===9&&T.documentElement.nodeName!=="HTML"||!!T.ownerDocument&&Q(T.ownerDocument)};var J=function(T,aa){var W=[],X="",Y,V=aa.nodeType?[aa]:aa;while((Y=I.match.PSEUDO.exec(T))){X+=Y[0];T=T.replace(I.match.PSEUDO,"")}T=I.relative[T]?T+"*":T;for(var Z=0,U=V.length;Z0||T.offsetHeight>0};F.selectors.filters.animated=function(T){return o.grep(o.timers,function(U){return T===U.elem}).length};o.multiFilter=function(V,T,U){if(U){V=":not("+V+")"}return F.matches(V,T)};o.dir=function(V,U){var T=[],W=V[U];while(W&&W!=document){if(W.nodeType==1){T.push(W)}W=W[U]}return T};o.nth=function(X,T,V,W){T=T||1;var U=0;for(;X;X=X[V]){if(X.nodeType==1&&++U==T){break}}return X};o.sibling=function(V,U){var T=[];for(;V;V=V.nextSibling){if(V.nodeType==1&&V!=U){T.push(V)}}return T};return;l.Sizzle=F})();o.event={add:function(I,F,H,K){if(I.nodeType==3||I.nodeType==8){return}if(I.setInterval&&I!=l){I=l}if(!H.guid){H.guid=this.guid++}if(K!==g){var G=H;H=this.proxy(G);H.data=K}var E=o.data(I,"events")||o.data(I,"events",{}),J=o.data(I,"handle")||o.data(I,"handle",function(){return typeof o!=="undefined"&&!o.event.triggered?o.event.handle.apply(arguments.callee.elem,arguments):g});J.elem=I;o.each(F.split(/\s+/),function(M,N){var O=N.split(".");N=O.shift();H.type=O.slice().sort().join(".");var L=E[N];if(o.event.specialAll[N]){o.event.specialAll[N].setup.call(I,K,O)}if(!L){L=E[N]={};if(!o.event.special[N]||o.event.special[N].setup.call(I,K,O)===false){if(I.addEventListener){I.addEventListener(N,J,false)}else{if(I.attachEvent){I.attachEvent("on"+N,J)}}}}L[H.guid]=H;o.event.global[N]=true});I=null},guid:1,global:{},remove:function(K,H,J){if(K.nodeType==3||K.nodeType==8){return}var G=o.data(K,"events"),F,E;if(G){if(H===g||(typeof H==="string"&&H.charAt(0)==".")){for(var I in G){this.remove(K,I+(H||""))}}else{if(H.type){J=H.handler;H=H.type}o.each(H.split(/\s+/),function(M,O){var Q=O.split(".");O=Q.shift();var N=RegExp("(^|\\.)"+Q.slice().sort().join(".*\\.")+"(\\.|$)");if(G[O]){if(J){delete G[O][J.guid]}else{for(var P in G[O]){if(N.test(G[O][P].type)){delete G[O][P]}}}if(o.event.specialAll[O]){o.event.specialAll[O].teardown.call(K,Q)}for(F in G[O]){break}if(!F){if(!o.event.special[O]||o.event.special[O].teardown.call(K,Q)===false){if(K.removeEventListener){K.removeEventListener(O,o.data(K,"handle"),false)}else{if(K.detachEvent){K.detachEvent("on"+O,o.data(K,"handle"))}}}F=null;delete G[O]}}})}for(F in G){break}if(!F){var L=o.data(K,"handle");if(L){L.elem=null}o.removeData(K,"events");o.removeData(K,"handle")}}},trigger:function(I,K,H,E){var G=I.type||I;if(!E){I=typeof I==="object"?I[h]?I:o.extend(o.Event(G),I):o.Event(G);if(G.indexOf("!")>=0){I.type=G=G.slice(0,-1);I.exclusive=true}if(!H){I.stopPropagation();if(this.global[G]){o.each(o.cache,function(){if(this.events&&this.events[G]){o.event.trigger(I,K,this.handle.elem)}})}}if(!H||H.nodeType==3||H.nodeType==8){return g}I.result=g;I.target=H;K=o.makeArray(K);K.unshift(I)}I.currentTarget=H;var J=o.data(H,"handle");if(J){J.apply(H,K)}if((!H[G]||(o.nodeName(H,"a")&&G=="click"))&&H["on"+G]&&H["on"+G].apply(H,K)===false){I.result=false}if(!E&&H[G]&&!I.isDefaultPrevented()&&!(o.nodeName(H,"a")&&G=="click")){this.triggered=true;try{H[G]()}catch(L){}}this.triggered=false;if(!I.isPropagationStopped()){var F=H.parentNode||H.ownerDocument;if(F){o.event.trigger(I,K,F,true)}}},handle:function(K){var J,E;K=arguments[0]=o.event.fix(K||l.event);K.currentTarget=this;var L=K.type.split(".");K.type=L.shift();J=!L.length&&!K.exclusive;var I=RegExp("(^|\\.)"+L.slice().sort().join(".*\\.")+"(\\.|$)");E=(o.data(this,"events")||{})[K.type];for(var G in E){var H=E[G];if(J||I.test(H.type)){K.handler=H;K.data=H.data;var F=H.apply(this,arguments);if(F!==g){K.result=F;if(F===false){K.preventDefault();K.stopPropagation()}}if(K.isImmediatePropagationStopped()){break}}}},props:"altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode metaKey newValue originalTarget pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target toElement view wheelDelta which".split(" "),fix:function(H){if(H[h]){return H}var F=H;H=o.Event(F);for(var G=this.props.length,J;G;){J=this.props[--G];H[J]=F[J]}if(!H.target){H.target=H.srcElement||document}if(H.target.nodeType==3){H.target=H.target.parentNode}if(!H.relatedTarget&&H.fromElement){H.relatedTarget=H.fromElement==H.target?H.toElement:H.fromElement}if(H.pageX==null&&H.clientX!=null){var I=document.documentElement,E=document.body;H.pageX=H.clientX+(I&&I.scrollLeft||E&&E.scrollLeft||0)-(I.clientLeft||0);H.pageY=H.clientY+(I&&I.scrollTop||E&&E.scrollTop||0)-(I.clientTop||0)}if(!H.which&&((H.charCode||H.charCode===0)?H.charCode:H.keyCode)){H.which=H.charCode||H.keyCode}if(!H.metaKey&&H.ctrlKey){H.metaKey=H.ctrlKey}if(!H.which&&H.button){H.which=(H.button&1?1:(H.button&2?3:(H.button&4?2:0)))}return H},proxy:function(F,E){E=E||function(){return F.apply(this,arguments)};E.guid=F.guid=F.guid||E.guid||this.guid++;return E},special:{ready:{setup:B,teardown:function(){}}},specialAll:{live:{setup:function(E,F){o.event.add(this,F[0],c)},teardown:function(G){if(G.length){var E=0,F=RegExp("(^|\\.)"+G[0]+"(\\.|$)");o.each((o.data(this,"events").live||{}),function(){if(F.test(this.type)){E++}});if(E<1){o.event.remove(this,G[0],c)}}}}}};o.Event=function(E){if(!this.preventDefault){return new o.Event(E)}if(E&&E.type){this.originalEvent=E;this.type=E.type}else{this.type=E}this.timeStamp=e();this[h]=true};function k(){return false}function u(){return true}o.Event.prototype={preventDefault:function(){this.isDefaultPrevented=u;var E=this.originalEvent;if(!E){return}if(E.preventDefault){E.preventDefault()}E.returnValue=false},stopPropagation:function(){this.isPropagationStopped=u;var E=this.originalEvent;if(!E){return}if(E.stopPropagation){E.stopPropagation()}E.cancelBubble=true},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=u;this.stopPropagation()},isDefaultPrevented:k,isPropagationStopped:k,isImmediatePropagationStopped:k};var a=function(F){var E=F.relatedTarget;while(E&&E!=this){try{E=E.parentNode}catch(G){E=this}}if(E!=this){F.type=F.data;o.event.handle.apply(this,arguments)}};o.each({mouseover:"mouseenter",mouseout:"mouseleave"},function(F,E){o.event.special[E]={setup:function(){o.event.add(this,F,a,E)},teardown:function(){o.event.remove(this,F,a)}}});o.fn.extend({bind:function(F,G,E){return F=="unload"?this.one(F,G,E):this.each(function(){o.event.add(this,F,E||G,E&&G)})},one:function(G,H,F){var E=o.event.proxy(F||H,function(I){o(this).unbind(I,E);return(F||H).apply(this,arguments)});return this.each(function(){o.event.add(this,G,E,F&&H)})},unbind:function(F,E){return this.each(function(){o.event.remove(this,F,E)})},trigger:function(E,F){return this.each(function(){o.event.trigger(E,F,this)})},triggerHandler:function(E,G){if(this[0]){var F=o.Event(E);F.preventDefault();F.stopPropagation();o.event.trigger(F,G,this[0]);return F.result}},toggle:function(G){var E=arguments,F=1;while(F=0){var E=G.slice(I,G.length);G=G.slice(0,I)}var H="GET";if(J){if(o.isFunction(J)){K=J;J=null}else{if(typeof J==="object"){J=o.param(J);H="POST"}}}var F=this;o.ajax({url:G,type:H,dataType:"html",data:J,complete:function(M,L){if(L=="success"||L=="notmodified"){F.html(E?o("
").append(M.responseText.replace(//g,"")).find(E):M.responseText)}if(K){F.each(K,[M.responseText,L,M])}}});return this},serialize:function(){return o.param(this.serializeArray())},serializeArray:function(){return this.map(function(){return this.elements?o.makeArray(this.elements):this}).filter(function(){return this.name&&!this.disabled&&(this.checked||/select|textarea/i.test(this.nodeName)||/text|hidden|password|search/i.test(this.type))}).map(function(E,F){var G=o(this).val();return G==null?null:o.isArray(G)?o.map(G,function(I,H){return{name:F.name,value:I}}):{name:F.name,value:G}}).get()}});o.each("ajaxStart,ajaxStop,ajaxComplete,ajaxError,ajaxSuccess,ajaxSend".split(","),function(E,F){o.fn[F]=function(G){return this.bind(F,G)}});var r=e();o.extend({get:function(E,G,H,F){if(o.isFunction(G)){H=G;G=null}return o.ajax({type:"GET",url:E,data:G,success:H,dataType:F})},getScript:function(E,F){return o.get(E,null,F,"script")},getJSON:function(E,F,G){return o.get(E,F,G,"json")},post:function(E,G,H,F){if(o.isFunction(G)){H=G;G={}}return o.ajax({type:"POST",url:E,data:G,success:H,dataType:F})},ajaxSetup:function(E){o.extend(o.ajaxSettings,E)},ajaxSettings:{url:location.href,global:true,type:"GET",contentType:"application/x-www-form-urlencoded",processData:true,async:true,xhr:function(){return l.ActiveXObject?new ActiveXObject("Microsoft.XMLHTTP"):new XMLHttpRequest()},accepts:{xml:"application/xml, text/xml",html:"text/html",script:"text/javascript, application/javascript",json:"application/json, text/javascript",text:"text/plain",_default:"*/*"}},lastModified:{},ajax:function(M){M=o.extend(true,M,o.extend(true,{},o.ajaxSettings,M));var W,F=/=\?(&|$)/g,R,V,G=M.type.toUpperCase();if(M.data&&M.processData&&typeof M.data!=="string"){M.data=o.param(M.data)}if(M.dataType=="jsonp"){if(G=="GET"){if(!M.url.match(F)){M.url+=(M.url.match(/\?/)?"&":"?")+(M.jsonp||"callback")+"=?"}}else{if(!M.data||!M.data.match(F)){M.data=(M.data?M.data+"&":"")+(M.jsonp||"callback")+"=?"}}M.dataType="json"}if(M.dataType=="json"&&(M.data&&M.data.match(F)||M.url.match(F))){W="jsonp"+r++;if(M.data){M.data=(M.data+"").replace(F,"="+W+"$1")}M.url=M.url.replace(F,"="+W+"$1");M.dataType="script";l[W]=function(X){V=X;I();L();l[W]=g;try{delete l[W]}catch(Y){}if(H){H.removeChild(T)}}}if(M.dataType=="script"&&M.cache==null){M.cache=false}if(M.cache===false&&G=="GET"){var E=e();var U=M.url.replace(/(\?|&)_=.*?(&|$)/,"$1_="+E+"$2");M.url=U+((U==M.url)?(M.url.match(/\?/)?"&":"?")+"_="+E:"")}if(M.data&&G=="GET"){M.url+=(M.url.match(/\?/)?"&":"?")+M.data;M.data=null}if(M.global&&!o.active++){o.event.trigger("ajaxStart")}var Q=/^(\w+:)?\/\/([^\/?#]+)/.exec(M.url);if(M.dataType=="script"&&G=="GET"&&Q&&(Q[1]&&Q[1]!=location.protocol||Q[2]!=location.host)){var H=document.getElementsByTagName("head")[0];var T=document.createElement("script");T.src=M.url;if(M.scriptCharset){T.charset=M.scriptCharset}if(!W){var O=false;T.onload=T.onreadystatechange=function(){if(!O&&(!this.readyState||this.readyState=="loaded"||this.readyState=="complete")){O=true;I();L();T.onload=T.onreadystatechange=null;H.removeChild(T)}}}H.appendChild(T);return g}var K=false;var J=M.xhr();if(M.username){J.open(G,M.url,M.async,M.username,M.password)}else{J.open(G,M.url,M.async)}try{if(M.data){J.setRequestHeader("Content-Type",M.contentType)}if(M.ifModified){J.setRequestHeader("If-Modified-Since",o.lastModified[M.url]||"Thu, 01 Jan 1970 00:00:00 GMT")}J.setRequestHeader("X-Requested-With","XMLHttpRequest");J.setRequestHeader("Accept",M.dataType&&M.accepts[M.dataType]?M.accepts[M.dataType]+", */*":M.accepts._default)}catch(S){}if(M.beforeSend&&M.beforeSend(J,M)===false){if(M.global&&!--o.active){o.event.trigger("ajaxStop")}J.abort();return false}if(M.global){o.event.trigger("ajaxSend",[J,M])}var N=function(X){if(J.readyState==0){if(P){clearInterval(P);P=null;if(M.global&&!--o.active){o.event.trigger("ajaxStop")}}}else{if(!K&&J&&(J.readyState==4||X=="timeout")){K=true;if(P){clearInterval(P);P=null}R=X=="timeout"?"timeout":!o.httpSuccess(J)?"error":M.ifModified&&o.httpNotModified(J,M.url)?"notmodified":"success";if(R=="success"){try{V=o.httpData(J,M.dataType,M)}catch(Z){R="parsererror"}}if(R=="success"){var Y;try{Y=J.getResponseHeader("Last-Modified")}catch(Z){}if(M.ifModified&&Y){o.lastModified[M.url]=Y}if(!W){I()}}else{o.handleError(M,J,R)}L();if(X){J.abort()}if(M.async){J=null}}}};if(M.async){var P=setInterval(N,13);if(M.timeout>0){setTimeout(function(){if(J&&!K){N("timeout")}},M.timeout)}}try{J.send(M.data)}catch(S){o.handleError(M,J,null,S)}if(!M.async){N()}function I(){if(M.success){M.success(V,R)}if(M.global){o.event.trigger("ajaxSuccess",[J,M])}}function L(){if(M.complete){M.complete(J,R)}if(M.global){o.event.trigger("ajaxComplete",[J,M])}if(M.global&&!--o.active){o.event.trigger("ajaxStop")}}return J},handleError:function(F,H,E,G){if(F.error){F.error(H,E,G)}if(F.global){o.event.trigger("ajaxError",[H,F,G])}},active:0,httpSuccess:function(F){try{return !F.status&&location.protocol=="file:"||(F.status>=200&&F.status<300)||F.status==304||F.status==1223}catch(E){}return false},httpNotModified:function(G,E){try{var H=G.getResponseHeader("Last-Modified");return G.status==304||H==o.lastModified[E]}catch(F){}return false},httpData:function(J,H,G){var F=J.getResponseHeader("content-type"),E=H=="xml"||!H&&F&&F.indexOf("xml")>=0,I=E?J.responseXML:J.responseText;if(E&&I.documentElement.tagName=="parsererror"){throw"parsererror"}if(G&&G.dataFilter){I=G.dataFilter(I,H)}if(typeof I==="string"){if(H=="script"){o.globalEval(I)}if(H=="json"){I=l["eval"]("("+I+")")}}return I},param:function(E){var G=[];function H(I,J){G[G.length]=encodeURIComponent(I)+"="+encodeURIComponent(J)}if(o.isArray(E)||E.jquery){o.each(E,function(){H(this.name,this.value)})}else{for(var F in E){if(o.isArray(E[F])){o.each(E[F],function(){H(F,this)})}else{H(F,o.isFunction(E[F])?E[F]():E[F])}}}return G.join("&").replace(/%20/g,"+")}});var m={},n,d=[["height","marginTop","marginBottom","paddingTop","paddingBottom"],["width","marginLeft","marginRight","paddingLeft","paddingRight"],["opacity"]];function t(F,E){var G={};o.each(d.concat.apply([],d.slice(0,E)),function(){G[this]=F});return G}o.fn.extend({show:function(J,L){if(J){return this.animate(t("show",3),J,L)}else{for(var H=0,F=this.length;H").appendTo("body");K=I.css("display");if(K==="none"){K="block"}I.remove();m[G]=K}o.data(this[H],"olddisplay",K)}}for(var H=0,F=this.length;H=0;H--){if(G[H].elem==this){if(E){G[H](true)}G.splice(H,1)}}});if(!E){this.dequeue()}return this}});o.each({slideDown:t("show",1),slideUp:t("hide",1),slideToggle:t("toggle",1),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"}},function(E,F){o.fn[E]=function(G,H){return this.animate(F,G,H)}});o.extend({speed:function(G,H,F){var E=typeof G==="object"?G:{complete:F||!F&&H||o.isFunction(G)&&G,duration:G,easing:F&&H||H&&!o.isFunction(H)&&H};E.duration=o.fx.off?0:typeof E.duration==="number"?E.duration:o.fx.speeds[E.duration]||o.fx.speeds._default;E.old=E.complete;E.complete=function(){if(E.queue!==false){o(this).dequeue()}if(o.isFunction(E.old)){E.old.call(this)}};return E},easing:{linear:function(G,H,E,F){return E+F*G},swing:function(G,H,E,F){return((-Math.cos(G*Math.PI)/2)+0.5)*F+E}},timers:[],fx:function(F,E,G){this.options=E;this.elem=F;this.prop=G;if(!E.orig){E.orig={}}}});o.fx.prototype={update:function(){if(this.options.step){this.options.step.call(this.elem,this.now,this)}(o.fx.step[this.prop]||o.fx.step._default)(this);if((this.prop=="height"||this.prop=="width")&&this.elem.style){this.elem.style.display="block"}},cur:function(F){if(this.elem[this.prop]!=null&&(!this.elem.style||this.elem.style[this.prop]==null)){return this.elem[this.prop]}var E=parseFloat(o.css(this.elem,this.prop,F));return E&&E>-10000?E:parseFloat(o.curCSS(this.elem,this.prop))||0},custom:function(I,H,G){this.startTime=e();this.start=I;this.end=H;this.unit=G||this.unit||"px";this.now=this.start;this.pos=this.state=0;var E=this;function F(J){return E.step(J)}F.elem=this.elem;if(F()&&o.timers.push(F)&&!n){n=setInterval(function(){var K=o.timers;for(var J=0;J=this.options.duration+this.startTime){this.now=this.end;this.pos=this.state=1;this.update();this.options.curAnim[this.prop]=true;var E=true;for(var F in this.options.curAnim){if(this.options.curAnim[F]!==true){E=false}}if(E){if(this.options.display!=null){this.elem.style.overflow=this.options.overflow;this.elem.style.display=this.options.display;if(o.css(this.elem,"display")=="none"){this.elem.style.display="block"}}if(this.options.hide){o(this.elem).hide()}if(this.options.hide||this.options.show){for(var I in this.options.curAnim){o.attr(this.elem.style,I,this.options.orig[I])}}this.options.complete.call(this.elem)}return false}else{var J=G-this.startTime;this.state=J/this.options.duration;this.pos=o.easing[this.options.easing||(o.easing.swing?"swing":"linear")](this.state,J,0,1,this.options.duration);this.now=this.start+((this.end-this.start)*this.pos);this.update()}return true}};o.extend(o.fx,{speeds:{slow:600,fast:200,_default:400},step:{opacity:function(E){o.attr(E.elem.style,"opacity",E.now)},_default:function(E){if(E.elem.style&&E.elem.style[E.prop]!=null){E.elem.style[E.prop]=E.now+E.unit}else{E.elem[E.prop]=E.now}}}});if(document.documentElement.getBoundingClientRect){o.fn.offset=function(){if(!this[0]){return{top:0,left:0}}if(this[0]===this[0].ownerDocument.body){return o.offset.bodyOffset(this[0])}var G=this[0].getBoundingClientRect(),J=this[0].ownerDocument,F=J.body,E=J.documentElement,L=E.clientTop||F.clientTop||0,K=E.clientLeft||F.clientLeft||0,I=G.top+(self.pageYOffset||o.boxModel&&E.scrollTop||F.scrollTop)-L,H=G.left+(self.pageXOffset||o.boxModel&&E.scrollLeft||F.scrollLeft)-K;return{top:I,left:H}}}else{o.fn.offset=function(){if(!this[0]){return{top:0,left:0}}if(this[0]===this[0].ownerDocument.body){return o.offset.bodyOffset(this[0])}o.offset.initialized||o.offset.initialize();var J=this[0],G=J.offsetParent,F=J,O=J.ownerDocument,M,H=O.documentElement,K=O.body,L=O.defaultView,E=L.getComputedStyle(J,null),N=J.offsetTop,I=J.offsetLeft;while((J=J.parentNode)&&J!==K&&J!==H){M=L.getComputedStyle(J,null);N-=J.scrollTop,I-=J.scrollLeft;if(J===G){N+=J.offsetTop,I+=J.offsetLeft;if(o.offset.doesNotAddBorder&&!(o.offset.doesAddBorderForTableAndCells&&/^t(able|d|h)$/i.test(J.tagName))){N+=parseInt(M.borderTopWidth,10)||0,I+=parseInt(M.borderLeftWidth,10)||0}F=G,G=J.offsetParent}if(o.offset.subtractsBorderForOverflowNotVisible&&M.overflow!=="visible"){N+=parseInt(M.borderTopWidth,10)||0,I+=parseInt(M.borderLeftWidth,10)||0}E=M}if(E.position==="relative"||E.position==="static"){N+=K.offsetTop,I+=K.offsetLeft}if(E.position==="fixed"){N+=Math.max(H.scrollTop,K.scrollTop),I+=Math.max(H.scrollLeft,K.scrollLeft)}return{top:N,left:I}}}o.offset={initialize:function(){if(this.initialized){return}var L=document.body,F=document.createElement("div"),H,G,N,I,M,E,J=L.style.marginTop,K='
';M={position:"absolute",top:0,left:0,margin:0,border:0,width:"1px",height:"1px",visibility:"hidden"};for(E in M){F.style[E]=M[E]}F.innerHTML=K;L.insertBefore(F,L.firstChild);H=F.firstChild,G=H.firstChild,I=H.nextSibling.firstChild.firstChild;this.doesNotAddBorder=(G.offsetTop!==5);this.doesAddBorderForTableAndCells=(I.offsetTop===5);H.style.overflow="hidden",H.style.position="relative";this.subtractsBorderForOverflowNotVisible=(G.offsetTop===-5);L.style.marginTop="1px";this.doesNotIncludeMarginInBodyOffset=(L.offsetTop===0);L.style.marginTop=J;L.removeChild(F);this.initialized=true},bodyOffset:function(E){o.offset.initialized||o.offset.initialize();var G=E.offsetTop,F=E.offsetLeft;if(o.offset.doesNotIncludeMarginInBodyOffset){G+=parseInt(o.curCSS(E,"marginTop",true),10)||0,F+=parseInt(o.curCSS(E,"marginLeft",true),10)||0}return{top:G,left:F}}};o.fn.extend({position:function(){var I=0,H=0,F;if(this[0]){var G=this.offsetParent(),J=this.offset(),E=/^body|html$/i.test(G[0].tagName)?{top:0,left:0}:G.offset();J.top-=j(this,"marginTop");J.left-=j(this,"marginLeft");E.top+=j(G,"borderTopWidth");E.left+=j(G,"borderLeftWidth");F={top:J.top-E.top,left:J.left-E.left}}return F},offsetParent:function(){var E=this[0].offsetParent||document.body;while(E&&(!/^body|html$/i.test(E.tagName)&&o.css(E,"position")=="static")){E=E.offsetParent}return o(E)}});o.each(["Left","Top"],function(F,E){var G="scroll"+E;o.fn[G]=function(H){if(!this[0]){return null}return H!==g?this.each(function(){this==l||this==document?l.scrollTo(!F?H:o(l).scrollLeft(),F?H:o(l).scrollTop()):this[G]=H}):this[0]==l||this[0]==document?self[F?"pageYOffset":"pageXOffset"]||o.boxModel&&document.documentElement[G]||document.body[G]:this[0][G]}});o.each(["Height","Width"],function(I,G){var E=I?"Left":"Top",H=I?"Right":"Bottom",F=G.toLowerCase();o.fn["inner"+G]=function(){return this[0]?o.css(this[0],F,false,"padding"):null};o.fn["outer"+G]=function(K){return this[0]?o.css(this[0],F,false,K?"margin":"border"):null};var J=G.toLowerCase();o.fn[J]=function(K){return this[0]==l?document.compatMode=="CSS1Compat"&&document.documentElement["client"+G]||document.body["client"+G]:this[0]==document?Math.max(document.documentElement["client"+G],document.body["scroll"+G],document.documentElement["scroll"+G],document.body["offset"+G],document.documentElement["offset"+G]):K===g?(this.length?o.css(this[0],J):null):this.css(J,typeof K==="string"?K:K+"px")}})})(); \ No newline at end of file diff --git a/backoffice/static/js/jquery-1.4.4.min.js b/backoffice/static/js/jquery-1.4.4.min.js new file mode 100644 index 0000000..8f3ca2e --- /dev/null +++ b/backoffice/static/js/jquery-1.4.4.min.js @@ -0,0 +1,167 @@ +/*! + * jQuery JavaScript Library v1.4.4 + * http://jquery.com/ + * + * Copyright 2010, John Resig + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * Includes Sizzle.js + * http://sizzlejs.com/ + * Copyright 2010, The Dojo Foundation + * Released under the MIT, BSD, and GPL Licenses. + * + * Date: Thu Nov 11 19:04:53 2010 -0500 + */ +(function(E,B){function ka(a,b,d){if(d===B&&a.nodeType===1){d=a.getAttribute("data-"+b);if(typeof d==="string"){try{d=d==="true"?true:d==="false"?false:d==="null"?null:!c.isNaN(d)?parseFloat(d):Ja.test(d)?c.parseJSON(d):d}catch(e){}c.data(a,b,d)}else d=B}return d}function U(){return false}function ca(){return true}function la(a,b,d){d[0].type=a;return c.event.handle.apply(b,d)}function Ka(a){var b,d,e,f,h,l,k,o,x,r,A,C=[];f=[];h=c.data(this,this.nodeType?"events":"__events__");if(typeof h==="function")h= +h.events;if(!(a.liveFired===this||!h||!h.live||a.button&&a.type==="click")){if(a.namespace)A=RegExp("(^|\\.)"+a.namespace.split(".").join("\\.(?:.*\\.)?")+"(\\.|$)");a.liveFired=this;var J=h.live.slice(0);for(k=0;kd)break;a.currentTarget=f.elem;a.data=f.handleObj.data;a.handleObj=f.handleObj;A=f.handleObj.origHandler.apply(f.elem,arguments);if(A===false||a.isPropagationStopped()){d=f.level;if(A===false)b=false;if(a.isImmediatePropagationStopped())break}}return b}}function Y(a,b){return(a&&a!=="*"?a+".":"")+b.replace(La, +"`").replace(Ma,"&")}function ma(a,b,d){if(c.isFunction(b))return c.grep(a,function(f,h){return!!b.call(f,h,f)===d});else if(b.nodeType)return c.grep(a,function(f){return f===b===d});else if(typeof b==="string"){var e=c.grep(a,function(f){return f.nodeType===1});if(Na.test(b))return c.filter(b,e,!d);else b=c.filter(b,e)}return c.grep(a,function(f){return c.inArray(f,b)>=0===d})}function na(a,b){var d=0;b.each(function(){if(this.nodeName===(a[d]&&a[d].nodeName)){var e=c.data(a[d++]),f=c.data(this, +e);if(e=e&&e.events){delete f.handle;f.events={};for(var h in e)for(var l in e[h])c.event.add(this,h,e[h][l],e[h][l].data)}}})}function Oa(a,b){b.src?c.ajax({url:b.src,async:false,dataType:"script"}):c.globalEval(b.text||b.textContent||b.innerHTML||"");b.parentNode&&b.parentNode.removeChild(b)}function oa(a,b,d){var e=b==="width"?a.offsetWidth:a.offsetHeight;if(d==="border")return e;c.each(b==="width"?Pa:Qa,function(){d||(e-=parseFloat(c.css(a,"padding"+this))||0);if(d==="margin")e+=parseFloat(c.css(a, +"margin"+this))||0;else e-=parseFloat(c.css(a,"border"+this+"Width"))||0});return e}function da(a,b,d,e){if(c.isArray(b)&&b.length)c.each(b,function(f,h){d||Ra.test(a)?e(a,h):da(a+"["+(typeof h==="object"||c.isArray(h)?f:"")+"]",h,d,e)});else if(!d&&b!=null&&typeof b==="object")c.isEmptyObject(b)?e(a,""):c.each(b,function(f,h){da(a+"["+f+"]",h,d,e)});else e(a,b)}function S(a,b){var d={};c.each(pa.concat.apply([],pa.slice(0,b)),function(){d[this]=a});return d}function qa(a){if(!ea[a]){var b=c("<"+ +a+">").appendTo("body"),d=b.css("display");b.remove();if(d==="none"||d==="")d="block";ea[a]=d}return ea[a]}function fa(a){return c.isWindow(a)?a:a.nodeType===9?a.defaultView||a.parentWindow:false}var t=E.document,c=function(){function a(){if(!b.isReady){try{t.documentElement.doScroll("left")}catch(j){setTimeout(a,1);return}b.ready()}}var b=function(j,s){return new b.fn.init(j,s)},d=E.jQuery,e=E.$,f,h=/^(?:[^<]*(<[\w\W]+>)[^>]*$|#([\w\-]+)$)/,l=/\S/,k=/^\s+/,o=/\s+$/,x=/\W/,r=/\d/,A=/^<(\w+)\s*\/?>(?:<\/\1>)?$/, +C=/^[\],:{}\s]*$/,J=/\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g,w=/"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g,I=/(?:^|:|,)(?:\s*\[)+/g,L=/(webkit)[ \/]([\w.]+)/,g=/(opera)(?:.*version)?[ \/]([\w.]+)/,i=/(msie) ([\w.]+)/,n=/(mozilla)(?:.*? rv:([\w.]+))?/,m=navigator.userAgent,p=false,q=[],u,y=Object.prototype.toString,F=Object.prototype.hasOwnProperty,M=Array.prototype.push,N=Array.prototype.slice,O=String.prototype.trim,D=Array.prototype.indexOf,R={};b.fn=b.prototype={init:function(j, +s){var v,z,H;if(!j)return this;if(j.nodeType){this.context=this[0]=j;this.length=1;return this}if(j==="body"&&!s&&t.body){this.context=t;this[0]=t.body;this.selector="body";this.length=1;return this}if(typeof j==="string")if((v=h.exec(j))&&(v[1]||!s))if(v[1]){H=s?s.ownerDocument||s:t;if(z=A.exec(j))if(b.isPlainObject(s)){j=[t.createElement(z[1])];b.fn.attr.call(j,s,true)}else j=[H.createElement(z[1])];else{z=b.buildFragment([v[1]],[H]);j=(z.cacheable?z.fragment.cloneNode(true):z.fragment).childNodes}return b.merge(this, +j)}else{if((z=t.getElementById(v[2]))&&z.parentNode){if(z.id!==v[2])return f.find(j);this.length=1;this[0]=z}this.context=t;this.selector=j;return this}else if(!s&&!x.test(j)){this.selector=j;this.context=t;j=t.getElementsByTagName(j);return b.merge(this,j)}else return!s||s.jquery?(s||f).find(j):b(s).find(j);else if(b.isFunction(j))return f.ready(j);if(j.selector!==B){this.selector=j.selector;this.context=j.context}return b.makeArray(j,this)},selector:"",jquery:"1.4.4",length:0,size:function(){return this.length}, +toArray:function(){return N.call(this,0)},get:function(j){return j==null?this.toArray():j<0?this.slice(j)[0]:this[j]},pushStack:function(j,s,v){var z=b();b.isArray(j)?M.apply(z,j):b.merge(z,j);z.prevObject=this;z.context=this.context;if(s==="find")z.selector=this.selector+(this.selector?" ":"")+v;else if(s)z.selector=this.selector+"."+s+"("+v+")";return z},each:function(j,s){return b.each(this,j,s)},ready:function(j){b.bindReady();if(b.isReady)j.call(t,b);else q&&q.push(j);return this},eq:function(j){return j=== +-1?this.slice(j):this.slice(j,+j+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(N.apply(this,arguments),"slice",N.call(arguments).join(","))},map:function(j){return this.pushStack(b.map(this,function(s,v){return j.call(s,v,s)}))},end:function(){return this.prevObject||b(null)},push:M,sort:[].sort,splice:[].splice};b.fn.init.prototype=b.fn;b.extend=b.fn.extend=function(){var j,s,v,z,H,G=arguments[0]||{},K=1,Q=arguments.length,ga=false; +if(typeof G==="boolean"){ga=G;G=arguments[1]||{};K=2}if(typeof G!=="object"&&!b.isFunction(G))G={};if(Q===K){G=this;--K}for(;K0))if(q){var s=0,v=q;for(q=null;j=v[s++];)j.call(t,b);b.fn.trigger&&b(t).trigger("ready").unbind("ready")}}},bindReady:function(){if(!p){p=true;if(t.readyState==="complete")return setTimeout(b.ready,1);if(t.addEventListener){t.addEventListener("DOMContentLoaded",u,false);E.addEventListener("load",b.ready,false)}else if(t.attachEvent){t.attachEvent("onreadystatechange",u);E.attachEvent("onload", +b.ready);var j=false;try{j=E.frameElement==null}catch(s){}t.documentElement.doScroll&&j&&a()}}},isFunction:function(j){return b.type(j)==="function"},isArray:Array.isArray||function(j){return b.type(j)==="array"},isWindow:function(j){return j&&typeof j==="object"&&"setInterval"in j},isNaN:function(j){return j==null||!r.test(j)||isNaN(j)},type:function(j){return j==null?String(j):R[y.call(j)]||"object"},isPlainObject:function(j){if(!j||b.type(j)!=="object"||j.nodeType||b.isWindow(j))return false;if(j.constructor&& +!F.call(j,"constructor")&&!F.call(j.constructor.prototype,"isPrototypeOf"))return false;for(var s in j);return s===B||F.call(j,s)},isEmptyObject:function(j){for(var s in j)return false;return true},error:function(j){throw j;},parseJSON:function(j){if(typeof j!=="string"||!j)return null;j=b.trim(j);if(C.test(j.replace(J,"@").replace(w,"]").replace(I,"")))return E.JSON&&E.JSON.parse?E.JSON.parse(j):(new Function("return "+j))();else b.error("Invalid JSON: "+j)},noop:function(){},globalEval:function(j){if(j&& +l.test(j)){var s=t.getElementsByTagName("head")[0]||t.documentElement,v=t.createElement("script");v.type="text/javascript";if(b.support.scriptEval)v.appendChild(t.createTextNode(j));else v.text=j;s.insertBefore(v,s.firstChild);s.removeChild(v)}},nodeName:function(j,s){return j.nodeName&&j.nodeName.toUpperCase()===s.toUpperCase()},each:function(j,s,v){var z,H=0,G=j.length,K=G===B||b.isFunction(j);if(v)if(K)for(z in j){if(s.apply(j[z],v)===false)break}else for(;H
a";var f=d.getElementsByTagName("*"),h=d.getElementsByTagName("a")[0],l=t.createElement("select"), +k=l.appendChild(t.createElement("option"));if(!(!f||!f.length||!h)){c.support={leadingWhitespace:d.firstChild.nodeType===3,tbody:!d.getElementsByTagName("tbody").length,htmlSerialize:!!d.getElementsByTagName("link").length,style:/red/.test(h.getAttribute("style")),hrefNormalized:h.getAttribute("href")==="/a",opacity:/^0.55$/.test(h.style.opacity),cssFloat:!!h.style.cssFloat,checkOn:d.getElementsByTagName("input")[0].value==="on",optSelected:k.selected,deleteExpando:true,optDisabled:false,checkClone:false, +scriptEval:false,noCloneEvent:true,boxModel:null,inlineBlockNeedsLayout:false,shrinkWrapBlocks:false,reliableHiddenOffsets:true};l.disabled=true;c.support.optDisabled=!k.disabled;b.type="text/javascript";try{b.appendChild(t.createTextNode("window."+e+"=1;"))}catch(o){}a.insertBefore(b,a.firstChild);if(E[e]){c.support.scriptEval=true;delete E[e]}try{delete b.test}catch(x){c.support.deleteExpando=false}a.removeChild(b);if(d.attachEvent&&d.fireEvent){d.attachEvent("onclick",function r(){c.support.noCloneEvent= +false;d.detachEvent("onclick",r)});d.cloneNode(true).fireEvent("onclick")}d=t.createElement("div");d.innerHTML="";a=t.createDocumentFragment();a.appendChild(d.firstChild);c.support.checkClone=a.cloneNode(true).cloneNode(true).lastChild.checked;c(function(){var r=t.createElement("div");r.style.width=r.style.paddingLeft="1px";t.body.appendChild(r);c.boxModel=c.support.boxModel=r.offsetWidth===2;if("zoom"in r.style){r.style.display="inline";r.style.zoom= +1;c.support.inlineBlockNeedsLayout=r.offsetWidth===2;r.style.display="";r.innerHTML="
";c.support.shrinkWrapBlocks=r.offsetWidth!==2}r.innerHTML="
t
";var A=r.getElementsByTagName("td");c.support.reliableHiddenOffsets=A[0].offsetHeight===0;A[0].style.display="";A[1].style.display="none";c.support.reliableHiddenOffsets=c.support.reliableHiddenOffsets&&A[0].offsetHeight===0;r.innerHTML="";t.body.removeChild(r).style.display= +"none"});a=function(r){var A=t.createElement("div");r="on"+r;var C=r in A;if(!C){A.setAttribute(r,"return;");C=typeof A[r]==="function"}return C};c.support.submitBubbles=a("submit");c.support.changeBubbles=a("change");a=b=d=f=h=null}})();var ra={},Ja=/^(?:\{.*\}|\[.*\])$/;c.extend({cache:{},uuid:0,expando:"jQuery"+c.now(),noData:{embed:true,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",applet:true},data:function(a,b,d){if(c.acceptData(a)){a=a==E?ra:a;var e=a.nodeType,f=e?a[c.expando]:null,h= +c.cache;if(!(e&&!f&&typeof b==="string"&&d===B)){if(e)f||(a[c.expando]=f=++c.uuid);else h=a;if(typeof b==="object")if(e)h[f]=c.extend(h[f],b);else c.extend(h,b);else if(e&&!h[f])h[f]={};a=e?h[f]:h;if(d!==B)a[b]=d;return typeof b==="string"?a[b]:a}}},removeData:function(a,b){if(c.acceptData(a)){a=a==E?ra:a;var d=a.nodeType,e=d?a[c.expando]:a,f=c.cache,h=d?f[e]:e;if(b){if(h){delete h[b];d&&c.isEmptyObject(h)&&c.removeData(a)}}else if(d&&c.support.deleteExpando)delete a[c.expando];else if(a.removeAttribute)a.removeAttribute(c.expando); +else if(d)delete f[e];else for(var l in a)delete a[l]}},acceptData:function(a){if(a.nodeName){var b=c.noData[a.nodeName.toLowerCase()];if(b)return!(b===true||a.getAttribute("classid")!==b)}return true}});c.fn.extend({data:function(a,b){var d=null;if(typeof a==="undefined"){if(this.length){var e=this[0].attributes,f;d=c.data(this[0]);for(var h=0,l=e.length;h-1)return true;return false},val:function(a){if(!arguments.length){var b=this[0];if(b){if(c.nodeName(b,"option")){var d=b.attributes.value;return!d||d.specified?b.value:b.text}if(c.nodeName(b,"select")){var e=b.selectedIndex;d=[];var f=b.options;b=b.type==="select-one"; +if(e<0)return null;var h=b?e:0;for(e=b?e+1:f.length;h=0;else if(c.nodeName(this,"select")){var A=c.makeArray(r);c("option",this).each(function(){this.selected=c.inArray(c(this).val(),A)>=0});if(!A.length)this.selectedIndex=-1}else this.value=r}})}});c.extend({attrFn:{val:true,css:true,html:true,text:true,data:true,width:true,height:true,offset:true}, +attr:function(a,b,d,e){if(!a||a.nodeType===3||a.nodeType===8)return B;if(e&&b in c.attrFn)return c(a)[b](d);e=a.nodeType!==1||!c.isXMLDoc(a);var f=d!==B;b=e&&c.props[b]||b;var h=Ta.test(b);if((b in a||a[b]!==B)&&e&&!h){if(f){b==="type"&&Ua.test(a.nodeName)&&a.parentNode&&c.error("type property can't be changed");if(d===null)a.nodeType===1&&a.removeAttribute(b);else a[b]=d}if(c.nodeName(a,"form")&&a.getAttributeNode(b))return a.getAttributeNode(b).nodeValue;if(b==="tabIndex")return(b=a.getAttributeNode("tabIndex"))&& +b.specified?b.value:Va.test(a.nodeName)||Wa.test(a.nodeName)&&a.href?0:B;return a[b]}if(!c.support.style&&e&&b==="style"){if(f)a.style.cssText=""+d;return a.style.cssText}f&&a.setAttribute(b,""+d);if(!a.attributes[b]&&a.hasAttribute&&!a.hasAttribute(b))return B;a=!c.support.hrefNormalized&&e&&h?a.getAttribute(b,2):a.getAttribute(b);return a===null?B:a}});var X=/\.(.*)$/,ia=/^(?:textarea|input|select)$/i,La=/\./g,Ma=/ /g,Xa=/[^\w\s.|`]/g,Ya=function(a){return a.replace(Xa,"\\$&")},ua={focusin:0,focusout:0}; +c.event={add:function(a,b,d,e){if(!(a.nodeType===3||a.nodeType===8)){if(c.isWindow(a)&&a!==E&&!a.frameElement)a=E;if(d===false)d=U;else if(!d)return;var f,h;if(d.handler){f=d;d=f.handler}if(!d.guid)d.guid=c.guid++;if(h=c.data(a)){var l=a.nodeType?"events":"__events__",k=h[l],o=h.handle;if(typeof k==="function"){o=k.handle;k=k.events}else if(!k){a.nodeType||(h[l]=h=function(){});h.events=k={}}if(!o)h.handle=o=function(){return typeof c!=="undefined"&&!c.event.triggered?c.event.handle.apply(o.elem, +arguments):B};o.elem=a;b=b.split(" ");for(var x=0,r;l=b[x++];){h=f?c.extend({},f):{handler:d,data:e};if(l.indexOf(".")>-1){r=l.split(".");l=r.shift();h.namespace=r.slice(0).sort().join(".")}else{r=[];h.namespace=""}h.type=l;if(!h.guid)h.guid=d.guid;var A=k[l],C=c.event.special[l]||{};if(!A){A=k[l]=[];if(!C.setup||C.setup.call(a,e,r,o)===false)if(a.addEventListener)a.addEventListener(l,o,false);else a.attachEvent&&a.attachEvent("on"+l,o)}if(C.add){C.add.call(a,h);if(!h.handler.guid)h.handler.guid= +d.guid}A.push(h);c.event.global[l]=true}a=null}}},global:{},remove:function(a,b,d,e){if(!(a.nodeType===3||a.nodeType===8)){if(d===false)d=U;var f,h,l=0,k,o,x,r,A,C,J=a.nodeType?"events":"__events__",w=c.data(a),I=w&&w[J];if(w&&I){if(typeof I==="function"){w=I;I=I.events}if(b&&b.type){d=b.handler;b=b.type}if(!b||typeof b==="string"&&b.charAt(0)==="."){b=b||"";for(f in I)c.event.remove(a,f+b)}else{for(b=b.split(" ");f=b[l++];){r=f;k=f.indexOf(".")<0;o=[];if(!k){o=f.split(".");f=o.shift();x=RegExp("(^|\\.)"+ +c.map(o.slice(0).sort(),Ya).join("\\.(?:.*\\.)?")+"(\\.|$)")}if(A=I[f])if(d){r=c.event.special[f]||{};for(h=e||0;h=0){a.type=f=f.slice(0,-1);a.exclusive=true}if(!d){a.stopPropagation();c.event.global[f]&&c.each(c.cache,function(){this.events&&this.events[f]&&c.event.trigger(a,b,this.handle.elem)})}if(!d||d.nodeType===3||d.nodeType=== +8)return B;a.result=B;a.target=d;b=c.makeArray(b);b.unshift(a)}a.currentTarget=d;(e=d.nodeType?c.data(d,"handle"):(c.data(d,"__events__")||{}).handle)&&e.apply(d,b);e=d.parentNode||d.ownerDocument;try{if(!(d&&d.nodeName&&c.noData[d.nodeName.toLowerCase()]))if(d["on"+f]&&d["on"+f].apply(d,b)===false){a.result=false;a.preventDefault()}}catch(h){}if(!a.isPropagationStopped()&&e)c.event.trigger(a,b,e,true);else if(!a.isDefaultPrevented()){var l;e=a.target;var k=f.replace(X,""),o=c.nodeName(e,"a")&&k=== +"click",x=c.event.special[k]||{};if((!x._default||x._default.call(d,a)===false)&&!o&&!(e&&e.nodeName&&c.noData[e.nodeName.toLowerCase()])){try{if(e[k]){if(l=e["on"+k])e["on"+k]=null;c.event.triggered=true;e[k]()}}catch(r){}if(l)e["on"+k]=l;c.event.triggered=false}}},handle:function(a){var b,d,e,f;d=[];var h=c.makeArray(arguments);a=h[0]=c.event.fix(a||E.event);a.currentTarget=this;b=a.type.indexOf(".")<0&&!a.exclusive;if(!b){e=a.type.split(".");a.type=e.shift();d=e.slice(0).sort();e=RegExp("(^|\\.)"+ +d.join("\\.(?:.*\\.)?")+"(\\.|$)")}a.namespace=a.namespace||d.join(".");f=c.data(this,this.nodeType?"events":"__events__");if(typeof f==="function")f=f.events;d=(f||{})[a.type];if(f&&d){d=d.slice(0);f=0;for(var l=d.length;f-1?c.map(a.options,function(e){return e.selected}).join("-"):"";else if(a.nodeName.toLowerCase()==="select")d=a.selectedIndex;return d},Z=function(a,b){var d=a.target,e,f;if(!(!ia.test(d.nodeName)||d.readOnly)){e=c.data(d,"_change_data");f=xa(d);if(a.type!=="focusout"||d.type!=="radio")c.data(d,"_change_data",f);if(!(e===B||f===e))if(e!=null||f){a.type="change";a.liveFired= +B;return c.event.trigger(a,b,d)}}};c.event.special.change={filters:{focusout:Z,beforedeactivate:Z,click:function(a){var b=a.target,d=b.type;if(d==="radio"||d==="checkbox"||b.nodeName.toLowerCase()==="select")return Z.call(this,a)},keydown:function(a){var b=a.target,d=b.type;if(a.keyCode===13&&b.nodeName.toLowerCase()!=="textarea"||a.keyCode===32&&(d==="checkbox"||d==="radio")||d==="select-multiple")return Z.call(this,a)},beforeactivate:function(a){a=a.target;c.data(a,"_change_data",xa(a))}},setup:function(){if(this.type=== +"file")return false;for(var a in V)c.event.add(this,a+".specialChange",V[a]);return ia.test(this.nodeName)},teardown:function(){c.event.remove(this,".specialChange");return ia.test(this.nodeName)}};V=c.event.special.change.filters;V.focus=V.beforeactivate}t.addEventListener&&c.each({focus:"focusin",blur:"focusout"},function(a,b){function d(e){e=c.event.fix(e);e.type=b;return c.event.trigger(e,null,e.target)}c.event.special[b]={setup:function(){ua[b]++===0&&t.addEventListener(a,d,true)},teardown:function(){--ua[b]=== +0&&t.removeEventListener(a,d,true)}}});c.each(["bind","one"],function(a,b){c.fn[b]=function(d,e,f){if(typeof d==="object"){for(var h in d)this[b](h,e,d[h],f);return this}if(c.isFunction(e)||e===false){f=e;e=B}var l=b==="one"?c.proxy(f,function(o){c(this).unbind(o,l);return f.apply(this,arguments)}):f;if(d==="unload"&&b!=="one")this.one(d,e,f);else{h=0;for(var k=this.length;h0?this.bind(b,d,e):this.trigger(b)};if(c.attrFn)c.attrFn[b]=true});E.attachEvent&&!E.addEventListener&&c(E).bind("unload",function(){for(var a in c.cache)if(c.cache[a].handle)try{c.event.remove(c.cache[a].handle.elem)}catch(b){}}); +(function(){function a(g,i,n,m,p,q){p=0;for(var u=m.length;p0){F=y;break}}y=y[g]}m[p]=F}}}var d=/((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^\[\]]*\]|['"][^'"]*['"]|[^\[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g,e=0,f=Object.prototype.toString,h=false,l=true;[0,0].sort(function(){l=false;return 0});var k=function(g,i,n,m){n=n||[];var p=i=i||t;if(i.nodeType!==1&&i.nodeType!==9)return[];if(!g||typeof g!=="string")return n;var q,u,y,F,M,N=true,O=k.isXML(i),D=[],R=g;do{d.exec("");if(q=d.exec(R)){R=q[3];D.push(q[1]);if(q[2]){F=q[3]; +break}}}while(q);if(D.length>1&&x.exec(g))if(D.length===2&&o.relative[D[0]])u=L(D[0]+D[1],i);else for(u=o.relative[D[0]]?[i]:k(D.shift(),i);D.length;){g=D.shift();if(o.relative[g])g+=D.shift();u=L(g,u)}else{if(!m&&D.length>1&&i.nodeType===9&&!O&&o.match.ID.test(D[0])&&!o.match.ID.test(D[D.length-1])){q=k.find(D.shift(),i,O);i=q.expr?k.filter(q.expr,q.set)[0]:q.set[0]}if(i){q=m?{expr:D.pop(),set:C(m)}:k.find(D.pop(),D.length===1&&(D[0]==="~"||D[0]==="+")&&i.parentNode?i.parentNode:i,O);u=q.expr?k.filter(q.expr, +q.set):q.set;if(D.length>0)y=C(u);else N=false;for(;D.length;){q=M=D.pop();if(o.relative[M])q=D.pop();else M="";if(q==null)q=i;o.relative[M](y,q,O)}}else y=[]}y||(y=u);y||k.error(M||g);if(f.call(y)==="[object Array]")if(N)if(i&&i.nodeType===1)for(g=0;y[g]!=null;g++){if(y[g]&&(y[g]===true||y[g].nodeType===1&&k.contains(i,y[g])))n.push(u[g])}else for(g=0;y[g]!=null;g++)y[g]&&y[g].nodeType===1&&n.push(u[g]);else n.push.apply(n,y);else C(y,n);if(F){k(F,p,n,m);k.uniqueSort(n)}return n};k.uniqueSort=function(g){if(w){h= +l;g.sort(w);if(h)for(var i=1;i0};k.find=function(g,i,n){var m;if(!g)return[];for(var p=0,q=o.order.length;p":function(g,i){var n,m=typeof i==="string",p=0,q=g.length;if(m&&!/\W/.test(i))for(i=i.toLowerCase();p=0))n||m.push(u);else if(n)i[q]=false;return false},ID:function(g){return g[1].replace(/\\/g,"")},TAG:function(g){return g[1].toLowerCase()},CHILD:function(g){if(g[1]==="nth"){var i=/(-?)(\d*)n((?:\+|-)?\d*)/.exec(g[2]==="even"&&"2n"||g[2]==="odd"&&"2n+1"||!/\D/.test(g[2])&&"0n+"+g[2]||g[2]);g[2]=i[1]+(i[2]||1)-0;g[3]=i[3]-0}g[0]=e++;return g},ATTR:function(g,i,n, +m,p,q){i=g[1].replace(/\\/g,"");if(!q&&o.attrMap[i])g[1]=o.attrMap[i];if(g[2]==="~=")g[4]=" "+g[4]+" ";return g},PSEUDO:function(g,i,n,m,p){if(g[1]==="not")if((d.exec(g[3])||"").length>1||/^\w/.test(g[3]))g[3]=k(g[3],null,null,i);else{g=k.filter(g[3],i,n,true^p);n||m.push.apply(m,g);return false}else if(o.match.POS.test(g[0])||o.match.CHILD.test(g[0]))return true;return g},POS:function(g){g.unshift(true);return g}},filters:{enabled:function(g){return g.disabled===false&&g.type!=="hidden"},disabled:function(g){return g.disabled=== +true},checked:function(g){return g.checked===true},selected:function(g){return g.selected===true},parent:function(g){return!!g.firstChild},empty:function(g){return!g.firstChild},has:function(g,i,n){return!!k(n[3],g).length},header:function(g){return/h\d/i.test(g.nodeName)},text:function(g){return"text"===g.type},radio:function(g){return"radio"===g.type},checkbox:function(g){return"checkbox"===g.type},file:function(g){return"file"===g.type},password:function(g){return"password"===g.type},submit:function(g){return"submit"=== +g.type},image:function(g){return"image"===g.type},reset:function(g){return"reset"===g.type},button:function(g){return"button"===g.type||g.nodeName.toLowerCase()==="button"},input:function(g){return/input|select|textarea|button/i.test(g.nodeName)}},setFilters:{first:function(g,i){return i===0},last:function(g,i,n,m){return i===m.length-1},even:function(g,i){return i%2===0},odd:function(g,i){return i%2===1},lt:function(g,i,n){return in[3]-0},nth:function(g,i,n){return n[3]- +0===i},eq:function(g,i,n){return n[3]-0===i}},filter:{PSEUDO:function(g,i,n,m){var p=i[1],q=o.filters[p];if(q)return q(g,n,i,m);else if(p==="contains")return(g.textContent||g.innerText||k.getText([g])||"").indexOf(i[3])>=0;else if(p==="not"){i=i[3];n=0;for(m=i.length;n=0}},ID:function(g,i){return g.nodeType===1&&g.getAttribute("id")===i},TAG:function(g,i){return i==="*"&&g.nodeType===1||g.nodeName.toLowerCase()=== +i},CLASS:function(g,i){return(" "+(g.className||g.getAttribute("class"))+" ").indexOf(i)>-1},ATTR:function(g,i){var n=i[1];n=o.attrHandle[n]?o.attrHandle[n](g):g[n]!=null?g[n]:g.getAttribute(n);var m=n+"",p=i[2],q=i[4];return n==null?p==="!=":p==="="?m===q:p==="*="?m.indexOf(q)>=0:p==="~="?(" "+m+" ").indexOf(q)>=0:!q?m&&n!==false:p==="!="?m!==q:p==="^="?m.indexOf(q)===0:p==="$="?m.substr(m.length-q.length)===q:p==="|="?m===q||m.substr(0,q.length+1)===q+"-":false},POS:function(g,i,n,m){var p=o.setFilters[i[2]]; +if(p)return p(g,n,i,m)}}},x=o.match.POS,r=function(g,i){return"\\"+(i-0+1)},A;for(A in o.match){o.match[A]=RegExp(o.match[A].source+/(?![^\[]*\])(?![^\(]*\))/.source);o.leftMatch[A]=RegExp(/(^(?:.|\r|\n)*?)/.source+o.match[A].source.replace(/\\(\d+)/g,r))}var C=function(g,i){g=Array.prototype.slice.call(g,0);if(i){i.push.apply(i,g);return i}return g};try{Array.prototype.slice.call(t.documentElement.childNodes,0)}catch(J){C=function(g,i){var n=0,m=i||[];if(f.call(g)==="[object Array]")Array.prototype.push.apply(m, +g);else if(typeof g.length==="number")for(var p=g.length;n";n.insertBefore(g,n.firstChild);if(t.getElementById(i)){o.find.ID=function(m,p,q){if(typeof p.getElementById!=="undefined"&&!q)return(p=p.getElementById(m[1]))?p.id===m[1]||typeof p.getAttributeNode!=="undefined"&&p.getAttributeNode("id").nodeValue===m[1]?[p]:B:[]};o.filter.ID=function(m,p){var q=typeof m.getAttributeNode!=="undefined"&&m.getAttributeNode("id");return m.nodeType===1&&q&&q.nodeValue===p}}n.removeChild(g); +n=g=null})();(function(){var g=t.createElement("div");g.appendChild(t.createComment(""));if(g.getElementsByTagName("*").length>0)o.find.TAG=function(i,n){var m=n.getElementsByTagName(i[1]);if(i[1]==="*"){for(var p=[],q=0;m[q];q++)m[q].nodeType===1&&p.push(m[q]);m=p}return m};g.innerHTML="";if(g.firstChild&&typeof g.firstChild.getAttribute!=="undefined"&&g.firstChild.getAttribute("href")!=="#")o.attrHandle.href=function(i){return i.getAttribute("href",2)};g=null})();t.querySelectorAll&& +function(){var g=k,i=t.createElement("div");i.innerHTML="

";if(!(i.querySelectorAll&&i.querySelectorAll(".TEST").length===0)){k=function(m,p,q,u){p=p||t;m=m.replace(/\=\s*([^'"\]]*)\s*\]/g,"='$1']");if(!u&&!k.isXML(p))if(p.nodeType===9)try{return C(p.querySelectorAll(m),q)}catch(y){}else if(p.nodeType===1&&p.nodeName.toLowerCase()!=="object"){var F=p.getAttribute("id"),M=F||"__sizzle__";F||p.setAttribute("id",M);try{return C(p.querySelectorAll("#"+M+" "+m),q)}catch(N){}finally{F|| +p.removeAttribute("id")}}return g(m,p,q,u)};for(var n in g)k[n]=g[n];i=null}}();(function(){var g=t.documentElement,i=g.matchesSelector||g.mozMatchesSelector||g.webkitMatchesSelector||g.msMatchesSelector,n=false;try{i.call(t.documentElement,"[test!='']:sizzle")}catch(m){n=true}if(i)k.matchesSelector=function(p,q){q=q.replace(/\=\s*([^'"\]]*)\s*\]/g,"='$1']");if(!k.isXML(p))try{if(n||!o.match.PSEUDO.test(q)&&!/!=/.test(q))return i.call(p,q)}catch(u){}return k(q,null,null,[p]).length>0}})();(function(){var g= +t.createElement("div");g.innerHTML="
";if(!(!g.getElementsByClassName||g.getElementsByClassName("e").length===0)){g.lastChild.className="e";if(g.getElementsByClassName("e").length!==1){o.order.splice(1,0,"CLASS");o.find.CLASS=function(i,n,m){if(typeof n.getElementsByClassName!=="undefined"&&!m)return n.getElementsByClassName(i[1])};g=null}}})();k.contains=t.documentElement.contains?function(g,i){return g!==i&&(g.contains?g.contains(i):true)}:t.documentElement.compareDocumentPosition? +function(g,i){return!!(g.compareDocumentPosition(i)&16)}:function(){return false};k.isXML=function(g){return(g=(g?g.ownerDocument||g:0).documentElement)?g.nodeName!=="HTML":false};var L=function(g,i){for(var n,m=[],p="",q=i.nodeType?[i]:i;n=o.match.PSEUDO.exec(g);){p+=n[0];g=g.replace(o.match.PSEUDO,"")}g=o.relative[g]?g+"*":g;n=0;for(var u=q.length;n0)for(var h=d;h0},closest:function(a,b){var d=[],e,f,h=this[0];if(c.isArray(a)){var l,k={},o=1;if(h&&a.length){e=0;for(f=a.length;e-1:c(h).is(e))d.push({selector:l,elem:h,level:o})}h= +h.parentNode;o++}}return d}l=cb.test(a)?c(a,b||this.context):null;e=0;for(f=this.length;e-1:c.find.matchesSelector(h,a)){d.push(h);break}else{h=h.parentNode;if(!h||!h.ownerDocument||h===b)break}d=d.length>1?c.unique(d):d;return this.pushStack(d,"closest",a)},index:function(a){if(!a||typeof a==="string")return c.inArray(this[0],a?c(a):this.parent().children());return c.inArray(a.jquery?a[0]:a,this)},add:function(a,b){var d=typeof a==="string"?c(a,b||this.context): +c.makeArray(a),e=c.merge(this.get(),d);return this.pushStack(!d[0]||!d[0].parentNode||d[0].parentNode.nodeType===11||!e[0]||!e[0].parentNode||e[0].parentNode.nodeType===11?e:c.unique(e))},andSelf:function(){return this.add(this.prevObject)}});c.each({parent:function(a){return(a=a.parentNode)&&a.nodeType!==11?a:null},parents:function(a){return c.dir(a,"parentNode")},parentsUntil:function(a,b,d){return c.dir(a,"parentNode",d)},next:function(a){return c.nth(a,2,"nextSibling")},prev:function(a){return c.nth(a, +2,"previousSibling")},nextAll:function(a){return c.dir(a,"nextSibling")},prevAll:function(a){return c.dir(a,"previousSibling")},nextUntil:function(a,b,d){return c.dir(a,"nextSibling",d)},prevUntil:function(a,b,d){return c.dir(a,"previousSibling",d)},siblings:function(a){return c.sibling(a.parentNode.firstChild,a)},children:function(a){return c.sibling(a.firstChild)},contents:function(a){return c.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:c.makeArray(a.childNodes)}},function(a, +b){c.fn[a]=function(d,e){var f=c.map(this,b,d);Za.test(a)||(e=d);if(e&&typeof e==="string")f=c.filter(e,f);f=this.length>1?c.unique(f):f;if((this.length>1||ab.test(e))&&$a.test(a))f=f.reverse();return this.pushStack(f,a,bb.call(arguments).join(","))}});c.extend({filter:function(a,b,d){if(d)a=":not("+a+")";return b.length===1?c.find.matchesSelector(b[0],a)?[b[0]]:[]:c.find.matches(a,b)},dir:function(a,b,d){var e=[];for(a=a[b];a&&a.nodeType!==9&&(d===B||a.nodeType!==1||!c(a).is(d));){a.nodeType===1&& +e.push(a);a=a[b]}return e},nth:function(a,b,d){b=b||1;for(var e=0;a;a=a[d])if(a.nodeType===1&&++e===b)break;return a},sibling:function(a,b){for(var d=[];a;a=a.nextSibling)a.nodeType===1&&a!==b&&d.push(a);return d}});var za=/ jQuery\d+="(?:\d+|null)"/g,$=/^\s+/,Aa=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/ig,Ba=/<([\w:]+)/,db=/\s]+\/)>/g,P={option:[1, +""],legend:[1,"
","
"],thead:[1,"","
"],tr:[2,"","
"],td:[3,"","
"],col:[2,"","
"],area:[1,"",""],_default:[0,"",""]};P.optgroup=P.option;P.tbody=P.tfoot=P.colgroup=P.caption=P.thead;P.th=P.td;if(!c.support.htmlSerialize)P._default=[1,"div
","
"];c.fn.extend({text:function(a){if(c.isFunction(a))return this.each(function(b){var d= +c(this);d.text(a.call(this,b,d.text()))});if(typeof a!=="object"&&a!==B)return this.empty().append((this[0]&&this[0].ownerDocument||t).createTextNode(a));return c.text(this)},wrapAll:function(a){if(c.isFunction(a))return this.each(function(d){c(this).wrapAll(a.call(this,d))});if(this[0]){var b=c(a,this[0].ownerDocument).eq(0).clone(true);this[0].parentNode&&b.insertBefore(this[0]);b.map(function(){for(var d=this;d.firstChild&&d.firstChild.nodeType===1;)d=d.firstChild;return d}).append(this)}return this}, +wrapInner:function(a){if(c.isFunction(a))return this.each(function(b){c(this).wrapInner(a.call(this,b))});return this.each(function(){var b=c(this),d=b.contents();d.length?d.wrapAll(a):b.append(a)})},wrap:function(a){return this.each(function(){c(this).wrapAll(a)})},unwrap:function(){return this.parent().each(function(){c.nodeName(this,"body")||c(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.appendChild(a)})}, +prepend:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.insertBefore(a,this.firstChild)})},before:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this)});else if(arguments.length){var a=c(arguments[0]);a.push.apply(a,this.toArray());return this.pushStack(a,"before",arguments)}},after:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b, +this.nextSibling)});else if(arguments.length){var a=this.pushStack(this,"after",arguments);a.push.apply(a,c(arguments[0]).toArray());return a}},remove:function(a,b){for(var d=0,e;(e=this[d])!=null;d++)if(!a||c.filter(a,[e]).length){if(!b&&e.nodeType===1){c.cleanData(e.getElementsByTagName("*"));c.cleanData([e])}e.parentNode&&e.parentNode.removeChild(e)}return this},empty:function(){for(var a=0,b;(b=this[a])!=null;a++)for(b.nodeType===1&&c.cleanData(b.getElementsByTagName("*"));b.firstChild;)b.removeChild(b.firstChild); +return this},clone:function(a){var b=this.map(function(){if(!c.support.noCloneEvent&&!c.isXMLDoc(this)){var d=this.outerHTML,e=this.ownerDocument;if(!d){d=e.createElement("div");d.appendChild(this.cloneNode(true));d=d.innerHTML}return c.clean([d.replace(za,"").replace(fb,'="$1">').replace($,"")],e)[0]}else return this.cloneNode(true)});if(a===true){na(this,b);na(this.find("*"),b.find("*"))}return b},html:function(a){if(a===B)return this[0]&&this[0].nodeType===1?this[0].innerHTML.replace(za,""):null; +else if(typeof a==="string"&&!Ca.test(a)&&(c.support.leadingWhitespace||!$.test(a))&&!P[(Ba.exec(a)||["",""])[1].toLowerCase()]){a=a.replace(Aa,"<$1>");try{for(var b=0,d=this.length;b0||e.cacheable||this.length>1?h.cloneNode(true):h)}k.length&&c.each(k,Oa)}return this}});c.buildFragment=function(a,b,d){var e,f,h;b=b&&b[0]?b[0].ownerDocument||b[0]:t;if(a.length===1&&typeof a[0]==="string"&&a[0].length<512&&b===t&&!Ca.test(a[0])&&(c.support.checkClone||!Da.test(a[0]))){f=true;if(h=c.fragments[a[0]])if(h!==1)e=h}if(!e){e=b.createDocumentFragment();c.clean(a,b,e,d)}if(f)c.fragments[a[0]]=h?e:1;return{fragment:e,cacheable:f}};c.fragments={};c.each({appendTo:"append", +prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){c.fn[a]=function(d){var e=[];d=c(d);var f=this.length===1&&this[0].parentNode;if(f&&f.nodeType===11&&f.childNodes.length===1&&d.length===1){d[b](this[0]);return this}else{f=0;for(var h=d.length;f0?this.clone(true):this).get();c(d[f])[b](l);e=e.concat(l)}return this.pushStack(e,a,d.selector)}}});c.extend({clean:function(a,b,d,e){b=b||t;if(typeof b.createElement==="undefined")b=b.ownerDocument|| +b[0]&&b[0].ownerDocument||t;for(var f=[],h=0,l;(l=a[h])!=null;h++){if(typeof l==="number")l+="";if(l){if(typeof l==="string"&&!eb.test(l))l=b.createTextNode(l);else if(typeof l==="string"){l=l.replace(Aa,"<$1>");var k=(Ba.exec(l)||["",""])[1].toLowerCase(),o=P[k]||P._default,x=o[0],r=b.createElement("div");for(r.innerHTML=o[1]+l+o[2];x--;)r=r.lastChild;if(!c.support.tbody){x=db.test(l);k=k==="table"&&!x?r.firstChild&&r.firstChild.childNodes:o[1]===""&&!x?r.childNodes:[];for(o=k.length- +1;o>=0;--o)c.nodeName(k[o],"tbody")&&!k[o].childNodes.length&&k[o].parentNode.removeChild(k[o])}!c.support.leadingWhitespace&&$.test(l)&&r.insertBefore(b.createTextNode($.exec(l)[0]),r.firstChild);l=r.childNodes}if(l.nodeType)f.push(l);else f=c.merge(f,l)}}if(d)for(h=0;f[h];h++)if(e&&c.nodeName(f[h],"script")&&(!f[h].type||f[h].type.toLowerCase()==="text/javascript"))e.push(f[h].parentNode?f[h].parentNode.removeChild(f[h]):f[h]);else{f[h].nodeType===1&&f.splice.apply(f,[h+1,0].concat(c.makeArray(f[h].getElementsByTagName("script")))); +d.appendChild(f[h])}return f},cleanData:function(a){for(var b,d,e=c.cache,f=c.event.special,h=c.support.deleteExpando,l=0,k;(k=a[l])!=null;l++)if(!(k.nodeName&&c.noData[k.nodeName.toLowerCase()]))if(d=k[c.expando]){if((b=e[d])&&b.events)for(var o in b.events)f[o]?c.event.remove(k,o):c.removeEvent(k,o,b.handle);if(h)delete k[c.expando];else k.removeAttribute&&k.removeAttribute(c.expando);delete e[d]}}});var Ea=/alpha\([^)]*\)/i,gb=/opacity=([^)]*)/,hb=/-([a-z])/ig,ib=/([A-Z])/g,Fa=/^-?\d+(?:px)?$/i, +jb=/^-?\d/,kb={position:"absolute",visibility:"hidden",display:"block"},Pa=["Left","Right"],Qa=["Top","Bottom"],W,Ga,aa,lb=function(a,b){return b.toUpperCase()};c.fn.css=function(a,b){if(arguments.length===2&&b===B)return this;return c.access(this,a,b,true,function(d,e,f){return f!==B?c.style(d,e,f):c.css(d,e)})};c.extend({cssHooks:{opacity:{get:function(a,b){if(b){var d=W(a,"opacity","opacity");return d===""?"1":d}else return a.style.opacity}}},cssNumber:{zIndex:true,fontWeight:true,opacity:true, +zoom:true,lineHeight:true},cssProps:{"float":c.support.cssFloat?"cssFloat":"styleFloat"},style:function(a,b,d,e){if(!(!a||a.nodeType===3||a.nodeType===8||!a.style)){var f,h=c.camelCase(b),l=a.style,k=c.cssHooks[h];b=c.cssProps[h]||h;if(d!==B){if(!(typeof d==="number"&&isNaN(d)||d==null)){if(typeof d==="number"&&!c.cssNumber[h])d+="px";if(!k||!("set"in k)||(d=k.set(a,d))!==B)try{l[b]=d}catch(o){}}}else{if(k&&"get"in k&&(f=k.get(a,false,e))!==B)return f;return l[b]}}},css:function(a,b,d){var e,f=c.camelCase(b), +h=c.cssHooks[f];b=c.cssProps[f]||f;if(h&&"get"in h&&(e=h.get(a,true,d))!==B)return e;else if(W)return W(a,b,f)},swap:function(a,b,d){var e={},f;for(f in b){e[f]=a.style[f];a.style[f]=b[f]}d.call(a);for(f in b)a.style[f]=e[f]},camelCase:function(a){return a.replace(hb,lb)}});c.curCSS=c.css;c.each(["height","width"],function(a,b){c.cssHooks[b]={get:function(d,e,f){var h;if(e){if(d.offsetWidth!==0)h=oa(d,b,f);else c.swap(d,kb,function(){h=oa(d,b,f)});if(h<=0){h=W(d,b,b);if(h==="0px"&&aa)h=aa(d,b,b); +if(h!=null)return h===""||h==="auto"?"0px":h}if(h<0||h==null){h=d.style[b];return h===""||h==="auto"?"0px":h}return typeof h==="string"?h:h+"px"}},set:function(d,e){if(Fa.test(e)){e=parseFloat(e);if(e>=0)return e+"px"}else return e}}});if(!c.support.opacity)c.cssHooks.opacity={get:function(a,b){return gb.test((b&&a.currentStyle?a.currentStyle.filter:a.style.filter)||"")?parseFloat(RegExp.$1)/100+"":b?"1":""},set:function(a,b){var d=a.style;d.zoom=1;var e=c.isNaN(b)?"":"alpha(opacity="+b*100+")",f= +d.filter||"";d.filter=Ea.test(f)?f.replace(Ea,e):d.filter+" "+e}};if(t.defaultView&&t.defaultView.getComputedStyle)Ga=function(a,b,d){var e;d=d.replace(ib,"-$1").toLowerCase();if(!(b=a.ownerDocument.defaultView))return B;if(b=b.getComputedStyle(a,null)){e=b.getPropertyValue(d);if(e===""&&!c.contains(a.ownerDocument.documentElement,a))e=c.style(a,d)}return e};if(t.documentElement.currentStyle)aa=function(a,b){var d,e,f=a.currentStyle&&a.currentStyle[b],h=a.style;if(!Fa.test(f)&&jb.test(f)){d=h.left; +e=a.runtimeStyle.left;a.runtimeStyle.left=a.currentStyle.left;h.left=b==="fontSize"?"1em":f||0;f=h.pixelLeft+"px";h.left=d;a.runtimeStyle.left=e}return f===""?"auto":f};W=Ga||aa;if(c.expr&&c.expr.filters){c.expr.filters.hidden=function(a){var b=a.offsetHeight;return a.offsetWidth===0&&b===0||!c.support.reliableHiddenOffsets&&(a.style.display||c.css(a,"display"))==="none"};c.expr.filters.visible=function(a){return!c.expr.filters.hidden(a)}}var mb=c.now(),nb=/)<[^<]*)*<\/script>/gi, +ob=/^(?:select|textarea)/i,pb=/^(?:color|date|datetime|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i,qb=/^(?:GET|HEAD)$/,Ra=/\[\]$/,T=/\=\?(&|$)/,ja=/\?/,rb=/([?&])_=[^&]*/,sb=/^(\w+:)?\/\/([^\/?#]+)/,tb=/%20/g,ub=/#.*$/,Ha=c.fn.load;c.fn.extend({load:function(a,b,d){if(typeof a!=="string"&&Ha)return Ha.apply(this,arguments);else if(!this.length)return this;var e=a.indexOf(" ");if(e>=0){var f=a.slice(e,a.length);a=a.slice(0,e)}e="GET";if(b)if(c.isFunction(b)){d=b;b=null}else if(typeof b=== +"object"){b=c.param(b,c.ajaxSettings.traditional);e="POST"}var h=this;c.ajax({url:a,type:e,dataType:"html",data:b,complete:function(l,k){if(k==="success"||k==="notmodified")h.html(f?c("
").append(l.responseText.replace(nb,"")).find(f):l.responseText);d&&h.each(d,[l.responseText,k,l])}});return this},serialize:function(){return c.param(this.serializeArray())},serializeArray:function(){return this.map(function(){return this.elements?c.makeArray(this.elements):this}).filter(function(){return this.name&& +!this.disabled&&(this.checked||ob.test(this.nodeName)||pb.test(this.type))}).map(function(a,b){var d=c(this).val();return d==null?null:c.isArray(d)?c.map(d,function(e){return{name:b.name,value:e}}):{name:b.name,value:d}}).get()}});c.each("ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split(" "),function(a,b){c.fn[b]=function(d){return this.bind(b,d)}});c.extend({get:function(a,b,d,e){if(c.isFunction(b)){e=e||d;d=b;b=null}return c.ajax({type:"GET",url:a,data:b,success:d,dataType:e})}, +getScript:function(a,b){return c.get(a,null,b,"script")},getJSON:function(a,b,d){return c.get(a,b,d,"json")},post:function(a,b,d,e){if(c.isFunction(b)){e=e||d;d=b;b={}}return c.ajax({type:"POST",url:a,data:b,success:d,dataType:e})},ajaxSetup:function(a){c.extend(c.ajaxSettings,a)},ajaxSettings:{url:location.href,global:true,type:"GET",contentType:"application/x-www-form-urlencoded",processData:true,async:true,xhr:function(){return new E.XMLHttpRequest},accepts:{xml:"application/xml, text/xml",html:"text/html", +script:"text/javascript, application/javascript",json:"application/json, text/javascript",text:"text/plain",_default:"*/*"}},ajax:function(a){var b=c.extend(true,{},c.ajaxSettings,a),d,e,f,h=b.type.toUpperCase(),l=qb.test(h);b.url=b.url.replace(ub,"");b.context=a&&a.context!=null?a.context:b;if(b.data&&b.processData&&typeof b.data!=="string")b.data=c.param(b.data,b.traditional);if(b.dataType==="jsonp"){if(h==="GET")T.test(b.url)||(b.url+=(ja.test(b.url)?"&":"?")+(b.jsonp||"callback")+"=?");else if(!b.data|| +!T.test(b.data))b.data=(b.data?b.data+"&":"")+(b.jsonp||"callback")+"=?";b.dataType="json"}if(b.dataType==="json"&&(b.data&&T.test(b.data)||T.test(b.url))){d=b.jsonpCallback||"jsonp"+mb++;if(b.data)b.data=(b.data+"").replace(T,"="+d+"$1");b.url=b.url.replace(T,"="+d+"$1");b.dataType="script";var k=E[d];E[d]=function(m){if(c.isFunction(k))k(m);else{E[d]=B;try{delete E[d]}catch(p){}}f=m;c.handleSuccess(b,w,e,f);c.handleComplete(b,w,e,f);r&&r.removeChild(A)}}if(b.dataType==="script"&&b.cache===null)b.cache= +false;if(b.cache===false&&l){var o=c.now(),x=b.url.replace(rb,"$1_="+o);b.url=x+(x===b.url?(ja.test(b.url)?"&":"?")+"_="+o:"")}if(b.data&&l)b.url+=(ja.test(b.url)?"&":"?")+b.data;b.global&&c.active++===0&&c.event.trigger("ajaxStart");o=(o=sb.exec(b.url))&&(o[1]&&o[1].toLowerCase()!==location.protocol||o[2].toLowerCase()!==location.host);if(b.dataType==="script"&&h==="GET"&&o){var r=t.getElementsByTagName("head")[0]||t.documentElement,A=t.createElement("script");if(b.scriptCharset)A.charset=b.scriptCharset; +A.src=b.url;if(!d){var C=false;A.onload=A.onreadystatechange=function(){if(!C&&(!this.readyState||this.readyState==="loaded"||this.readyState==="complete")){C=true;c.handleSuccess(b,w,e,f);c.handleComplete(b,w,e,f);A.onload=A.onreadystatechange=null;r&&A.parentNode&&r.removeChild(A)}}}r.insertBefore(A,r.firstChild);return B}var J=false,w=b.xhr();if(w){b.username?w.open(h,b.url,b.async,b.username,b.password):w.open(h,b.url,b.async);try{if(b.data!=null&&!l||a&&a.contentType)w.setRequestHeader("Content-Type", +b.contentType);if(b.ifModified){c.lastModified[b.url]&&w.setRequestHeader("If-Modified-Since",c.lastModified[b.url]);c.etag[b.url]&&w.setRequestHeader("If-None-Match",c.etag[b.url])}o||w.setRequestHeader("X-Requested-With","XMLHttpRequest");w.setRequestHeader("Accept",b.dataType&&b.accepts[b.dataType]?b.accepts[b.dataType]+", */*; q=0.01":b.accepts._default)}catch(I){}if(b.beforeSend&&b.beforeSend.call(b.context,w,b)===false){b.global&&c.active--===1&&c.event.trigger("ajaxStop");w.abort();return false}b.global&& +c.triggerGlobal(b,"ajaxSend",[w,b]);var L=w.onreadystatechange=function(m){if(!w||w.readyState===0||m==="abort"){J||c.handleComplete(b,w,e,f);J=true;if(w)w.onreadystatechange=c.noop}else if(!J&&w&&(w.readyState===4||m==="timeout")){J=true;w.onreadystatechange=c.noop;e=m==="timeout"?"timeout":!c.httpSuccess(w)?"error":b.ifModified&&c.httpNotModified(w,b.url)?"notmodified":"success";var p;if(e==="success")try{f=c.httpData(w,b.dataType,b)}catch(q){e="parsererror";p=q}if(e==="success"||e==="notmodified")d|| +c.handleSuccess(b,w,e,f);else c.handleError(b,w,e,p);d||c.handleComplete(b,w,e,f);m==="timeout"&&w.abort();if(b.async)w=null}};try{var g=w.abort;w.abort=function(){w&&Function.prototype.call.call(g,w);L("abort")}}catch(i){}b.async&&b.timeout>0&&setTimeout(function(){w&&!J&&L("timeout")},b.timeout);try{w.send(l||b.data==null?null:b.data)}catch(n){c.handleError(b,w,null,n);c.handleComplete(b,w,e,f)}b.async||L();return w}},param:function(a,b){var d=[],e=function(h,l){l=c.isFunction(l)?l():l;d[d.length]= +encodeURIComponent(h)+"="+encodeURIComponent(l)};if(b===B)b=c.ajaxSettings.traditional;if(c.isArray(a)||a.jquery)c.each(a,function(){e(this.name,this.value)});else for(var f in a)da(f,a[f],b,e);return d.join("&").replace(tb,"+")}});c.extend({active:0,lastModified:{},etag:{},handleError:function(a,b,d,e){a.error&&a.error.call(a.context,b,d,e);a.global&&c.triggerGlobal(a,"ajaxError",[b,a,e])},handleSuccess:function(a,b,d,e){a.success&&a.success.call(a.context,e,d,b);a.global&&c.triggerGlobal(a,"ajaxSuccess", +[b,a])},handleComplete:function(a,b,d){a.complete&&a.complete.call(a.context,b,d);a.global&&c.triggerGlobal(a,"ajaxComplete",[b,a]);a.global&&c.active--===1&&c.event.trigger("ajaxStop")},triggerGlobal:function(a,b,d){(a.context&&a.context.url==null?c(a.context):c.event).trigger(b,d)},httpSuccess:function(a){try{return!a.status&&location.protocol==="file:"||a.status>=200&&a.status<300||a.status===304||a.status===1223}catch(b){}return false},httpNotModified:function(a,b){var d=a.getResponseHeader("Last-Modified"), +e=a.getResponseHeader("Etag");if(d)c.lastModified[b]=d;if(e)c.etag[b]=e;return a.status===304},httpData:function(a,b,d){var e=a.getResponseHeader("content-type")||"",f=b==="xml"||!b&&e.indexOf("xml")>=0;a=f?a.responseXML:a.responseText;f&&a.documentElement.nodeName==="parsererror"&&c.error("parsererror");if(d&&d.dataFilter)a=d.dataFilter(a,b);if(typeof a==="string")if(b==="json"||!b&&e.indexOf("json")>=0)a=c.parseJSON(a);else if(b==="script"||!b&&e.indexOf("javascript")>=0)c.globalEval(a);return a}}); +if(E.ActiveXObject)c.ajaxSettings.xhr=function(){if(E.location.protocol!=="file:")try{return new E.XMLHttpRequest}catch(a){}try{return new E.ActiveXObject("Microsoft.XMLHTTP")}catch(b){}};c.support.ajax=!!c.ajaxSettings.xhr();var ea={},vb=/^(?:toggle|show|hide)$/,wb=/^([+\-]=)?([\d+.\-]+)(.*)$/,ba,pa=[["height","marginTop","marginBottom","paddingTop","paddingBottom"],["width","marginLeft","marginRight","paddingLeft","paddingRight"],["opacity"]];c.fn.extend({show:function(a,b,d){if(a||a===0)return this.animate(S("show", +3),a,b,d);else{d=0;for(var e=this.length;d=0;e--)if(d[e].elem===this){b&&d[e](true);d.splice(e,1)}});b||this.dequeue();return this}});c.each({slideDown:S("show",1),slideUp:S("hide",1),slideToggle:S("toggle",1),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(a,b){c.fn[a]=function(d,e,f){return this.animate(b, +d,e,f)}});c.extend({speed:function(a,b,d){var e=a&&typeof a==="object"?c.extend({},a):{complete:d||!d&&b||c.isFunction(a)&&a,duration:a,easing:d&&b||b&&!c.isFunction(b)&&b};e.duration=c.fx.off?0:typeof e.duration==="number"?e.duration:e.duration in c.fx.speeds?c.fx.speeds[e.duration]:c.fx.speeds._default;e.old=e.complete;e.complete=function(){e.queue!==false&&c(this).dequeue();c.isFunction(e.old)&&e.old.call(this)};return e},easing:{linear:function(a,b,d,e){return d+e*a},swing:function(a,b,d,e){return(-Math.cos(a* +Math.PI)/2+0.5)*e+d}},timers:[],fx:function(a,b,d){this.options=b;this.elem=a;this.prop=d;if(!b.orig)b.orig={}}});c.fx.prototype={update:function(){this.options.step&&this.options.step.call(this.elem,this.now,this);(c.fx.step[this.prop]||c.fx.step._default)(this)},cur:function(){if(this.elem[this.prop]!=null&&(!this.elem.style||this.elem.style[this.prop]==null))return this.elem[this.prop];var a=parseFloat(c.css(this.elem,this.prop));return a&&a>-1E4?a:0},custom:function(a,b,d){function e(l){return f.step(l)} +var f=this,h=c.fx;this.startTime=c.now();this.start=a;this.end=b;this.unit=d||this.unit||"px";this.now=this.start;this.pos=this.state=0;e.elem=this.elem;if(e()&&c.timers.push(e)&&!ba)ba=setInterval(h.tick,h.interval)},show:function(){this.options.orig[this.prop]=c.style(this.elem,this.prop);this.options.show=true;this.custom(this.prop==="width"||this.prop==="height"?1:0,this.cur());c(this.elem).show()},hide:function(){this.options.orig[this.prop]=c.style(this.elem,this.prop);this.options.hide=true; +this.custom(this.cur(),0)},step:function(a){var b=c.now(),d=true;if(a||b>=this.options.duration+this.startTime){this.now=this.end;this.pos=this.state=1;this.update();this.options.curAnim[this.prop]=true;for(var e in this.options.curAnim)if(this.options.curAnim[e]!==true)d=false;if(d){if(this.options.overflow!=null&&!c.support.shrinkWrapBlocks){var f=this.elem,h=this.options;c.each(["","X","Y"],function(k,o){f.style["overflow"+o]=h.overflow[k]})}this.options.hide&&c(this.elem).hide();if(this.options.hide|| +this.options.show)for(var l in this.options.curAnim)c.style(this.elem,l,this.options.orig[l]);this.options.complete.call(this.elem)}return false}else{a=b-this.startTime;this.state=a/this.options.duration;b=this.options.easing||(c.easing.swing?"swing":"linear");this.pos=c.easing[this.options.specialEasing&&this.options.specialEasing[this.prop]||b](this.state,a,0,1,this.options.duration);this.now=this.start+(this.end-this.start)*this.pos;this.update()}return true}};c.extend(c.fx,{tick:function(){for(var a= +c.timers,b=0;b-1;e={};var x={};if(o)x=f.position();l=o?x.top:parseInt(l,10)||0;k=o?x.left:parseInt(k,10)||0;if(c.isFunction(b))b=b.call(a,d,h);if(b.top!=null)e.top=b.top-h.top+l;if(b.left!=null)e.left=b.left-h.left+k;"using"in b?b.using.call(a, +e):f.css(e)}};c.fn.extend({position:function(){if(!this[0])return null;var a=this[0],b=this.offsetParent(),d=this.offset(),e=Ia.test(b[0].nodeName)?{top:0,left:0}:b.offset();d.top-=parseFloat(c.css(a,"marginTop"))||0;d.left-=parseFloat(c.css(a,"marginLeft"))||0;e.top+=parseFloat(c.css(b[0],"borderTopWidth"))||0;e.left+=parseFloat(c.css(b[0],"borderLeftWidth"))||0;return{top:d.top-e.top,left:d.left-e.left}},offsetParent:function(){return this.map(function(){for(var a=this.offsetParent||t.body;a&&!Ia.test(a.nodeName)&& +c.css(a,"position")==="static";)a=a.offsetParent;return a})}});c.each(["Left","Top"],function(a,b){var d="scroll"+b;c.fn[d]=function(e){var f=this[0],h;if(!f)return null;if(e!==B)return this.each(function(){if(h=fa(this))h.scrollTo(!a?e:c(h).scrollLeft(),a?e:c(h).scrollTop());else this[d]=e});else return(h=fa(f))?"pageXOffset"in h?h[a?"pageYOffset":"pageXOffset"]:c.support.boxModel&&h.document.documentElement[d]||h.document.body[d]:f[d]}});c.each(["Height","Width"],function(a,b){var d=b.toLowerCase(); +c.fn["inner"+b]=function(){return this[0]?parseFloat(c.css(this[0],d,"padding")):null};c.fn["outer"+b]=function(e){return this[0]?parseFloat(c.css(this[0],d,e?"margin":"border")):null};c.fn[d]=function(e){var f=this[0];if(!f)return e==null?null:this;if(c.isFunction(e))return this.each(function(l){var k=c(this);k[d](e.call(this,l,k[d]()))});if(c.isWindow(f))return f.document.compatMode==="CSS1Compat"&&f.document.documentElement["client"+b]||f.document.body["client"+b];else if(f.nodeType===9)return Math.max(f.documentElement["client"+ +b],f.body["scroll"+b],f.documentElement["scroll"+b],f.body["offset"+b],f.documentElement["offset"+b]);else if(e===B){f=c.css(f,d);var h=parseFloat(f);return c.isNaN(h)?f:h}else return this.css(d,typeof e==="string"?e:e+"px")}})})(window); diff --git a/backoffice/static/js/jquery-ui-1.8.8.min.js b/backoffice/static/js/jquery-ui-1.8.8.min.js new file mode 100644 index 0000000..16fb949 --- /dev/null +++ b/backoffice/static/js/jquery-ui-1.8.8.min.js @@ -0,0 +1,404 @@ +/*! + * jQuery UI 1.8.8 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI + */ +(function(b,c){function f(g){return!b(g).parents().andSelf().filter(function(){return b.curCSS(this,"visibility")==="hidden"||b.expr.filters.hidden(this)}).length}b.ui=b.ui||{};if(!b.ui.version){b.extend(b.ui,{version:"1.8.8",keyCode:{ALT:18,BACKSPACE:8,CAPS_LOCK:20,COMMA:188,COMMAND:91,COMMAND_LEFT:91,COMMAND_RIGHT:93,CONTROL:17,DELETE:46,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,INSERT:45,LEFT:37,MENU:93,NUMPAD_ADD:107,NUMPAD_DECIMAL:110,NUMPAD_DIVIDE:111,NUMPAD_ENTER:108,NUMPAD_MULTIPLY:106, +NUMPAD_SUBTRACT:109,PAGE_DOWN:34,PAGE_UP:33,PERIOD:190,RIGHT:39,SHIFT:16,SPACE:32,TAB:9,UP:38,WINDOWS:91}});b.fn.extend({_focus:b.fn.focus,focus:function(g,e){return typeof g==="number"?this.each(function(){var a=this;setTimeout(function(){b(a).focus();e&&e.call(a)},g)}):this._focus.apply(this,arguments)},scrollParent:function(){var g;g=b.browser.msie&&/(static|relative)/.test(this.css("position"))||/absolute/.test(this.css("position"))?this.parents().filter(function(){return/(relative|absolute|fixed)/.test(b.curCSS(this, +"position",1))&&/(auto|scroll)/.test(b.curCSS(this,"overflow",1)+b.curCSS(this,"overflow-y",1)+b.curCSS(this,"overflow-x",1))}).eq(0):this.parents().filter(function(){return/(auto|scroll)/.test(b.curCSS(this,"overflow",1)+b.curCSS(this,"overflow-y",1)+b.curCSS(this,"overflow-x",1))}).eq(0);return/fixed/.test(this.css("position"))||!g.length?b(document):g},zIndex:function(g){if(g!==c)return this.css("zIndex",g);if(this.length){g=b(this[0]);for(var e;g.length&&g[0]!==document;){e=g.css("position"); +if(e==="absolute"||e==="relative"||e==="fixed"){e=parseInt(g.css("zIndex"),10);if(!isNaN(e)&&e!==0)return e}g=g.parent()}}return 0},disableSelection:function(){return this.bind((b.support.selectstart?"selectstart":"mousedown")+".ui-disableSelection",function(g){g.preventDefault()})},enableSelection:function(){return this.unbind(".ui-disableSelection")}});b.each(["Width","Height"],function(g,e){function a(j,n,q,l){b.each(d,function(){n-=parseFloat(b.curCSS(j,"padding"+this,true))||0;if(q)n-=parseFloat(b.curCSS(j, +"border"+this+"Width",true))||0;if(l)n-=parseFloat(b.curCSS(j,"margin"+this,true))||0});return n}var d=e==="Width"?["Left","Right"]:["Top","Bottom"],h=e.toLowerCase(),i={innerWidth:b.fn.innerWidth,innerHeight:b.fn.innerHeight,outerWidth:b.fn.outerWidth,outerHeight:b.fn.outerHeight};b.fn["inner"+e]=function(j){if(j===c)return i["inner"+e].call(this);return this.each(function(){b(this).css(h,a(this,j)+"px")})};b.fn["outer"+e]=function(j,n){if(typeof j!=="number")return i["outer"+e].call(this,j);return this.each(function(){b(this).css(h, +a(this,j,true,n)+"px")})}});b.extend(b.expr[":"],{data:function(g,e,a){return!!b.data(g,a[3])},focusable:function(g){var e=g.nodeName.toLowerCase(),a=b.attr(g,"tabindex");if("area"===e){e=g.parentNode;a=e.name;if(!g.href||!a||e.nodeName.toLowerCase()!=="map")return false;g=b("img[usemap=#"+a+"]")[0];return!!g&&f(g)}return(/input|select|textarea|button|object/.test(e)?!g.disabled:"a"==e?g.href||!isNaN(a):!isNaN(a))&&f(g)},tabbable:function(g){var e=b.attr(g,"tabindex");return(isNaN(e)||e>=0)&&b(g).is(":focusable")}}); +b(function(){var g=document.body,e=g.appendChild(e=document.createElement("div"));b.extend(e.style,{minHeight:"100px",height:"auto",padding:0,borderWidth:0});b.support.minHeight=e.offsetHeight===100;b.support.selectstart="onselectstart"in e;g.removeChild(e).style.display="none"});b.extend(b.ui,{plugin:{add:function(g,e,a){g=b.ui[g].prototype;for(var d in a){g.plugins[d]=g.plugins[d]||[];g.plugins[d].push([e,a[d]])}},call:function(g,e,a){if((e=g.plugins[e])&&g.element[0].parentNode)for(var d=0;d0)return true;g[e]=1;a=g[e]>0;g[e]=0;return a},isOverAxis:function(g,e,a){return g>e&&g=9)&&!c.button)return this._mouseUp(c);if(this._mouseStarted){this._mouseDrag(c); +return c.preventDefault()}if(this._mouseDistanceMet(c)&&this._mouseDelayMet(c))(this._mouseStarted=this._mouseStart(this._mouseDownEvent,c)!==false)?this._mouseDrag(c):this._mouseUp(c);return!this._mouseStarted},_mouseUp:function(c){b(document).unbind("mousemove."+this.widgetName,this._mouseMoveDelegate).unbind("mouseup."+this.widgetName,this._mouseUpDelegate);if(this._mouseStarted){this._mouseStarted=false;c.target==this._mouseDownEvent.target&&b.data(c.target,this.widgetName+".preventClickEvent", +true);this._mouseStop(c)}return false},_mouseDistanceMet:function(c){return Math.max(Math.abs(this._mouseDownEvent.pageX-c.pageX),Math.abs(this._mouseDownEvent.pageY-c.pageY))>=this.options.distance},_mouseDelayMet:function(){return this.mouseDelayMet},_mouseStart:function(){},_mouseDrag:function(){},_mouseStop:function(){},_mouseCapture:function(){return true}})})(jQuery); +(function(b){b.widget("ui.draggable",b.ui.mouse,{widgetEventPrefix:"drag",options:{addClasses:true,appendTo:"parent",axis:false,connectToSortable:false,containment:false,cursor:"auto",cursorAt:false,grid:false,handle:false,helper:"original",iframeFix:false,opacity:false,refreshPositions:false,revert:false,revertDuration:500,scope:"default",scroll:true,scrollSensitivity:20,scrollSpeed:20,snap:false,snapMode:"both",snapTolerance:20,stack:false,zIndex:false},_create:function(){if(this.options.helper== +"original"&&!/^(?:r|a|f)/.test(this.element.css("position")))this.element[0].style.position="relative";this.options.addClasses&&this.element.addClass("ui-draggable");this.options.disabled&&this.element.addClass("ui-draggable-disabled");this._mouseInit()},destroy:function(){if(this.element.data("draggable")){this.element.removeData("draggable").unbind(".draggable").removeClass("ui-draggable ui-draggable-dragging ui-draggable-disabled");this._mouseDestroy();return this}},_mouseCapture:function(c){var f= +this.options;if(this.helper||f.disabled||b(c.target).is(".ui-resizable-handle"))return false;this.handle=this._getHandle(c);if(!this.handle)return false;return true},_mouseStart:function(c){var f=this.options;this.helper=this._createHelper(c);this._cacheHelperProportions();if(b.ui.ddmanager)b.ui.ddmanager.current=this;this._cacheMargins();this.cssPosition=this.helper.css("position");this.scrollParent=this.helper.scrollParent();this.offset=this.positionAbs=this.element.offset();this.offset={top:this.offset.top- +this.margins.top,left:this.offset.left-this.margins.left};b.extend(this.offset,{click:{left:c.pageX-this.offset.left,top:c.pageY-this.offset.top},parent:this._getParentOffset(),relative:this._getRelativeOffset()});this.originalPosition=this.position=this._generatePosition(c);this.originalPageX=c.pageX;this.originalPageY=c.pageY;f.cursorAt&&this._adjustOffsetFromHelper(f.cursorAt);f.containment&&this._setContainment();if(this._trigger("start",c)===false){this._clear();return false}this._cacheHelperProportions(); +b.ui.ddmanager&&!f.dropBehaviour&&b.ui.ddmanager.prepareOffsets(this,c);this.helper.addClass("ui-draggable-dragging");this._mouseDrag(c,true);return true},_mouseDrag:function(c,f){this.position=this._generatePosition(c);this.positionAbs=this._convertPositionTo("absolute");if(!f){f=this._uiHash();if(this._trigger("drag",c,f)===false){this._mouseUp({});return false}this.position=f.position}if(!this.options.axis||this.options.axis!="y")this.helper[0].style.left=this.position.left+"px";if(!this.options.axis|| +this.options.axis!="x")this.helper[0].style.top=this.position.top+"px";b.ui.ddmanager&&b.ui.ddmanager.drag(this,c);return false},_mouseStop:function(c){var f=false;if(b.ui.ddmanager&&!this.options.dropBehaviour)f=b.ui.ddmanager.drop(this,c);if(this.dropped){f=this.dropped;this.dropped=false}if(!this.element[0]||!this.element[0].parentNode)return false;if(this.options.revert=="invalid"&&!f||this.options.revert=="valid"&&f||this.options.revert===true||b.isFunction(this.options.revert)&&this.options.revert.call(this.element, +f)){var g=this;b(this.helper).animate(this.originalPosition,parseInt(this.options.revertDuration,10),function(){g._trigger("stop",c)!==false&&g._clear()})}else this._trigger("stop",c)!==false&&this._clear();return false},cancel:function(){this.helper.is(".ui-draggable-dragging")?this._mouseUp({}):this._clear();return this},_getHandle:function(c){var f=!this.options.handle||!b(this.options.handle,this.element).length?true:false;b(this.options.handle,this.element).find("*").andSelf().each(function(){if(this== +c.target)f=true});return f},_createHelper:function(c){var f=this.options;c=b.isFunction(f.helper)?b(f.helper.apply(this.element[0],[c])):f.helper=="clone"?this.element.clone():this.element;c.parents("body").length||c.appendTo(f.appendTo=="parent"?this.element[0].parentNode:f.appendTo);c[0]!=this.element[0]&&!/(fixed|absolute)/.test(c.css("position"))&&c.css("position","absolute");return c},_adjustOffsetFromHelper:function(c){if(typeof c=="string")c=c.split(" ");if(b.isArray(c))c={left:+c[0],top:+c[1]|| +0};if("left"in c)this.offset.click.left=c.left+this.margins.left;if("right"in c)this.offset.click.left=this.helperProportions.width-c.right+this.margins.left;if("top"in c)this.offset.click.top=c.top+this.margins.top;if("bottom"in c)this.offset.click.top=this.helperProportions.height-c.bottom+this.margins.top},_getParentOffset:function(){this.offsetParent=this.helper.offsetParent();var c=this.offsetParent.offset();if(this.cssPosition=="absolute"&&this.scrollParent[0]!=document&&b.ui.contains(this.scrollParent[0], +this.offsetParent[0])){c.left+=this.scrollParent.scrollLeft();c.top+=this.scrollParent.scrollTop()}if(this.offsetParent[0]==document.body||this.offsetParent[0].tagName&&this.offsetParent[0].tagName.toLowerCase()=="html"&&b.browser.msie)c={top:0,left:0};return{top:c.top+(parseInt(this.offsetParent.css("borderTopWidth"),10)||0),left:c.left+(parseInt(this.offsetParent.css("borderLeftWidth"),10)||0)}},_getRelativeOffset:function(){if(this.cssPosition=="relative"){var c=this.element.position();return{top:c.top- +(parseInt(this.helper.css("top"),10)||0)+this.scrollParent.scrollTop(),left:c.left-(parseInt(this.helper.css("left"),10)||0)+this.scrollParent.scrollLeft()}}else return{top:0,left:0}},_cacheMargins:function(){this.margins={left:parseInt(this.element.css("marginLeft"),10)||0,top:parseInt(this.element.css("marginTop"),10)||0}},_cacheHelperProportions:function(){this.helperProportions={width:this.helper.outerWidth(),height:this.helper.outerHeight()}},_setContainment:function(){var c=this.options;if(c.containment== +"parent")c.containment=this.helper[0].parentNode;if(c.containment=="document"||c.containment=="window")this.containment=[(c.containment=="document"?0:b(window).scrollLeft())-this.offset.relative.left-this.offset.parent.left,(c.containment=="document"?0:b(window).scrollTop())-this.offset.relative.top-this.offset.parent.top,(c.containment=="document"?0:b(window).scrollLeft())+b(c.containment=="document"?document:window).width()-this.helperProportions.width-this.margins.left,(c.containment=="document"? +0:b(window).scrollTop())+(b(c.containment=="document"?document:window).height()||document.body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top];if(!/^(document|window|parent)$/.test(c.containment)&&c.containment.constructor!=Array){var f=b(c.containment)[0];if(f){c=b(c.containment).offset();var g=b(f).css("overflow")!="hidden";this.containment=[c.left+(parseInt(b(f).css("borderLeftWidth"),10)||0)+(parseInt(b(f).css("paddingLeft"),10)||0)-this.margins.left,c.top+(parseInt(b(f).css("borderTopWidth"), +10)||0)+(parseInt(b(f).css("paddingTop"),10)||0)-this.margins.top,c.left+(g?Math.max(f.scrollWidth,f.offsetWidth):f.offsetWidth)-(parseInt(b(f).css("borderLeftWidth"),10)||0)-(parseInt(b(f).css("paddingRight"),10)||0)-this.helperProportions.width-this.margins.left,c.top+(g?Math.max(f.scrollHeight,f.offsetHeight):f.offsetHeight)-(parseInt(b(f).css("borderTopWidth"),10)||0)-(parseInt(b(f).css("paddingBottom"),10)||0)-this.helperProportions.height-this.margins.top]}}else if(c.containment.constructor== +Array)this.containment=c.containment},_convertPositionTo:function(c,f){if(!f)f=this.position;c=c=="absolute"?1:-1;var g=this.cssPosition=="absolute"&&!(this.scrollParent[0]!=document&&b.ui.contains(this.scrollParent[0],this.offsetParent[0]))?this.offsetParent:this.scrollParent,e=/(html|body)/i.test(g[0].tagName);return{top:f.top+this.offset.relative.top*c+this.offset.parent.top*c-(b.browser.safari&&b.browser.version<526&&this.cssPosition=="fixed"?0:(this.cssPosition=="fixed"?-this.scrollParent.scrollTop(): +e?0:g.scrollTop())*c),left:f.left+this.offset.relative.left*c+this.offset.parent.left*c-(b.browser.safari&&b.browser.version<526&&this.cssPosition=="fixed"?0:(this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():e?0:g.scrollLeft())*c)}},_generatePosition:function(c){var f=this.options,g=this.cssPosition=="absolute"&&!(this.scrollParent[0]!=document&&b.ui.contains(this.scrollParent[0],this.offsetParent[0]))?this.offsetParent:this.scrollParent,e=/(html|body)/i.test(g[0].tagName),a=c.pageX,d=c.pageY; +if(this.originalPosition){if(this.containment){if(c.pageX-this.offset.click.leftthis.containment[2])a=this.containment[2]+this.offset.click.left;if(c.pageY-this.offset.click.top>this.containment[3])d=this.containment[3]+this.offset.click.top}if(f.grid){d=this.originalPageY+Math.round((d-this.originalPageY)/ +f.grid[1])*f.grid[1];d=this.containment?!(d-this.offset.click.topthis.containment[3])?d:!(d-this.offset.click.topthis.containment[2])?a:!(a-this.offset.click.left
').css({width:this.offsetWidth+"px",height:this.offsetHeight+"px",position:"absolute",opacity:"0.001",zIndex:1E3}).css(b(this).offset()).appendTo("body")})}, +stop:function(){b("div.ui-draggable-iframeFix").each(function(){this.parentNode.removeChild(this)})}});b.ui.plugin.add("draggable","opacity",{start:function(c,f){c=b(f.helper);f=b(this).data("draggable").options;if(c.css("opacity"))f._opacity=c.css("opacity");c.css("opacity",f.opacity)},stop:function(c,f){c=b(this).data("draggable").options;c._opacity&&b(f.helper).css("opacity",c._opacity)}});b.ui.plugin.add("draggable","scroll",{start:function(){var c=b(this).data("draggable");if(c.scrollParent[0]!= +document&&c.scrollParent[0].tagName!="HTML")c.overflowOffset=c.scrollParent.offset()},drag:function(c){var f=b(this).data("draggable"),g=f.options,e=false;if(f.scrollParent[0]!=document&&f.scrollParent[0].tagName!="HTML"){if(!g.axis||g.axis!="x")if(f.overflowOffset.top+f.scrollParent[0].offsetHeight-c.pageY=0;n--){var q=g.snapElements[n].left,l=q+g.snapElements[n].width,k=g.snapElements[n].top,m=k+g.snapElements[n].height;if(q-a=n&&d<=q||h>=n&&h<=q||dq)&&(e>= +i&&e<=j||a>=i&&a<=j||ej);default:return false}};b.ui.ddmanager={current:null,droppables:{"default":[]},prepareOffsets:function(c,f){var g=b.ui.ddmanager.droppables[c.options.scope]||[],e=f?f.type:null,a=(c.currentItem||c.element).find(":data(droppable)").andSelf(),d=0;a:for(;d').css({position:this.element.css("position"),width:this.element.outerWidth(),height:this.element.outerHeight(), +top:this.element.css("top"),left:this.element.css("left")}));this.element=this.element.parent().data("resizable",this.element.data("resizable"));this.elementIsWrapper=true;this.element.css({marginLeft:this.originalElement.css("marginLeft"),marginTop:this.originalElement.css("marginTop"),marginRight:this.originalElement.css("marginRight"),marginBottom:this.originalElement.css("marginBottom")});this.originalElement.css({marginLeft:0,marginTop:0,marginRight:0,marginBottom:0});this.originalResizeStyle= +this.originalElement.css("resize");this.originalElement.css("resize","none");this._proportionallyResizeElements.push(this.originalElement.css({position:"static",zoom:1,display:"block"}));this.originalElement.css({margin:this.originalElement.css("margin")});this._proportionallyResize()}this.handles=e.handles||(!b(".ui-resizable-handle",this.element).length?"e,s,se":{n:".ui-resizable-n",e:".ui-resizable-e",s:".ui-resizable-s",w:".ui-resizable-w",se:".ui-resizable-se",sw:".ui-resizable-sw",ne:".ui-resizable-ne", +nw:".ui-resizable-nw"});if(this.handles.constructor==String){if(this.handles=="all")this.handles="n,e,s,w,se,sw,ne,nw";var a=this.handles.split(",");this.handles={};for(var d=0;d');/sw|se|ne|nw/.test(h)&&i.css({zIndex:++e.zIndex});"se"==h&&i.addClass("ui-icon ui-icon-gripsmall-diagonal-se");this.handles[h]=".ui-resizable-"+h;this.element.append(i)}}this._renderAxis=function(j){j=j||this.element;for(var n in this.handles){if(this.handles[n].constructor== +String)this.handles[n]=b(this.handles[n],this.element).show();if(this.elementIsWrapper&&this.originalElement[0].nodeName.match(/textarea|input|select|button/i)){var q=b(this.handles[n],this.element),l=0;l=/sw|ne|nw|se|n|s/.test(n)?q.outerHeight():q.outerWidth();q=["padding",/ne|nw|n/.test(n)?"Top":/se|sw|s/.test(n)?"Bottom":/^e$/.test(n)?"Right":"Left"].join("");j.css(q,l);this._proportionallyResize()}b(this.handles[n])}};this._renderAxis(this.element);this._handles=b(".ui-resizable-handle",this.element).disableSelection(); +this._handles.mouseover(function(){if(!g.resizing){if(this.className)var j=this.className.match(/ui-resizable-(se|sw|ne|nw|n|e|s|w)/i);g.axis=j&&j[1]?j[1]:"se"}});if(e.autoHide){this._handles.hide();b(this.element).addClass("ui-resizable-autohide").hover(function(){b(this).removeClass("ui-resizable-autohide");g._handles.show()},function(){if(!g.resizing){b(this).addClass("ui-resizable-autohide");g._handles.hide()}})}this._mouseInit()},destroy:function(){this._mouseDestroy();var g=function(a){b(a).removeClass("ui-resizable ui-resizable-disabled ui-resizable-resizing").removeData("resizable").unbind(".resizable").find(".ui-resizable-handle").remove()}; +if(this.elementIsWrapper){g(this.element);var e=this.element;e.after(this.originalElement.css({position:e.css("position"),width:e.outerWidth(),height:e.outerHeight(),top:e.css("top"),left:e.css("left")})).remove()}this.originalElement.css("resize",this.originalResizeStyle);g(this.originalElement);return this},_mouseCapture:function(g){var e=false;for(var a in this.handles)if(b(this.handles[a])[0]==g.target)e=true;return!this.options.disabled&&e},_mouseStart:function(g){var e=this.options,a=this.element.position(), +d=this.element;this.resizing=true;this.documentScroll={top:b(document).scrollTop(),left:b(document).scrollLeft()};if(d.is(".ui-draggable")||/absolute/.test(d.css("position")))d.css({position:"absolute",top:a.top,left:a.left});b.browser.opera&&/relative/.test(d.css("position"))&&d.css({position:"relative",top:"auto",left:"auto"});this._renderProxy();a=c(this.helper.css("left"));var h=c(this.helper.css("top"));if(e.containment){a+=b(e.containment).scrollLeft()||0;h+=b(e.containment).scrollTop()||0}this.offset= +this.helper.offset();this.position={left:a,top:h};this.size=this._helper?{width:d.outerWidth(),height:d.outerHeight()}:{width:d.width(),height:d.height()};this.originalSize=this._helper?{width:d.outerWidth(),height:d.outerHeight()}:{width:d.width(),height:d.height()};this.originalPosition={left:a,top:h};this.sizeDiff={width:d.outerWidth()-d.width(),height:d.outerHeight()-d.height()};this.originalMousePosition={left:g.pageX,top:g.pageY};this.aspectRatio=typeof e.aspectRatio=="number"?e.aspectRatio: +this.originalSize.width/this.originalSize.height||1;e=b(".ui-resizable-"+this.axis).css("cursor");b("body").css("cursor",e=="auto"?this.axis+"-resize":e);d.addClass("ui-resizable-resizing");this._propagate("start",g);return true},_mouseDrag:function(g){var e=this.helper,a=this.originalMousePosition,d=this._change[this.axis];if(!d)return false;a=d.apply(this,[g,g.pageX-a.left||0,g.pageY-a.top||0]);if(this._aspectRatio||g.shiftKey)a=this._updateRatio(a,g);a=this._respectSize(a,g);this._propagate("resize", +g);e.css({top:this.position.top+"px",left:this.position.left+"px",width:this.size.width+"px",height:this.size.height+"px"});!this._helper&&this._proportionallyResizeElements.length&&this._proportionallyResize();this._updateCache(a);this._trigger("resize",g,this.ui());return false},_mouseStop:function(g){this.resizing=false;var e=this.options,a=this;if(this._helper){var d=this._proportionallyResizeElements,h=d.length&&/textarea/i.test(d[0].nodeName);d=h&&b.ui.hasScroll(d[0],"left")?0:a.sizeDiff.height; +h={width:a.size.width-(h?0:a.sizeDiff.width),height:a.size.height-d};d=parseInt(a.element.css("left"),10)+(a.position.left-a.originalPosition.left)||null;var i=parseInt(a.element.css("top"),10)+(a.position.top-a.originalPosition.top)||null;e.animate||this.element.css(b.extend(h,{top:i,left:d}));a.helper.height(a.size.height);a.helper.width(a.size.width);this._helper&&!e.animate&&this._proportionallyResize()}b("body").css("cursor","auto");this.element.removeClass("ui-resizable-resizing");this._propagate("stop", +g);this._helper&&this.helper.remove();return false},_updateCache:function(g){this.offset=this.helper.offset();if(f(g.left))this.position.left=g.left;if(f(g.top))this.position.top=g.top;if(f(g.height))this.size.height=g.height;if(f(g.width))this.size.width=g.width},_updateRatio:function(g){var e=this.position,a=this.size,d=this.axis;if(g.height)g.width=a.height*this.aspectRatio;else if(g.width)g.height=a.width/this.aspectRatio;if(d=="sw"){g.left=e.left+(a.width-g.width);g.top=null}if(d=="nw"){g.top= +e.top+(a.height-g.height);g.left=e.left+(a.width-g.width)}return g},_respectSize:function(g){var e=this.options,a=this.axis,d=f(g.width)&&e.maxWidth&&e.maxWidthg.width,j=f(g.height)&&e.minHeight&&e.minHeight>g.height;if(i)g.width=e.minWidth;if(j)g.height=e.minHeight;if(d)g.width=e.maxWidth;if(h)g.height=e.maxHeight;var n=this.originalPosition.left+this.originalSize.width,q=this.position.top+this.size.height, +l=/sw|nw|w/.test(a);a=/nw|ne|n/.test(a);if(i&&l)g.left=n-e.minWidth;if(d&&l)g.left=n-e.maxWidth;if(j&&a)g.top=q-e.minHeight;if(h&&a)g.top=q-e.maxHeight;if((e=!g.width&&!g.height)&&!g.left&&g.top)g.top=null;else if(e&&!g.top&&g.left)g.left=null;return g},_proportionallyResize:function(){if(this._proportionallyResizeElements.length)for(var g=this.helper||this.element,e=0;e');var e=b.browser.msie&&b.browser.version<7,a=e?1:0;e=e?2:-1;this.helper.addClass(this._helper).css({width:this.element.outerWidth()+e,height:this.element.outerHeight()+e,position:"absolute",left:this.elementOffset.left-a+"px",top:this.elementOffset.top-a+"px",zIndex:++g.zIndex});this.helper.appendTo("body").disableSelection()}else this.helper=this.element},_change:{e:function(g,e){return{width:this.originalSize.width+ +e}},w:function(g,e){return{left:this.originalPosition.left+e,width:this.originalSize.width-e}},n:function(g,e,a){return{top:this.originalPosition.top+a,height:this.originalSize.height-a}},s:function(g,e,a){return{height:this.originalSize.height+a}},se:function(g,e,a){return b.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[g,e,a]))},sw:function(g,e,a){return b.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[g,e,a]))},ne:function(g,e,a){return b.extend(this._change.n.apply(this, +arguments),this._change.e.apply(this,[g,e,a]))},nw:function(g,e,a){return b.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[g,e,a]))}},_propagate:function(g,e){b.ui.plugin.call(this,g,[e,this.ui()]);g!="resize"&&this._trigger(g,e,this.ui())},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize,originalPosition:this.originalPosition}}});b.extend(b.ui.resizable, +{version:"1.8.8"});b.ui.plugin.add("resizable","alsoResize",{start:function(){var g=b(this).data("resizable").options,e=function(a){b(a).each(function(){var d=b(this);d.data("resizable-alsoresize",{width:parseInt(d.width(),10),height:parseInt(d.height(),10),left:parseInt(d.css("left"),10),top:parseInt(d.css("top"),10),position:d.css("position")})})};if(typeof g.alsoResize=="object"&&!g.alsoResize.parentNode)if(g.alsoResize.length){g.alsoResize=g.alsoResize[0];e(g.alsoResize)}else b.each(g.alsoResize, +function(a){e(a)});else e(g.alsoResize)},resize:function(g,e){var a=b(this).data("resizable");g=a.options;var d=a.originalSize,h=a.originalPosition,i={height:a.size.height-d.height||0,width:a.size.width-d.width||0,top:a.position.top-h.top||0,left:a.position.left-h.left||0},j=function(n,q){b(n).each(function(){var l=b(this),k=b(this).data("resizable-alsoresize"),m={},o=q&&q.length?q:l.parents(e.originalElement[0]).length?["width","height"]:["width","height","top","left"];b.each(o,function(p,s){if((p= +(k[s]||0)+(i[s]||0))&&p>=0)m[s]=p||null});if(b.browser.opera&&/relative/.test(l.css("position"))){a._revertToRelativePosition=true;l.css({position:"absolute",top:"auto",left:"auto"})}l.css(m)})};typeof g.alsoResize=="object"&&!g.alsoResize.nodeType?b.each(g.alsoResize,function(n,q){j(n,q)}):j(g.alsoResize)},stop:function(){var g=b(this).data("resizable"),e=g.options,a=function(d){b(d).each(function(){var h=b(this);h.css({position:h.data("resizable-alsoresize").position})})};if(g._revertToRelativePosition){g._revertToRelativePosition= +false;typeof e.alsoResize=="object"&&!e.alsoResize.nodeType?b.each(e.alsoResize,function(d){a(d)}):a(e.alsoResize)}b(this).removeData("resizable-alsoresize")}});b.ui.plugin.add("resizable","animate",{stop:function(g){var e=b(this).data("resizable"),a=e.options,d=e._proportionallyResizeElements,h=d.length&&/textarea/i.test(d[0].nodeName),i=h&&b.ui.hasScroll(d[0],"left")?0:e.sizeDiff.height;h={width:e.size.width-(h?0:e.sizeDiff.width),height:e.size.height-i};i=parseInt(e.element.css("left"),10)+(e.position.left- +e.originalPosition.left)||null;var j=parseInt(e.element.css("top"),10)+(e.position.top-e.originalPosition.top)||null;e.element.animate(b.extend(h,j&&i?{top:j,left:i}:{}),{duration:a.animateDuration,easing:a.animateEasing,step:function(){var n={width:parseInt(e.element.css("width"),10),height:parseInt(e.element.css("height"),10),top:parseInt(e.element.css("top"),10),left:parseInt(e.element.css("left"),10)};d&&d.length&&b(d[0]).css({width:n.width,height:n.height});e._updateCache(n);e._propagate("resize", +g)}})}});b.ui.plugin.add("resizable","containment",{start:function(){var g=b(this).data("resizable"),e=g.element,a=g.options.containment;if(e=a instanceof b?a.get(0):/parent/.test(a)?e.parent().get(0):a){g.containerElement=b(e);if(/document/.test(a)||a==document){g.containerOffset={left:0,top:0};g.containerPosition={left:0,top:0};g.parentData={element:b(document),left:0,top:0,width:b(document).width(),height:b(document).height()||document.body.parentNode.scrollHeight}}else{var d=b(e),h=[];b(["Top", +"Right","Left","Bottom"]).each(function(n,q){h[n]=c(d.css("padding"+q))});g.containerOffset=d.offset();g.containerPosition=d.position();g.containerSize={height:d.innerHeight()-h[3],width:d.innerWidth()-h[1]};a=g.containerOffset;var i=g.containerSize.height,j=g.containerSize.width;j=b.ui.hasScroll(e,"left")?e.scrollWidth:j;i=b.ui.hasScroll(e)?e.scrollHeight:i;g.parentData={element:e,left:a.left,top:a.top,width:j,height:i}}}},resize:function(g){var e=b(this).data("resizable"),a=e.options,d=e.containerOffset, +h=e.position;g=e._aspectRatio||g.shiftKey;var i={top:0,left:0},j=e.containerElement;if(j[0]!=document&&/static/.test(j.css("position")))i=d;if(h.left<(e._helper?d.left:0)){e.size.width+=e._helper?e.position.left-d.left:e.position.left-i.left;if(g)e.size.height=e.size.width/a.aspectRatio;e.position.left=a.helper?d.left:0}if(h.top<(e._helper?d.top:0)){e.size.height+=e._helper?e.position.top-d.top:e.position.top;if(g)e.size.width=e.size.height*a.aspectRatio;e.position.top=e._helper?d.top:0}e.offset.left= +e.parentData.left+e.position.left;e.offset.top=e.parentData.top+e.position.top;a=Math.abs((e._helper?e.offset.left-i.left:e.offset.left-i.left)+e.sizeDiff.width);d=Math.abs((e._helper?e.offset.top-i.top:e.offset.top-d.top)+e.sizeDiff.height);h=e.containerElement.get(0)==e.element.parent().get(0);i=/relative|absolute/.test(e.containerElement.css("position"));if(h&&i)a-=e.parentData.left;if(a+e.size.width>=e.parentData.width){e.size.width=e.parentData.width-a;if(g)e.size.height=e.size.width/e.aspectRatio}if(d+ +e.size.height>=e.parentData.height){e.size.height=e.parentData.height-d;if(g)e.size.width=e.size.height*e.aspectRatio}},stop:function(){var g=b(this).data("resizable"),e=g.options,a=g.containerOffset,d=g.containerPosition,h=g.containerElement,i=b(g.helper),j=i.offset(),n=i.outerWidth()-g.sizeDiff.width;i=i.outerHeight()-g.sizeDiff.height;g._helper&&!e.animate&&/relative/.test(h.css("position"))&&b(this).css({left:j.left-d.left-a.left,width:n,height:i});g._helper&&!e.animate&&/static/.test(h.css("position"))&& +b(this).css({left:j.left-d.left-a.left,width:n,height:i})}});b.ui.plugin.add("resizable","ghost",{start:function(){var g=b(this).data("resizable"),e=g.options,a=g.size;g.ghost=g.originalElement.clone();g.ghost.css({opacity:0.25,display:"block",position:"relative",height:a.height,width:a.width,margin:0,left:0,top:0}).addClass("ui-resizable-ghost").addClass(typeof e.ghost=="string"?e.ghost:"");g.ghost.appendTo(g.helper)},resize:function(){var g=b(this).data("resizable");g.ghost&&g.ghost.css({position:"relative", +height:g.size.height,width:g.size.width})},stop:function(){var g=b(this).data("resizable");g.ghost&&g.helper&&g.helper.get(0).removeChild(g.ghost.get(0))}});b.ui.plugin.add("resizable","grid",{resize:function(){var g=b(this).data("resizable"),e=g.options,a=g.size,d=g.originalSize,h=g.originalPosition,i=g.axis;e.grid=typeof e.grid=="number"?[e.grid,e.grid]:e.grid;var j=Math.round((a.width-d.width)/(e.grid[0]||1))*(e.grid[0]||1);e=Math.round((a.height-d.height)/(e.grid[1]||1))*(e.grid[1]||1);if(/^(se|s|e)$/.test(i)){g.size.width= +d.width+j;g.size.height=d.height+e}else if(/^(ne)$/.test(i)){g.size.width=d.width+j;g.size.height=d.height+e;g.position.top=h.top-e}else{if(/^(sw)$/.test(i)){g.size.width=d.width+j;g.size.height=d.height+e}else{g.size.width=d.width+j;g.size.height=d.height+e;g.position.top=h.top-e}g.position.left=h.left-j}}});var c=function(g){return parseInt(g,10)||0},f=function(g){return!isNaN(parseInt(g,10))}})(jQuery); +(function(b){b.widget("ui.selectable",b.ui.mouse,{options:{appendTo:"body",autoRefresh:true,distance:0,filter:"*",tolerance:"touch"},_create:function(){var c=this;this.element.addClass("ui-selectable");this.dragged=false;var f;this.refresh=function(){f=b(c.options.filter,c.element[0]);f.each(function(){var g=b(this),e=g.offset();b.data(this,"selectable-item",{element:this,$element:g,left:e.left,top:e.top,right:e.left+g.outerWidth(),bottom:e.top+g.outerHeight(),startselected:false,selected:g.hasClass("ui-selected"), +selecting:g.hasClass("ui-selecting"),unselecting:g.hasClass("ui-unselecting")})})};this.refresh();this.selectees=f.addClass("ui-selectee");this._mouseInit();this.helper=b("
")},destroy:function(){this.selectees.removeClass("ui-selectee").removeData("selectable-item");this.element.removeClass("ui-selectable ui-selectable-disabled").removeData("selectable").unbind(".selectable");this._mouseDestroy();return this},_mouseStart:function(c){var f=this;this.opos=[c.pageX, +c.pageY];if(!this.options.disabled){var g=this.options;this.selectees=b(g.filter,this.element[0]);this._trigger("start",c);b(g.appendTo).append(this.helper);this.helper.css({left:c.clientX,top:c.clientY,width:0,height:0});g.autoRefresh&&this.refresh();this.selectees.filter(".ui-selected").each(function(){var e=b.data(this,"selectable-item");e.startselected=true;if(!c.metaKey){e.$element.removeClass("ui-selected");e.selected=false;e.$element.addClass("ui-unselecting");e.unselecting=true;f._trigger("unselecting", +c,{unselecting:e.element})}});b(c.target).parents().andSelf().each(function(){var e=b.data(this,"selectable-item");if(e){var a=!c.metaKey||!e.$element.hasClass("ui-selected");e.$element.removeClass(a?"ui-unselecting":"ui-selected").addClass(a?"ui-selecting":"ui-unselecting");e.unselecting=!a;e.selecting=a;(e.selected=a)?f._trigger("selecting",c,{selecting:e.element}):f._trigger("unselecting",c,{unselecting:e.element});return false}})}},_mouseDrag:function(c){var f=this;this.dragged=true;if(!this.options.disabled){var g= +this.options,e=this.opos[0],a=this.opos[1],d=c.pageX,h=c.pageY;if(e>d){var i=d;d=e;e=i}if(a>h){i=h;h=a;a=i}this.helper.css({left:e,top:a,width:d-e,height:h-a});this.selectees.each(function(){var j=b.data(this,"selectable-item");if(!(!j||j.element==f.element[0])){var n=false;if(g.tolerance=="touch")n=!(j.left>d||j.righth||j.bottome&&j.righta&&j.bottom *",opacity:false,placeholder:false,revert:false,scroll:true,scrollSensitivity:20,scrollSpeed:20,scope:"default",tolerance:"intersect",zIndex:1E3},_create:function(){this.containerCache={};this.element.addClass("ui-sortable"); +this.refresh();this.floating=this.items.length?/left|right/.test(this.items[0].item.css("float")):false;this.offset=this.element.offset();this._mouseInit()},destroy:function(){this.element.removeClass("ui-sortable ui-sortable-disabled").removeData("sortable").unbind(".sortable");this._mouseDestroy();for(var c=this.items.length-1;c>=0;c--)this.items[c].item.removeData("sortable-item");return this},_setOption:function(c,f){if(c==="disabled"){this.options[c]=f;this.widget()[f?"addClass":"removeClass"]("ui-sortable-disabled")}else b.Widget.prototype._setOption.apply(this, +arguments)},_mouseCapture:function(c,f){if(this.reverting)return false;if(this.options.disabled||this.options.type=="static")return false;this._refreshItems(c);var g=null,e=this;b(c.target).parents().each(function(){if(b.data(this,"sortable-item")==e){g=b(this);return false}});if(b.data(c.target,"sortable-item")==e)g=b(c.target);if(!g)return false;if(this.options.handle&&!f){var a=false;b(this.options.handle,g).find("*").andSelf().each(function(){if(this==c.target)a=true});if(!a)return false}this.currentItem= +g;this._removeCurrentsFromItems();return true},_mouseStart:function(c,f,g){f=this.options;var e=this;this.currentContainer=this;this.refreshPositions();this.helper=this._createHelper(c);this._cacheHelperProportions();this._cacheMargins();this.scrollParent=this.helper.scrollParent();this.offset=this.currentItem.offset();this.offset={top:this.offset.top-this.margins.top,left:this.offset.left-this.margins.left};this.helper.css("position","absolute");this.cssPosition=this.helper.css("position");b.extend(this.offset, +{click:{left:c.pageX-this.offset.left,top:c.pageY-this.offset.top},parent:this._getParentOffset(),relative:this._getRelativeOffset()});this.originalPosition=this._generatePosition(c);this.originalPageX=c.pageX;this.originalPageY=c.pageY;f.cursorAt&&this._adjustOffsetFromHelper(f.cursorAt);this.domPosition={prev:this.currentItem.prev()[0],parent:this.currentItem.parent()[0]};this.helper[0]!=this.currentItem[0]&&this.currentItem.hide();this._createPlaceholder();f.containment&&this._setContainment(); +if(f.cursor){if(b("body").css("cursor"))this._storedCursor=b("body").css("cursor");b("body").css("cursor",f.cursor)}if(f.opacity){if(this.helper.css("opacity"))this._storedOpacity=this.helper.css("opacity");this.helper.css("opacity",f.opacity)}if(f.zIndex){if(this.helper.css("zIndex"))this._storedZIndex=this.helper.css("zIndex");this.helper.css("zIndex",f.zIndex)}if(this.scrollParent[0]!=document&&this.scrollParent[0].tagName!="HTML")this.overflowOffset=this.scrollParent.offset();this._trigger("start", +c,this._uiHash());this._preserveHelperProportions||this._cacheHelperProportions();if(!g)for(g=this.containers.length-1;g>=0;g--)this.containers[g]._trigger("activate",c,e._uiHash(this));if(b.ui.ddmanager)b.ui.ddmanager.current=this;b.ui.ddmanager&&!f.dropBehaviour&&b.ui.ddmanager.prepareOffsets(this,c);this.dragging=true;this.helper.addClass("ui-sortable-helper");this._mouseDrag(c);return true},_mouseDrag:function(c){this.position=this._generatePosition(c);this.positionAbs=this._convertPositionTo("absolute"); +if(!this.lastPositionAbs)this.lastPositionAbs=this.positionAbs;if(this.options.scroll){var f=this.options,g=false;if(this.scrollParent[0]!=document&&this.scrollParent[0].tagName!="HTML"){if(this.overflowOffset.top+this.scrollParent[0].offsetHeight-c.pageY=0;f--){g=this.items[f];var e=g.item[0],a=this._intersectsWithPointer(g);if(a)if(e!=this.currentItem[0]&&this.placeholder[a==1?"next":"prev"]()[0]!=e&&!b.ui.contains(this.placeholder[0],e)&&(this.options.type=="semi-dynamic"?!b.ui.contains(this.element[0],e):true)){this.direction=a==1?"down":"up";if(this.options.tolerance=="pointer"||this._intersectsWithSides(g))this._rearrange(c, +g);else break;this._trigger("change",c,this._uiHash());break}}this._contactContainers(c);b.ui.ddmanager&&b.ui.ddmanager.drag(this,c);this._trigger("sort",c,this._uiHash());this.lastPositionAbs=this.positionAbs;return false},_mouseStop:function(c,f){if(c){b.ui.ddmanager&&!this.options.dropBehaviour&&b.ui.ddmanager.drop(this,c);if(this.options.revert){var g=this;f=g.placeholder.offset();g.reverting=true;b(this.helper).animate({left:f.left-this.offset.parent.left-g.margins.left+(this.offsetParent[0]== +document.body?0:this.offsetParent[0].scrollLeft),top:f.top-this.offset.parent.top-g.margins.top+(this.offsetParent[0]==document.body?0:this.offsetParent[0].scrollTop)},parseInt(this.options.revert,10)||500,function(){g._clear(c)})}else this._clear(c,f);return false}},cancel:function(){var c=this;if(this.dragging){this._mouseUp();this.options.helper=="original"?this.currentItem.css(this._storedCSS).removeClass("ui-sortable-helper"):this.currentItem.show();for(var f=this.containers.length-1;f>=0;f--){this.containers[f]._trigger("deactivate", +null,c._uiHash(this));if(this.containers[f].containerCache.over){this.containers[f]._trigger("out",null,c._uiHash(this));this.containers[f].containerCache.over=0}}}this.placeholder[0].parentNode&&this.placeholder[0].parentNode.removeChild(this.placeholder[0]);this.options.helper!="original"&&this.helper&&this.helper[0].parentNode&&this.helper.remove();b.extend(this,{helper:null,dragging:false,reverting:false,_noFinalSort:null});this.domPosition.prev?b(this.domPosition.prev).after(this.currentItem): +b(this.domPosition.parent).prepend(this.currentItem);return this},serialize:function(c){var f=this._getItemsAsjQuery(c&&c.connected),g=[];c=c||{};b(f).each(function(){var e=(b(c.item||this).attr(c.attribute||"id")||"").match(c.expression||/(.+)[-=_](.+)/);if(e)g.push((c.key||e[1]+"[]")+"="+(c.key&&c.expression?e[1]:e[2]))});!g.length&&c.key&&g.push(c.key+"=");return g.join("&")},toArray:function(c){var f=this._getItemsAsjQuery(c&&c.connected),g=[];c=c||{};f.each(function(){g.push(b(c.item||this).attr(c.attribute|| +"id")||"")});return g},_intersectsWith:function(c){var f=this.positionAbs.left,g=f+this.helperProportions.width,e=this.positionAbs.top,a=e+this.helperProportions.height,d=c.left,h=d+c.width,i=c.top,j=i+c.height,n=this.offset.click.top,q=this.offset.click.left;n=e+n>i&&e+nd&&f+qc[this.floating?"width":"height"]?n:d0?"down":"up")}, +_getDragHorizontalDirection:function(){var c=this.positionAbs.left-this.lastPositionAbs.left;return c!=0&&(c>0?"right":"left")},refresh:function(c){this._refreshItems(c);this.refreshPositions();return this},_connectWith:function(){var c=this.options;return c.connectWith.constructor==String?[c.connectWith]:c.connectWith},_getItemsAsjQuery:function(c){var f=[],g=[],e=this._connectWith();if(e&&c)for(c=e.length-1;c>=0;c--)for(var a=b(e[c]),d=a.length-1;d>=0;d--){var h=b.data(a[d],"sortable");if(h&&h!= +this&&!h.options.disabled)g.push([b.isFunction(h.options.items)?h.options.items.call(h.element):b(h.options.items,h.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),h])}g.push([b.isFunction(this.options.items)?this.options.items.call(this.element,null,{options:this.options,item:this.currentItem}):b(this.options.items,this.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),this]);for(c=g.length-1;c>=0;c--)g[c][0].each(function(){f.push(this)});return b(f)},_removeCurrentsFromItems:function(){for(var c= +this.currentItem.find(":data(sortable-item)"),f=0;f=0;a--)for(var d=b(e[a]),h=d.length-1;h>=0;h--){var i=b.data(d[h],"sortable"); +if(i&&i!=this&&!i.options.disabled){g.push([b.isFunction(i.options.items)?i.options.items.call(i.element[0],c,{item:this.currentItem}):b(i.options.items,i.element),i]);this.containers.push(i)}}for(a=g.length-1;a>=0;a--){c=g[a][1];e=g[a][0];h=0;for(d=e.length;h= +0;f--){var g=this.items[f],e=this.options.toleranceElement?b(this.options.toleranceElement,g.item):g.item;if(!c){g.width=e.outerWidth();g.height=e.outerHeight()}e=e.offset();g.left=e.left;g.top=e.top}if(this.options.custom&&this.options.custom.refreshContainers)this.options.custom.refreshContainers.call(this);else for(f=this.containers.length-1;f>=0;f--){e=this.containers[f].element.offset();this.containers[f].containerCache.left=e.left;this.containers[f].containerCache.top=e.top;this.containers[f].containerCache.width= +this.containers[f].element.outerWidth();this.containers[f].containerCache.height=this.containers[f].element.outerHeight()}return this},_createPlaceholder:function(c){var f=c||this,g=f.options;if(!g.placeholder||g.placeholder.constructor==String){var e=g.placeholder;g.placeholder={element:function(){var a=b(document.createElement(f.currentItem[0].nodeName)).addClass(e||f.currentItem[0].className+" ui-sortable-placeholder").removeClass("ui-sortable-helper")[0];if(!e)a.style.visibility="hidden";return a}, +update:function(a,d){if(!(e&&!g.forcePlaceholderSize)){d.height()||d.height(f.currentItem.innerHeight()-parseInt(f.currentItem.css("paddingTop")||0,10)-parseInt(f.currentItem.css("paddingBottom")||0,10));d.width()||d.width(f.currentItem.innerWidth()-parseInt(f.currentItem.css("paddingLeft")||0,10)-parseInt(f.currentItem.css("paddingRight")||0,10))}}}}f.placeholder=b(g.placeholder.element.call(f.element,f.currentItem));f.currentItem.after(f.placeholder);g.placeholder.update(f,f.placeholder)},_contactContainers:function(c){for(var f= +null,g=null,e=this.containers.length-1;e>=0;e--)if(!b.ui.contains(this.currentItem[0],this.containers[e].element[0]))if(this._intersectsWith(this.containers[e].containerCache)){if(!(f&&b.ui.contains(this.containers[e].element[0],f.element[0]))){f=this.containers[e];g=e}}else if(this.containers[e].containerCache.over){this.containers[e]._trigger("out",c,this._uiHash(this));this.containers[e].containerCache.over=0}if(f)if(this.containers.length===1){this.containers[g]._trigger("over",c,this._uiHash(this)); +this.containers[g].containerCache.over=1}else if(this.currentContainer!=this.containers[g]){f=1E4;e=null;for(var a=this.positionAbs[this.containers[g].floating?"left":"top"],d=this.items.length-1;d>=0;d--)if(b.ui.contains(this.containers[g].element[0],this.items[d].item[0])){var h=this.items[d][this.containers[g].floating?"left":"top"];if(Math.abs(h-a)this.containment[2])a=this.containment[2]+this.offset.click.left;if(c.pageY-this.offset.click.top>this.containment[3])d=this.containment[3]+this.offset.click.top}if(f.grid){d=this.originalPageY+Math.round((d-this.originalPageY)/f.grid[1])*f.grid[1];d=this.containment?!(d-this.offset.click.topthis.containment[3])? +d:!(d-this.offset.click.topthis.containment[2])?a:!(a-this.offset.click.left=0;e--)if(b.ui.contains(this.containers[e].element[0],this.currentItem[0])&&!f){g.push(function(a){return function(d){a._trigger("receive", +d,this._uiHash(this))}}.call(this,this.containers[e]));g.push(function(a){return function(d){a._trigger("update",d,this._uiHash(this))}}.call(this,this.containers[e]))}}for(e=this.containers.length-1;e>=0;e--){f||g.push(function(a){return function(d){a._trigger("deactivate",d,this._uiHash(this))}}.call(this,this.containers[e]));if(this.containers[e].containerCache.over){g.push(function(a){return function(d){a._trigger("out",d,this._uiHash(this))}}.call(this,this.containers[e]));this.containers[e].containerCache.over= +0}}this._storedCursor&&b("body").css("cursor",this._storedCursor);this._storedOpacity&&this.helper.css("opacity",this._storedOpacity);if(this._storedZIndex)this.helper.css("zIndex",this._storedZIndex=="auto"?"":this._storedZIndex);this.dragging=false;if(this.cancelHelperRemoval){if(!f){this._trigger("beforeStop",c,this._uiHash());for(e=0;e").addClass("ui-effects-wrapper").css({fontSize:"100%",background:"transparent", +border:"none",margin:0,padding:0});l.wrap(m);m=l.parent();if(l.css("position")=="static"){m.css({position:"relative"});l.css({position:"relative"})}else{b.extend(k,{position:l.css("position"),zIndex:l.css("z-index")});b.each(["top","left","bottom","right"],function(o,p){k[p]=l.css(p);if(isNaN(parseInt(k[p],10)))k[p]="auto"});l.css({position:"relative",top:0,left:0,right:"auto",bottom:"auto"})}return m.css(k).show()},removeWrapper:function(l){if(l.parent().is(".ui-effects-wrapper"))return l.parent().replaceWith(l); +return l},setTransition:function(l,k,m,o){o=o||{};b.each(k,function(p,s){unit=l.cssUnit(s);if(unit[0]>0)o[s]=unit[0]*m+unit[1]});return o}});b.fn.extend({effect:function(l){var k=h.apply(this,arguments),m={options:k[1],duration:k[2],callback:k[3]};k=m.options.mode;var o=b.effects[l];if(b.fx.off||!o)return k?this[k](m.duration,m.callback):this.each(function(){m.callback&&m.callback.call(this)});return o.call(this,m)},_show:b.fn.show,show:function(l){if(i(l))return this._show.apply(this,arguments); +else{var k=h.apply(this,arguments);k[1].mode="show";return this.effect.apply(this,k)}},_hide:b.fn.hide,hide:function(l){if(i(l))return this._hide.apply(this,arguments);else{var k=h.apply(this,arguments);k[1].mode="hide";return this.effect.apply(this,k)}},__toggle:b.fn.toggle,toggle:function(l){if(i(l)||typeof l==="boolean"||b.isFunction(l))return this.__toggle.apply(this,arguments);else{var k=h.apply(this,arguments);k[1].mode="toggle";return this.effect.apply(this,k)}},cssUnit:function(l){var k=this.css(l), +m=[];b.each(["em","px","%","pt"],function(o,p){if(k.indexOf(p)>0)m=[parseFloat(k),p]});return m}});b.easing.jswing=b.easing.swing;b.extend(b.easing,{def:"easeOutQuad",swing:function(l,k,m,o,p){return b.easing[b.easing.def](l,k,m,o,p)},easeInQuad:function(l,k,m,o,p){return o*(k/=p)*k+m},easeOutQuad:function(l,k,m,o,p){return-o*(k/=p)*(k-2)+m},easeInOutQuad:function(l,k,m,o,p){if((k/=p/2)<1)return o/2*k*k+m;return-o/2*(--k*(k-2)-1)+m},easeInCubic:function(l,k,m,o,p){return o*(k/=p)*k*k+m},easeOutCubic:function(l, +k,m,o,p){return o*((k=k/p-1)*k*k+1)+m},easeInOutCubic:function(l,k,m,o,p){if((k/=p/2)<1)return o/2*k*k*k+m;return o/2*((k-=2)*k*k+2)+m},easeInQuart:function(l,k,m,o,p){return o*(k/=p)*k*k*k+m},easeOutQuart:function(l,k,m,o,p){return-o*((k=k/p-1)*k*k*k-1)+m},easeInOutQuart:function(l,k,m,o,p){if((k/=p/2)<1)return o/2*k*k*k*k+m;return-o/2*((k-=2)*k*k*k-2)+m},easeInQuint:function(l,k,m,o,p){return o*(k/=p)*k*k*k*k+m},easeOutQuint:function(l,k,m,o,p){return o*((k=k/p-1)*k*k*k*k+1)+m},easeInOutQuint:function(l, +k,m,o,p){if((k/=p/2)<1)return o/2*k*k*k*k*k+m;return o/2*((k-=2)*k*k*k*k+2)+m},easeInSine:function(l,k,m,o,p){return-o*Math.cos(k/p*(Math.PI/2))+o+m},easeOutSine:function(l,k,m,o,p){return o*Math.sin(k/p*(Math.PI/2))+m},easeInOutSine:function(l,k,m,o,p){return-o/2*(Math.cos(Math.PI*k/p)-1)+m},easeInExpo:function(l,k,m,o,p){return k==0?m:o*Math.pow(2,10*(k/p-1))+m},easeOutExpo:function(l,k,m,o,p){return k==p?m+o:o*(-Math.pow(2,-10*k/p)+1)+m},easeInOutExpo:function(l,k,m,o,p){if(k==0)return m;if(k== +p)return m+o;if((k/=p/2)<1)return o/2*Math.pow(2,10*(k-1))+m;return o/2*(-Math.pow(2,-10*--k)+2)+m},easeInCirc:function(l,k,m,o,p){return-o*(Math.sqrt(1-(k/=p)*k)-1)+m},easeOutCirc:function(l,k,m,o,p){return o*Math.sqrt(1-(k=k/p-1)*k)+m},easeInOutCirc:function(l,k,m,o,p){if((k/=p/2)<1)return-o/2*(Math.sqrt(1-k*k)-1)+m;return o/2*(Math.sqrt(1-(k-=2)*k)+1)+m},easeInElastic:function(l,k,m,o,p){l=1.70158;var s=0,r=o;if(k==0)return m;if((k/=p)==1)return m+o;s||(s=p*0.3);if(r").css({position:"absolute",visibility:"visible",left:-j*(d/g),top:-i*(h/f)}).parent().addClass("ui-effects-explode").css({position:"absolute",overflow:"hidden",width:d/g,height:h/f,left:a.left+j*(d/g)+(c.options.mode=="show"?(j-Math.floor(g/2))*(d/g):0),top:a.top+i*(h/f)+(c.options.mode=="show"?(i-Math.floor(f/2))*(h/f):0),opacity:c.options.mode=="show"?0:1}).animate({left:a.left+j*(d/g)+(c.options.mode=="show"?0:(j-Math.floor(g/2))*(d/g)),top:a.top+ +i*(h/f)+(c.options.mode=="show"?0:(i-Math.floor(f/2))*(h/f)),opacity:c.options.mode=="show"?1:0},c.duration||500);setTimeout(function(){c.options.mode=="show"?e.css({visibility:"visible"}):e.css({visibility:"visible"}).hide();c.callback&&c.callback.apply(e[0]);e.dequeue();b("div.ui-effects-explode").remove()},c.duration||500)})}})(jQuery); +(function(b){b.effects.fade=function(c){return this.queue(function(){var f=b(this),g=b.effects.setMode(f,c.options.mode||"hide");f.animate({opacity:g},{queue:false,duration:c.duration,easing:c.options.easing,complete:function(){c.callback&&c.callback.apply(this,arguments);f.dequeue()}})})}})(jQuery); +(function(b){b.effects.fold=function(c){return this.queue(function(){var f=b(this),g=["position","top","bottom","left","right"],e=b.effects.setMode(f,c.options.mode||"hide"),a=c.options.size||15,d=!!c.options.horizFirst,h=c.duration?c.duration/2:b.fx.speeds._default/2;b.effects.save(f,g);f.show();var i=b.effects.createWrapper(f).css({overflow:"hidden"}),j=e=="show"!=d,n=j?["width","height"]:["height","width"];j=j?[i.width(),i.height()]:[i.height(),i.width()];var q=/([0-9]+)%/.exec(a);if(q)a=parseInt(q[1], +10)/100*j[e=="hide"?0:1];if(e=="show")i.css(d?{height:0,width:a}:{height:a,width:0});d={};q={};d[n[0]]=e=="show"?j[0]:a;q[n[1]]=e=="show"?j[1]:0;i.animate(d,h,c.options.easing).animate(q,h,c.options.easing,function(){e=="hide"&&f.hide();b.effects.restore(f,g);b.effects.removeWrapper(f);c.callback&&c.callback.apply(f[0],arguments);f.dequeue()})})}})(jQuery); +(function(b){b.effects.highlight=function(c){return this.queue(function(){var f=b(this),g=["backgroundImage","backgroundColor","opacity"],e=b.effects.setMode(f,c.options.mode||"show"),a={backgroundColor:f.css("backgroundColor")};if(e=="hide")a.opacity=0;b.effects.save(f,g);f.show().css({backgroundImage:"none",backgroundColor:c.options.color||"#ffff99"}).animate(a,{queue:false,duration:c.duration,easing:c.options.easing,complete:function(){e=="hide"&&f.hide();b.effects.restore(f,g);e=="show"&&!b.support.opacity&& +this.style.removeAttribute("filter");c.callback&&c.callback.apply(this,arguments);f.dequeue()}})})}})(jQuery); +(function(b){b.effects.pulsate=function(c){return this.queue(function(){var f=b(this),g=b.effects.setMode(f,c.options.mode||"show");times=(c.options.times||5)*2-1;duration=c.duration?c.duration/2:b.fx.speeds._default/2;isVisible=f.is(":visible");animateTo=0;if(!isVisible){f.css("opacity",0).show();animateTo=1}if(g=="hide"&&isVisible||g=="show"&&!isVisible)times--;for(g=0;g').appendTo(document.body).addClass(c.options.className).css({top:e.top,left:e.left,height:f.innerHeight(),width:f.innerWidth(),position:"absolute"}).animate(g,c.duration,c.options.easing,function(){a.remove();c.callback&&c.callback.apply(f[0],arguments); +f.dequeue()})})}})(jQuery); +(function(b){b.widget("ui.accordion",{options:{active:0,animated:"slide",autoHeight:true,clearStyle:false,collapsible:false,event:"click",fillSpace:false,header:"> li > :first-child,> :not(li):even",icons:{header:"ui-icon-triangle-1-e",headerSelected:"ui-icon-triangle-1-s"},navigation:false,navigationFilter:function(){return this.href.toLowerCase()===location.href.toLowerCase()}},_create:function(){var c=this,f=c.options;c.running=0;c.element.addClass("ui-accordion ui-widget ui-helper-reset").children("li").addClass("ui-accordion-li-fix");c.headers= +c.element.find(f.header).addClass("ui-accordion-header ui-helper-reset ui-state-default ui-corner-all").bind("mouseenter.accordion",function(){f.disabled||b(this).addClass("ui-state-hover")}).bind("mouseleave.accordion",function(){f.disabled||b(this).removeClass("ui-state-hover")}).bind("focus.accordion",function(){f.disabled||b(this).addClass("ui-state-focus")}).bind("blur.accordion",function(){f.disabled||b(this).removeClass("ui-state-focus")});c.headers.next().addClass("ui-accordion-content ui-helper-reset ui-widget-content ui-corner-bottom"); +if(f.navigation){var g=c.element.find("a").filter(f.navigationFilter).eq(0);if(g.length){var e=g.closest(".ui-accordion-header");c.active=e.length?e:g.closest(".ui-accordion-content").prev()}}c.active=c._findActive(c.active||f.active).addClass("ui-state-default ui-state-active").toggleClass("ui-corner-all").toggleClass("ui-corner-top");c.active.next().addClass("ui-accordion-content-active");c._createIcons();c.resize();c.element.attr("role","tablist");c.headers.attr("role","tab").bind("keydown.accordion", +function(a){return c._keydown(a)}).next().attr("role","tabpanel");c.headers.not(c.active||"").attr({"aria-expanded":"false",tabIndex:-1}).next().hide();c.active.length?c.active.attr({"aria-expanded":"true",tabIndex:0}):c.headers.eq(0).attr("tabIndex",0);b.browser.safari||c.headers.find("a").attr("tabIndex",-1);f.event&&c.headers.bind(f.event.split(" ").join(".accordion ")+".accordion",function(a){c._clickHandler.call(c,a,this);a.preventDefault()})},_createIcons:function(){var c=this.options;if(c.icons){b("").addClass("ui-icon "+ +c.icons.header).prependTo(this.headers);this.active.children(".ui-icon").toggleClass(c.icons.header).toggleClass(c.icons.headerSelected);this.element.addClass("ui-accordion-icons")}},_destroyIcons:function(){this.headers.children(".ui-icon").remove();this.element.removeClass("ui-accordion-icons")},destroy:function(){var c=this.options;this.element.removeClass("ui-accordion ui-widget ui-helper-reset").removeAttr("role");this.headers.unbind(".accordion").removeClass("ui-accordion-header ui-accordion-disabled ui-helper-reset ui-state-default ui-corner-all ui-state-active ui-state-disabled ui-corner-top").removeAttr("role").removeAttr("aria-expanded").removeAttr("tabIndex"); +this.headers.find("a").removeAttr("tabIndex");this._destroyIcons();var f=this.headers.next().css("display","").removeAttr("role").removeClass("ui-helper-reset ui-widget-content ui-corner-bottom ui-accordion-content ui-accordion-content-active ui-accordion-disabled ui-state-disabled");if(c.autoHeight||c.fillHeight)f.css("height","");return b.Widget.prototype.destroy.call(this)},_setOption:function(c,f){b.Widget.prototype._setOption.apply(this,arguments);c=="active"&&this.activate(f);if(c=="icons"){this._destroyIcons(); +f&&this._createIcons()}if(c=="disabled")this.headers.add(this.headers.next())[f?"addClass":"removeClass"]("ui-accordion-disabled ui-state-disabled")},_keydown:function(c){if(!(this.options.disabled||c.altKey||c.ctrlKey)){var f=b.ui.keyCode,g=this.headers.length,e=this.headers.index(c.target),a=false;switch(c.keyCode){case f.RIGHT:case f.DOWN:a=this.headers[(e+1)%g];break;case f.LEFT:case f.UP:a=this.headers[(e-1+g)%g];break;case f.SPACE:case f.ENTER:this._clickHandler({target:c.target},c.target); +c.preventDefault()}if(a){b(c.target).attr("tabIndex",-1);b(a).attr("tabIndex",0);a.focus();return false}return true}},resize:function(){var c=this.options,f;if(c.fillSpace){if(b.browser.msie){var g=this.element.parent().css("overflow");this.element.parent().css("overflow","hidden")}f=this.element.parent().height();b.browser.msie&&this.element.parent().css("overflow",g);this.headers.each(function(){f-=b(this).outerHeight(true)});this.headers.next().each(function(){b(this).height(Math.max(0,f-b(this).innerHeight()+ +b(this).height()))}).css("overflow","auto")}else if(c.autoHeight){f=0;this.headers.next().each(function(){f=Math.max(f,b(this).height("").height())}).height(f)}return this},activate:function(c){this.options.active=c;c=this._findActive(c)[0];this._clickHandler({target:c},c);return this},_findActive:function(c){return c?typeof c==="number"?this.headers.filter(":eq("+c+")"):this.headers.not(this.headers.not(c)):c===false?b([]):this.headers.filter(":eq(0)")},_clickHandler:function(c,f){var g=this.options; +if(!g.disabled)if(c.target){c=b(c.currentTarget||f);f=c[0]===this.active[0];g.active=g.collapsible&&f?false:this.headers.index(c);if(!(this.running||!g.collapsible&&f)){var e=this.active;i=c.next();d=this.active.next();h={options:g,newHeader:f&&g.collapsible?b([]):c,oldHeader:this.active,newContent:f&&g.collapsible?b([]):i,oldContent:d};var a=this.headers.index(this.active[0])>this.headers.index(c[0]);this.active=f?b([]):c;this._toggle(i,d,h,f,a);e.removeClass("ui-state-active ui-corner-top").addClass("ui-state-default ui-corner-all").children(".ui-icon").removeClass(g.icons.headerSelected).addClass(g.icons.header); +if(!f){c.removeClass("ui-state-default ui-corner-all").addClass("ui-state-active ui-corner-top").children(".ui-icon").removeClass(g.icons.header).addClass(g.icons.headerSelected);c.next().addClass("ui-accordion-content-active")}}}else if(g.collapsible){this.active.removeClass("ui-state-active ui-corner-top").addClass("ui-state-default ui-corner-all").children(".ui-icon").removeClass(g.icons.headerSelected).addClass(g.icons.header);this.active.next().addClass("ui-accordion-content-active");var d=this.active.next(), +h={options:g,newHeader:b([]),oldHeader:g.active,newContent:b([]),oldContent:d},i=this.active=b([]);this._toggle(i,d,h)}},_toggle:function(c,f,g,e,a){var d=this,h=d.options;d.toShow=c;d.toHide=f;d.data=g;var i=function(){if(d)return d._completed.apply(d,arguments)};d._trigger("changestart",null,d.data);d.running=f.size()===0?c.size():f.size();if(h.animated){g={};g=h.collapsible&&e?{toShow:b([]),toHide:f,complete:i,down:a,autoHeight:h.autoHeight||h.fillSpace}:{toShow:c,toHide:f,complete:i,down:a,autoHeight:h.autoHeight|| +h.fillSpace};if(!h.proxied)h.proxied=h.animated;if(!h.proxiedDuration)h.proxiedDuration=h.duration;h.animated=b.isFunction(h.proxied)?h.proxied(g):h.proxied;h.duration=b.isFunction(h.proxiedDuration)?h.proxiedDuration(g):h.proxiedDuration;e=b.ui.accordion.animations;var j=h.duration,n=h.animated;if(n&&!e[n]&&!b.easing[n])n="slide";e[n]||(e[n]=function(q){this.slide(q,{easing:n,duration:j||700})});e[n](g)}else{if(h.collapsible&&e)c.toggle();else{f.hide();c.show()}i(true)}f.prev().attr({"aria-expanded":"false", +tabIndex:-1}).blur();c.prev().attr({"aria-expanded":"true",tabIndex:0}).focus()},_completed:function(c){this.running=c?0:--this.running;if(!this.running){this.options.clearStyle&&this.toShow.add(this.toHide).css({height:"",overflow:""});this.toHide.removeClass("ui-accordion-content-active");this.toHide.parent()[0].className=this.toHide.parent()[0].className;this._trigger("change",null,this.data)}}});b.extend(b.ui.accordion,{version:"1.8.8",animations:{slide:function(c,f){c=b.extend({easing:"swing", +duration:300},c,f);if(c.toHide.size())if(c.toShow.size()){var g=c.toShow.css("overflow"),e=0,a={},d={},h;f=c.toShow;h=f[0].style.width;f.width(parseInt(f.parent().width(),10)-parseInt(f.css("paddingLeft"),10)-parseInt(f.css("paddingRight"),10)-(parseInt(f.css("borderLeftWidth"),10)||0)-(parseInt(f.css("borderRightWidth"),10)||0));b.each(["height","paddingTop","paddingBottom"],function(i,j){d[j]="hide";i=(""+b.css(c.toShow[0],j)).match(/^([\d+-.]+)(.*)$/);a[j]={value:i[1],unit:i[2]||"px"}});c.toShow.css({height:0, +overflow:"hidden"}).show();c.toHide.filter(":hidden").each(c.complete).end().filter(":visible").animate(d,{step:function(i,j){if(j.prop=="height")e=j.end-j.start===0?0:(j.now-j.start)/(j.end-j.start);c.toShow[0].style[j.prop]=e*a[j.prop].value+a[j.prop].unit},duration:c.duration,easing:c.easing,complete:function(){c.autoHeight||c.toShow.css("height","");c.toShow.css({width:h,overflow:g});c.complete()}})}else c.toHide.animate({height:"hide",paddingTop:"hide",paddingBottom:"hide"},c);else c.toShow.animate({height:"show", +paddingTop:"show",paddingBottom:"show"},c)},bounceslide:function(c){this.slide(c,{easing:c.down?"easeOutBounce":"swing",duration:c.down?1E3:200})}}})})(jQuery); +(function(b){b.widget("ui.autocomplete",{options:{appendTo:"body",delay:300,minLength:1,position:{my:"left top",at:"left bottom",collision:"none"},source:null},pending:0,_create:function(){var c=this,f=this.element[0].ownerDocument,g;this.element.addClass("ui-autocomplete-input").attr("autocomplete","off").attr({role:"textbox","aria-autocomplete":"list","aria-haspopup":"true"}).bind("keydown.autocomplete",function(e){if(!(c.options.disabled||c.element.attr("readonly"))){g=false;var a=b.ui.keyCode; +switch(e.keyCode){case a.PAGE_UP:c._move("previousPage",e);break;case a.PAGE_DOWN:c._move("nextPage",e);break;case a.UP:c._move("previous",e);e.preventDefault();break;case a.DOWN:c._move("next",e);e.preventDefault();break;case a.ENTER:case a.NUMPAD_ENTER:if(c.menu.active){g=true;e.preventDefault()}case a.TAB:if(!c.menu.active)return;c.menu.select(e);break;case a.ESCAPE:c.element.val(c.term);c.close(e);break;default:clearTimeout(c.searching);c.searching=setTimeout(function(){if(c.term!=c.element.val()){c.selectedItem= +null;c.search(null,e)}},c.options.delay);break}}}).bind("keypress.autocomplete",function(e){if(g){g=false;e.preventDefault()}}).bind("focus.autocomplete",function(){if(!c.options.disabled){c.selectedItem=null;c.previous=c.element.val()}}).bind("blur.autocomplete",function(e){if(!c.options.disabled){clearTimeout(c.searching);c.closing=setTimeout(function(){c.close(e);c._change(e)},150)}});this._initSource();this.response=function(){return c._response.apply(c,arguments)};this.menu=b("
    ").addClass("ui-autocomplete").appendTo(b(this.options.appendTo|| +"body",f)[0]).mousedown(function(e){var a=c.menu.element[0];b(e.target).closest(".ui-menu-item").length||setTimeout(function(){b(document).one("mousedown",function(d){d.target!==c.element[0]&&d.target!==a&&!b.ui.contains(a,d.target)&&c.close()})},1);setTimeout(function(){clearTimeout(c.closing)},13)}).menu({focus:function(e,a){a=a.item.data("item.autocomplete");false!==c._trigger("focus",e,{item:a})&&/^key/.test(e.originalEvent.type)&&c.element.val(a.value)},selected:function(e,a){var d=a.item.data("item.autocomplete"), +h=c.previous;if(c.element[0]!==f.activeElement){c.element.focus();c.previous=h;setTimeout(function(){c.previous=h;c.selectedItem=d},1)}false!==c._trigger("select",e,{item:d})&&c.element.val(d.value);c.term=c.element.val();c.close(e);c.selectedItem=d},blur:function(){c.menu.element.is(":visible")&&c.element.val()!==c.term&&c.element.val(c.term)}}).zIndex(this.element.zIndex()+1).css({top:0,left:0}).hide().data("menu");b.fn.bgiframe&&this.menu.element.bgiframe()},destroy:function(){this.element.removeClass("ui-autocomplete-input").removeAttr("autocomplete").removeAttr("role").removeAttr("aria-autocomplete").removeAttr("aria-haspopup"); +this.menu.element.remove();b.Widget.prototype.destroy.call(this)},_setOption:function(c,f){b.Widget.prototype._setOption.apply(this,arguments);c==="source"&&this._initSource();if(c==="appendTo")this.menu.element.appendTo(b(f||"body",this.element[0].ownerDocument)[0]);c==="disabled"&&f&&this.xhr&&this.xhr.abort()},_initSource:function(){var c=this,f,g;if(b.isArray(this.options.source)){f=this.options.source;this.source=function(e,a){a(b.ui.autocomplete.filter(f,e.term))}}else if(typeof this.options.source=== +"string"){g=this.options.source;this.source=function(e,a){c.xhr&&c.xhr.abort();c.xhr=b.ajax({url:g,data:e,dataType:"json",success:function(d,h,i){i===c.xhr&&a(d);c.xhr=null},error:function(d){d===c.xhr&&a([]);c.xhr=null}})}}else this.source=this.options.source},search:function(c,f){c=c!=null?c:this.element.val();this.term=this.element.val();if(c.length").data("item.autocomplete",f).append(b("").text(f.label)).appendTo(c)},_move:function(c,f){if(this.menu.element.is(":visible"))if(this.menu.first()&&/^previous/.test(c)||this.menu.last()&&/^next/.test(c)){this.element.val(this.term);this.menu.deactivate()}else this.menu[c](f); +else this.search(null,f)},widget:function(){return this.menu.element}});b.extend(b.ui.autocomplete,{escapeRegex:function(c){return c.replace(/[-[\]{}()*+?.,\\^$|#\s]/g,"\\$&")},filter:function(c,f){var g=new RegExp(b.ui.autocomplete.escapeRegex(f),"i");return b.grep(c,function(e){return g.test(e.label||e.value||e)})}})})(jQuery); +(function(b){b.widget("ui.menu",{_create:function(){var c=this;this.element.addClass("ui-menu ui-widget ui-widget-content ui-corner-all").attr({role:"listbox","aria-activedescendant":"ui-active-menuitem"}).click(function(f){if(b(f.target).closest(".ui-menu-item a").length){f.preventDefault();c.select(f)}});this.refresh()},refresh:function(){var c=this;this.element.children("li:not(.ui-menu-item):has(a)").addClass("ui-menu-item").attr("role","menuitem").children("a").addClass("ui-corner-all").attr("tabindex", +-1).mouseenter(function(f){c.activate(f,b(this).parent())}).mouseleave(function(){c.deactivate()})},activate:function(c,f){this.deactivate();if(this.hasScroll()){var g=f.offset().top-this.element.offset().top,e=this.element.attr("scrollTop"),a=this.element.height();if(g<0)this.element.attr("scrollTop",e+g);else g>=a&&this.element.attr("scrollTop",e+g-a+f.height())}this.active=f.eq(0).children("a").addClass("ui-state-hover").attr("id","ui-active-menuitem").end();this._trigger("focus",c,{item:f})}, +deactivate:function(){if(this.active){this.active.children("a").removeClass("ui-state-hover").removeAttr("id");this._trigger("blur");this.active=null}},next:function(c){this.move("next",".ui-menu-item:first",c)},previous:function(c){this.move("prev",".ui-menu-item:last",c)},first:function(){return this.active&&!this.active.prevAll(".ui-menu-item").length},last:function(){return this.active&&!this.active.nextAll(".ui-menu-item").length},move:function(c,f,g){if(this.active){c=this.active[c+"All"](".ui-menu-item").eq(0); +c.length?this.activate(g,c):this.activate(g,this.element.children(f))}else this.activate(g,this.element.children(f))},nextPage:function(c){if(this.hasScroll())if(!this.active||this.last())this.activate(c,this.element.children(".ui-menu-item:first"));else{var f=this.active.offset().top,g=this.element.height(),e=this.element.children(".ui-menu-item").filter(function(){var a=b(this).offset().top-f-g+b(this).height();return a<10&&a>-10});e.length||(e=this.element.children(".ui-menu-item:last"));this.activate(c, +e)}else this.activate(c,this.element.children(".ui-menu-item").filter(!this.active||this.last()?":first":":last"))},previousPage:function(c){if(this.hasScroll())if(!this.active||this.first())this.activate(c,this.element.children(".ui-menu-item:last"));else{var f=this.active.offset().top,g=this.element.height();result=this.element.children(".ui-menu-item").filter(function(){var e=b(this).offset().top-f+g-b(this).height();return e<10&&e>-10});result.length||(result=this.element.children(".ui-menu-item:first")); +this.activate(c,result)}else this.activate(c,this.element.children(".ui-menu-item").filter(!this.active||this.first()?":last":":first"))},hasScroll:function(){return this.element.height()").addClass("ui-button-text").html(this.options.label).appendTo(e.empty()).text(),d=this.options.icons,h=d.primary&&d.secondary;if(d.primary||d.secondary){e.addClass("ui-button-text-icon"+(h?"s":d.primary?"-primary":"-secondary"));d.primary&&e.prepend("");d.secondary&&e.append("");if(!this.options.text){e.addClass(h?"ui-button-icons-only":"ui-button-icon-only").removeClass("ui-button-text-icons ui-button-text-icon-primary ui-button-text-icon-secondary"); +this.hasTitle||e.attr("title",a)}}else e.addClass("ui-button-text-only")}}});b.widget("ui.buttonset",{options:{items:":button, :submit, :reset, :checkbox, :radio, a, :data(button)"},_create:function(){this.element.addClass("ui-buttonset")},_init:function(){this.refresh()},_setOption:function(e,a){e==="disabled"&&this.buttons.button("option",e,a);b.Widget.prototype._setOption.apply(this,arguments)},refresh:function(){this.buttons=this.element.find(this.options.items).filter(":ui-button").button("refresh").end().not(":ui-button").button().end().map(function(){return b(this).button("widget")[0]}).removeClass("ui-corner-all ui-corner-left ui-corner-right").filter(":first").addClass("ui-corner-left").end().filter(":last").addClass("ui-corner-right").end().end()}, +destroy:function(){this.element.removeClass("ui-buttonset");this.buttons.map(function(){return b(this).button("widget")[0]}).removeClass("ui-corner-left ui-corner-right").end().button("destroy");b.Widget.prototype.destroy.call(this)}})})(jQuery); +(function(b,c){function f(){this.debug=false;this._curInst=null;this._keyEvent=false;this._disabledInputs=[];this._inDialog=this._datepickerShowing=false;this._mainDivId="ui-datepicker-div";this._inlineClass="ui-datepicker-inline";this._appendClass="ui-datepicker-append";this._triggerClass="ui-datepicker-trigger";this._dialogClass="ui-datepicker-dialog";this._disableClass="ui-datepicker-disabled";this._unselectableClass="ui-datepicker-unselectable";this._currentClass="ui-datepicker-current-day";this._dayOverClass= +"ui-datepicker-days-cell-over";this.regional=[];this.regional[""]={closeText:"Done",prevText:"Prev",nextText:"Next",currentText:"Today",monthNames:["January","February","March","April","May","June","July","August","September","October","November","December"],monthNamesShort:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"],dayNames:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],dayNamesShort:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],dayNamesMin:["Su", +"Mo","Tu","We","Th","Fr","Sa"],weekHeader:"Wk",dateFormat:"mm/dd/yy",firstDay:0,isRTL:false,showMonthAfterYear:false,yearSuffix:""};this._defaults={showOn:"focus",showAnim:"fadeIn",showOptions:{},defaultDate:null,appendText:"",buttonText:"...",buttonImage:"",buttonImageOnly:false,hideIfNoPrevNext:false,navigationAsDateFormat:false,gotoCurrent:false,changeMonth:false,changeYear:false,yearRange:"c-10:c+10",showOtherMonths:false,selectOtherMonths:false,showWeek:false,calculateWeek:this.iso8601Week,shortYearCutoff:"+10", +minDate:null,maxDate:null,duration:"fast",beforeShowDay:null,beforeShow:null,onSelect:null,onChangeMonthYear:null,onClose:null,numberOfMonths:1,showCurrentAtPos:0,stepMonths:1,stepBigMonths:12,altField:"",altFormat:"",constrainInput:true,showButtonPanel:false,autoSize:false};b.extend(this._defaults,this.regional[""]);this.dpDiv=b('
    ')}function g(a,d){b.extend(a,d);for(var h in d)if(d[h]== +null||d[h]==c)a[h]=d[h];return a}b.extend(b.ui,{datepicker:{version:"1.8.8"}});var e=(new Date).getTime();b.extend(f.prototype,{markerClassName:"hasDatepicker",log:function(){this.debug&&console.log.apply("",arguments)},_widgetDatepicker:function(){return this.dpDiv},setDefaults:function(a){g(this._defaults,a||{});return this},_attachDatepicker:function(a,d){var h=null;for(var i in this._defaults){var j=a.getAttribute("date:"+i);if(j){h=h||{};try{h[i]=eval(j)}catch(n){h[i]=j}}}i=a.nodeName.toLowerCase(); +j=i=="div"||i=="span";if(!a.id){this.uuid+=1;a.id="dp"+this.uuid}var q=this._newInst(b(a),j);q.settings=b.extend({},d||{},h||{});if(i=="input")this._connectDatepicker(a,q);else j&&this._inlineDatepicker(a,q)},_newInst:function(a,d){return{id:a[0].id.replace(/([^A-Za-z0-9_-])/g,"\\\\$1"),input:a,selectedDay:0,selectedMonth:0,selectedYear:0,drawMonth:0,drawYear:0,inline:d,dpDiv:!d?this.dpDiv:b('
    ')}}, +_connectDatepicker:function(a,d){var h=b(a);d.append=b([]);d.trigger=b([]);if(!h.hasClass(this.markerClassName)){this._attachments(h,d);h.addClass(this.markerClassName).keydown(this._doKeyDown).keypress(this._doKeyPress).keyup(this._doKeyUp).bind("setData.datepicker",function(i,j,n){d.settings[j]=n}).bind("getData.datepicker",function(i,j){return this._get(d,j)});this._autoSize(d);b.data(a,"datepicker",d)}},_attachments:function(a,d){var h=this._get(d,"appendText"),i=this._get(d,"isRTL");d.append&& +d.append.remove();if(h){d.append=b(''+h+"");a[i?"before":"after"](d.append)}a.unbind("focus",this._showDatepicker);d.trigger&&d.trigger.remove();h=this._get(d,"showOn");if(h=="focus"||h=="both")a.focus(this._showDatepicker);if(h=="button"||h=="both"){h=this._get(d,"buttonText");var j=this._get(d,"buttonImage");d.trigger=b(this._get(d,"buttonImageOnly")?b("").addClass(this._triggerClass).attr({src:j,alt:h,title:h}):b('').addClass(this._triggerClass).html(j== +""?h:b("").attr({src:j,alt:h,title:h})));a[i?"before":"after"](d.trigger);d.trigger.click(function(){b.datepicker._datepickerShowing&&b.datepicker._lastInput==a[0]?b.datepicker._hideDatepicker():b.datepicker._showDatepicker(a[0]);return false})}},_autoSize:function(a){if(this._get(a,"autoSize")&&!a.inline){var d=new Date(2009,11,20),h=this._get(a,"dateFormat");if(h.match(/[DM]/)){var i=function(j){for(var n=0,q=0,l=0;ln){n=j[l].length;q=l}return q};d.setMonth(i(this._get(a, +h.match(/MM/)?"monthNames":"monthNamesShort")));d.setDate(i(this._get(a,h.match(/DD/)?"dayNames":"dayNamesShort"))+20-d.getDay())}a.input.attr("size",this._formatDate(a,d).length)}},_inlineDatepicker:function(a,d){var h=b(a);if(!h.hasClass(this.markerClassName)){h.addClass(this.markerClassName).append(d.dpDiv).bind("setData.datepicker",function(i,j,n){d.settings[j]=n}).bind("getData.datepicker",function(i,j){return this._get(d,j)});b.data(a,"datepicker",d);this._setDate(d,this._getDefaultDate(d), +true);this._updateDatepicker(d);this._updateAlternate(d);d.dpDiv.show()}},_dialogDatepicker:function(a,d,h,i,j){a=this._dialogInst;if(!a){this.uuid+=1;this._dialogInput=b('');this._dialogInput.keydown(this._doKeyDown);b("body").append(this._dialogInput);a=this._dialogInst=this._newInst(this._dialogInput,false);a.settings={};b.data(this._dialogInput[0],"datepicker",a)}g(a.settings,i||{}); +d=d&&d.constructor==Date?this._formatDate(a,d):d;this._dialogInput.val(d);this._pos=j?j.length?j:[j.pageX,j.pageY]:null;if(!this._pos)this._pos=[document.documentElement.clientWidth/2-100+(document.documentElement.scrollLeft||document.body.scrollLeft),document.documentElement.clientHeight/2-150+(document.documentElement.scrollTop||document.body.scrollTop)];this._dialogInput.css("left",this._pos[0]+20+"px").css("top",this._pos[1]+"px");a.settings.onSelect=h;this._inDialog=true;this.dpDiv.addClass(this._dialogClass); +this._showDatepicker(this._dialogInput[0]);b.blockUI&&b.blockUI(this.dpDiv);b.data(this._dialogInput[0],"datepicker",a);return this},_destroyDatepicker:function(a){var d=b(a),h=b.data(a,"datepicker");if(d.hasClass(this.markerClassName)){var i=a.nodeName.toLowerCase();b.removeData(a,"datepicker");if(i=="input"){h.append.remove();h.trigger.remove();d.removeClass(this.markerClassName).unbind("focus",this._showDatepicker).unbind("keydown",this._doKeyDown).unbind("keypress",this._doKeyPress).unbind("keyup", +this._doKeyUp)}else if(i=="div"||i=="span")d.removeClass(this.markerClassName).empty()}},_enableDatepicker:function(a){var d=b(a),h=b.data(a,"datepicker");if(d.hasClass(this.markerClassName)){var i=a.nodeName.toLowerCase();if(i=="input"){a.disabled=false;h.trigger.filter("button").each(function(){this.disabled=false}).end().filter("img").css({opacity:"1.0",cursor:""})}else if(i=="div"||i=="span")d.children("."+this._inlineClass).children().removeClass("ui-state-disabled");this._disabledInputs=b.map(this._disabledInputs, +function(j){return j==a?null:j})}},_disableDatepicker:function(a){var d=b(a),h=b.data(a,"datepicker");if(d.hasClass(this.markerClassName)){var i=a.nodeName.toLowerCase();if(i=="input"){a.disabled=true;h.trigger.filter("button").each(function(){this.disabled=true}).end().filter("img").css({opacity:"0.5",cursor:"default"})}else if(i=="div"||i=="span")d.children("."+this._inlineClass).children().addClass("ui-state-disabled");this._disabledInputs=b.map(this._disabledInputs,function(j){return j==a?null: +j});this._disabledInputs[this._disabledInputs.length]=a}},_isDisabledDatepicker:function(a){if(!a)return false;for(var d=0;d-1}},_doKeyUp:function(a){a=b.datepicker._getInst(a.target);if(a.input.val()!=a.lastVal)try{if(b.datepicker.parseDate(b.datepicker._get(a,"dateFormat"),a.input?a.input.val():null,b.datepicker._getFormatConfig(a))){b.datepicker._setDateFromField(a);b.datepicker._updateAlternate(a);b.datepicker._updateDatepicker(a)}}catch(d){b.datepicker.log(d)}return true}, +_showDatepicker:function(a){a=a.target||a;if(a.nodeName.toLowerCase()!="input")a=b("input",a.parentNode)[0];if(!(b.datepicker._isDisabledDatepicker(a)||b.datepicker._lastInput==a)){var d=b.datepicker._getInst(a);b.datepicker._curInst&&b.datepicker._curInst!=d&&b.datepicker._curInst.dpDiv.stop(true,true);var h=b.datepicker._get(d,"beforeShow");g(d.settings,h?h.apply(a,[a,d]):{});d.lastVal=null;b.datepicker._lastInput=a;b.datepicker._setDateFromField(d);if(b.datepicker._inDialog)a.value="";if(!b.datepicker._pos){b.datepicker._pos= +b.datepicker._findPos(a);b.datepicker._pos[1]+=a.offsetHeight}var i=false;b(a).parents().each(function(){i|=b(this).css("position")=="fixed";return!i});if(i&&b.browser.opera){b.datepicker._pos[0]-=document.documentElement.scrollLeft;b.datepicker._pos[1]-=document.documentElement.scrollTop}h={left:b.datepicker._pos[0],top:b.datepicker._pos[1]};b.datepicker._pos=null;d.dpDiv.empty();d.dpDiv.css({position:"absolute",display:"block",top:"-1000px"});b.datepicker._updateDatepicker(d);h=b.datepicker._checkOffset(d, +h,i);d.dpDiv.css({position:b.datepicker._inDialog&&b.blockUI?"static":i?"fixed":"absolute",display:"none",left:h.left+"px",top:h.top+"px"});if(!d.inline){h=b.datepicker._get(d,"showAnim");var j=b.datepicker._get(d,"duration"),n=function(){b.datepicker._datepickerShowing=true;var q=d.dpDiv.find("iframe.ui-datepicker-cover");if(q.length){var l=b.datepicker._getBorders(d.dpDiv);q.css({left:-l[0],top:-l[1],width:d.dpDiv.outerWidth(),height:d.dpDiv.outerHeight()})}};d.dpDiv.zIndex(b(a).zIndex()+1);b.effects&& +b.effects[h]?d.dpDiv.show(h,b.datepicker._get(d,"showOptions"),j,n):d.dpDiv[h||"show"](h?j:null,n);if(!h||!j)n();d.input.is(":visible")&&!d.input.is(":disabled")&&d.input.focus();b.datepicker._curInst=d}}},_updateDatepicker:function(a){var d=this,h=b.datepicker._getBorders(a.dpDiv);a.dpDiv.empty().append(this._generateHTML(a));var i=a.dpDiv.find("iframe.ui-datepicker-cover");i.length&&i.css({left:-h[0],top:-h[1],width:a.dpDiv.outerWidth(),height:a.dpDiv.outerHeight()});a.dpDiv.find("button, .ui-datepicker-prev, .ui-datepicker-next, .ui-datepicker-calendar td a").bind("mouseout", +function(){b(this).removeClass("ui-state-hover");this.className.indexOf("ui-datepicker-prev")!=-1&&b(this).removeClass("ui-datepicker-prev-hover");this.className.indexOf("ui-datepicker-next")!=-1&&b(this).removeClass("ui-datepicker-next-hover")}).bind("mouseover",function(){if(!d._isDisabledDatepicker(a.inline?a.dpDiv.parent()[0]:a.input[0])){b(this).parents(".ui-datepicker-calendar").find("a").removeClass("ui-state-hover");b(this).addClass("ui-state-hover");this.className.indexOf("ui-datepicker-prev")!= +-1&&b(this).addClass("ui-datepicker-prev-hover");this.className.indexOf("ui-datepicker-next")!=-1&&b(this).addClass("ui-datepicker-next-hover")}}).end().find("."+this._dayOverClass+" a").trigger("mouseover").end();h=this._getNumberOfMonths(a);i=h[1];i>1?a.dpDiv.addClass("ui-datepicker-multi-"+i).css("width",17*i+"em"):a.dpDiv.removeClass("ui-datepicker-multi-2 ui-datepicker-multi-3 ui-datepicker-multi-4").width("");a.dpDiv[(h[0]!=1||h[1]!=1?"add":"remove")+"Class"]("ui-datepicker-multi");a.dpDiv[(this._get(a, +"isRTL")?"add":"remove")+"Class"]("ui-datepicker-rtl");a==b.datepicker._curInst&&b.datepicker._datepickerShowing&&a.input&&a.input.is(":visible")&&!a.input.is(":disabled")&&a.input.focus();if(a.yearshtml){var j=a.yearshtml;setTimeout(function(){j===a.yearshtml&&a.dpDiv.find("select.ui-datepicker-year:first").replaceWith(a.yearshtml);j=a.yearshtml=null},0)}},_getBorders:function(a){var d=function(h){return{thin:1,medium:2,thick:3}[h]||h};return[parseFloat(d(a.css("border-left-width"))),parseFloat(d(a.css("border-top-width")))]}, +_checkOffset:function(a,d,h){var i=a.dpDiv.outerWidth(),j=a.dpDiv.outerHeight(),n=a.input?a.input.outerWidth():0,q=a.input?a.input.outerHeight():0,l=document.documentElement.clientWidth+b(document).scrollLeft(),k=document.documentElement.clientHeight+b(document).scrollTop();d.left-=this._get(a,"isRTL")?i-n:0;d.left-=h&&d.left==a.input.offset().left?b(document).scrollLeft():0;d.top-=h&&d.top==a.input.offset().top+q?b(document).scrollTop():0;d.left-=Math.min(d.left,d.left+i>l&&l>i?Math.abs(d.left+i- +l):0);d.top-=Math.min(d.top,d.top+j>k&&k>j?Math.abs(j+q):0);return d},_findPos:function(a){for(var d=this._get(this._getInst(a),"isRTL");a&&(a.type=="hidden"||a.nodeType!=1);)a=a[d?"previousSibling":"nextSibling"];a=b(a).offset();return[a.left,a.top]},_hideDatepicker:function(a){var d=this._curInst;if(!(!d||a&&d!=b.data(a,"datepicker")))if(this._datepickerShowing){a=this._get(d,"showAnim");var h=this._get(d,"duration"),i=function(){b.datepicker._tidyDialog(d);this._curInst=null};b.effects&&b.effects[a]? +d.dpDiv.hide(a,b.datepicker._get(d,"showOptions"),h,i):d.dpDiv[a=="slideDown"?"slideUp":a=="fadeIn"?"fadeOut":"hide"](a?h:null,i);a||i();if(a=this._get(d,"onClose"))a.apply(d.input?d.input[0]:null,[d.input?d.input.val():"",d]);this._datepickerShowing=false;this._lastInput=null;if(this._inDialog){this._dialogInput.css({position:"absolute",left:"0",top:"-100px"});if(b.blockUI){b.unblockUI();b("body").append(this.dpDiv)}}this._inDialog=false}},_tidyDialog:function(a){a.dpDiv.removeClass(this._dialogClass).unbind(".ui-datepicker-calendar")}, +_checkExternalClick:function(a){if(b.datepicker._curInst){a=b(a.target);a[0].id!=b.datepicker._mainDivId&&a.parents("#"+b.datepicker._mainDivId).length==0&&!a.hasClass(b.datepicker.markerClassName)&&!a.hasClass(b.datepicker._triggerClass)&&b.datepicker._datepickerShowing&&!(b.datepicker._inDialog&&b.blockUI)&&b.datepicker._hideDatepicker()}},_adjustDate:function(a,d,h){a=b(a);var i=this._getInst(a[0]);if(!this._isDisabledDatepicker(a[0])){this._adjustInstDate(i,d+(h=="M"?this._get(i,"showCurrentAtPos"): +0),h);this._updateDatepicker(i)}},_gotoToday:function(a){a=b(a);var d=this._getInst(a[0]);if(this._get(d,"gotoCurrent")&&d.currentDay){d.selectedDay=d.currentDay;d.drawMonth=d.selectedMonth=d.currentMonth;d.drawYear=d.selectedYear=d.currentYear}else{var h=new Date;d.selectedDay=h.getDate();d.drawMonth=d.selectedMonth=h.getMonth();d.drawYear=d.selectedYear=h.getFullYear()}this._notifyChange(d);this._adjustDate(a)},_selectMonthYear:function(a,d,h){a=b(a);var i=this._getInst(a[0]);i._selectingMonthYear= +false;i["selected"+(h=="M"?"Month":"Year")]=i["draw"+(h=="M"?"Month":"Year")]=parseInt(d.options[d.selectedIndex].value,10);this._notifyChange(i);this._adjustDate(a)},_clickMonthYear:function(a){var d=this._getInst(b(a)[0]);d.input&&d._selectingMonthYear&&setTimeout(function(){d.input.focus()},0);d._selectingMonthYear=!d._selectingMonthYear},_selectDay:function(a,d,h,i){var j=b(a);if(!(b(i).hasClass(this._unselectableClass)||this._isDisabledDatepicker(j[0]))){j=this._getInst(j[0]);j.selectedDay=j.currentDay= +b("a",i).html();j.selectedMonth=j.currentMonth=d;j.selectedYear=j.currentYear=h;this._selectDate(a,this._formatDate(j,j.currentDay,j.currentMonth,j.currentYear))}},_clearDate:function(a){a=b(a);this._getInst(a[0]);this._selectDate(a,"")},_selectDate:function(a,d){a=this._getInst(b(a)[0]);d=d!=null?d:this._formatDate(a);a.input&&a.input.val(d);this._updateAlternate(a);var h=this._get(a,"onSelect");if(h)h.apply(a.input?a.input[0]:null,[d,a]);else a.input&&a.input.trigger("change");if(a.inline)this._updateDatepicker(a); +else{this._hideDatepicker();this._lastInput=a.input[0];typeof a.input[0]!="object"&&a.input.focus();this._lastInput=null}},_updateAlternate:function(a){var d=this._get(a,"altField");if(d){var h=this._get(a,"altFormat")||this._get(a,"dateFormat"),i=this._getDate(a),j=this.formatDate(h,i,this._getFormatConfig(a));b(d).each(function(){b(this).val(j)})}},noWeekends:function(a){a=a.getDay();return[a>0&&a<6,""]},iso8601Week:function(a){a=new Date(a.getTime());a.setDate(a.getDate()+4-(a.getDay()||7));var d= +a.getTime();a.setMonth(0);a.setDate(1);return Math.floor(Math.round((d-a)/864E5)/7)+1},parseDate:function(a,d,h){if(a==null||d==null)throw"Invalid arguments";d=typeof d=="object"?d.toString():d+"";if(d=="")return null;for(var i=(h?h.shortYearCutoff:null)||this._defaults.shortYearCutoff,j=(h?h.dayNamesShort:null)||this._defaults.dayNamesShort,n=(h?h.dayNames:null)||this._defaults.dayNames,q=(h?h.monthNamesShort:null)||this._defaults.monthNamesShort,l=(h?h.monthNames:null)||this._defaults.monthNames, +k=h=-1,m=-1,o=-1,p=false,s=function(x){(x=y+1-1){k=1;m=o;do{i=this._getDaysInMonth(h,k-1);if(m<=i)break;k++;m-=i}while(1)}B=this._daylightSavingAdjust(new Date(h,k-1,m));if(B.getFullYear()!=h||B.getMonth()+1!=k||B.getDate()!=m)throw"Invalid date";return B},ATOM:"yy-mm-dd",COOKIE:"D, dd M yy",ISO_8601:"yy-mm-dd",RFC_822:"D, d M y",RFC_850:"DD, dd-M-y",RFC_1036:"D, d M y", +RFC_1123:"D, d M yy",RFC_2822:"D, d M yy",RSS:"D, d M y",TICKS:"!",TIMESTAMP:"@",W3C:"yy-mm-dd",_ticksTo1970:(718685+Math.floor(492.5)-Math.floor(19.7)+Math.floor(4.925))*24*60*60*1E7,formatDate:function(a,d,h){if(!d)return"";var i=(h?h.dayNamesShort:null)||this._defaults.dayNamesShort,j=(h?h.dayNames:null)||this._defaults.dayNames,n=(h?h.monthNamesShort:null)||this._defaults.monthNamesShort;h=(h?h.monthNames:null)||this._defaults.monthNames;var q=function(s){(s=p+112?a.getHours()+2:0);return a},_setDate:function(a,d,h){var i=!d,j=a.selectedMonth,n=a.selectedYear;d=this._restrictMinMax(a,this._determineDate(a,d,new Date));a.selectedDay= +a.currentDay=d.getDate();a.drawMonth=a.selectedMonth=a.currentMonth=d.getMonth();a.drawYear=a.selectedYear=a.currentYear=d.getFullYear();if((j!=a.selectedMonth||n!=a.selectedYear)&&!h)this._notifyChange(a);this._adjustInstDate(a);if(a.input)a.input.val(i?"":this._formatDate(a))},_getDate:function(a){return!a.currentYear||a.input&&a.input.val()==""?null:this._daylightSavingAdjust(new Date(a.currentYear,a.currentMonth,a.currentDay))},_generateHTML:function(a){var d=new Date;d=this._daylightSavingAdjust(new Date(d.getFullYear(), +d.getMonth(),d.getDate()));var h=this._get(a,"isRTL"),i=this._get(a,"showButtonPanel"),j=this._get(a,"hideIfNoPrevNext"),n=this._get(a,"navigationAsDateFormat"),q=this._getNumberOfMonths(a),l=this._get(a,"showCurrentAtPos"),k=this._get(a,"stepMonths"),m=q[0]!=1||q[1]!=1,o=this._daylightSavingAdjust(!a.currentDay?new Date(9999,9,9):new Date(a.currentYear,a.currentMonth,a.currentDay)),p=this._getMinMaxDate(a,"min"),s=this._getMinMaxDate(a,"max");l=a.drawMonth-l;var r=a.drawYear;if(l<0){l+=12;r--}if(s){var u= +this._daylightSavingAdjust(new Date(s.getFullYear(),s.getMonth()-q[0]*q[1]+1,s.getDate()));for(u=p&&uu;){l--;if(l<0){l=11;r--}}}a.drawMonth=l;a.drawYear=r;u=this._get(a,"prevText");u=!n?u:this.formatDate(u,this._daylightSavingAdjust(new Date(r,l-k,1)),this._getFormatConfig(a));u=this._canAdjustMonth(a,-1,r,l)?''+u+"":j?"":''+u+"";var v=this._get(a,"nextText");v=!n?v:this.formatDate(v,this._daylightSavingAdjust(new Date(r,l+k,1)),this._getFormatConfig(a));j=this._canAdjustMonth(a,+1,r,l)?''+v+"":j?"":''+v+"";k=this._get(a,"currentText");v=this._get(a,"gotoCurrent")&&a.currentDay?o:d;k=!n?k:this.formatDate(k,v,this._getFormatConfig(a));n=!a.inline?'":"";i=i?'
    '+(h?n:"")+(this._isInRange(a,v)?'":"")+(h?"":n)+"
    ":"";n=parseInt(this._get(a,"firstDay"),10);n=isNaN(n)?0:n;k=this._get(a,"showWeek");v=this._get(a,"dayNames");this._get(a,"dayNamesShort");var w=this._get(a,"dayNamesMin"),y= +this._get(a,"monthNames"),B=this._get(a,"monthNamesShort"),x=this._get(a,"beforeShowDay"),C=this._get(a,"showOtherMonths"),J=this._get(a,"selectOtherMonths");this._get(a,"calculateWeek");for(var M=this._getDefaultDate(a),K="",G=0;G1)switch(H){case 0:D+=" ui-datepicker-group-first";A=" ui-corner-"+(h?"right":"left");break;case q[1]- +1:D+=" ui-datepicker-group-last";A=" ui-corner-"+(h?"left":"right");break;default:D+=" ui-datepicker-group-middle";A="";break}D+='">'}D+='
    '+(/all|left/.test(A)&&G==0?h?j:u:"")+(/all|right/.test(A)&&G==0?h?u:j:"")+this._generateMonthYearHeader(a,l,r,p,s,G>0||H>0,y,B)+'
    ';var E=k?'":"";for(A=0;A<7;A++){var z= +(A+n)%7;E+="=5?' class="ui-datepicker-week-end"':"")+'>'+w[z]+""}D+=E+"";E=this._getDaysInMonth(r,l);if(r==a.selectedYear&&l==a.selectedMonth)a.selectedDay=Math.min(a.selectedDay,E);A=(this._getFirstDayOfMonth(r,l)-n+7)%7;E=m?6:Math.ceil((A+E)/7);z=this._daylightSavingAdjust(new Date(r,l,1-A));for(var P=0;P";var Q=!k?"":'";for(A=0;A<7;A++){var I= +x?x.apply(a.input?a.input[0]:null,[z]):[true,""],F=z.getMonth()!=l,L=F&&!J||!I[0]||p&&zs;Q+='";z.setDate(z.getDate()+1);z=this._daylightSavingAdjust(z)}D+= +Q+""}l++;if(l>11){l=0;r++}D+="
    '+this._get(a,"weekHeader")+"
    '+this._get(a,"calculateWeek")(z)+""+(F&&!C?" ":L?''+z.getDate()+"":''+z.getDate()+"")+"
    "+(m?"
    "+(q[0]>0&&H==q[1]-1?'
    ':""):"");N+=D}K+=N}K+=i+(b.browser.msie&&parseInt(b.browser.version,10)<7&&!a.inline?'':"");a._keyEvent=false;return K},_generateMonthYearHeader:function(a,d,h,i,j,n,q,l){var k=this._get(a,"changeMonth"),m=this._get(a,"changeYear"),o=this._get(a,"showMonthAfterYear"),p='
    ', +s="";if(n||!k)s+=''+q[d]+"";else{q=i&&i.getFullYear()==h;var r=j&&j.getFullYear()==h;s+='"}o||(p+=s+(n||!(k&& +m)?" ":""));a.yearshtml="";if(n||!m)p+=''+h+"";else{l=this._get(a,"yearRange").split(":");var v=(new Date).getFullYear();q=function(w){w=w.match(/c[+-].*/)?h+parseInt(w.substring(1),10):w.match(/[+-].*/)?v+parseInt(w,10):parseInt(w,10);return isNaN(w)?v:w};d=q(l[0]);l=Math.max(d,q(l[1]||""));d=i?Math.max(d,i.getFullYear()):d;l=j?Math.min(l,j.getFullYear()):l;for(a.yearshtml+='";if(b.browser.mozilla)p+='";else{p+=a.yearshtml;a.yearshtml=null}}p+=this._get(a,"yearSuffix");if(o)p+=(n||!(k&&m)?" ":"")+s;p+="
    ";return p},_adjustInstDate:function(a,d,h){var i= +a.drawYear+(h=="Y"?d:0),j=a.drawMonth+(h=="M"?d:0);d=Math.min(a.selectedDay,this._getDaysInMonth(i,j))+(h=="D"?d:0);i=this._restrictMinMax(a,this._daylightSavingAdjust(new Date(i,j,d)));a.selectedDay=i.getDate();a.drawMonth=a.selectedMonth=i.getMonth();a.drawYear=a.selectedYear=i.getFullYear();if(h=="M"||h=="Y")this._notifyChange(a)},_restrictMinMax:function(a,d){var h=this._getMinMaxDate(a,"min");a=this._getMinMaxDate(a,"max");d=h&&da?a:d},_notifyChange:function(a){var d=this._get(a, +"onChangeMonthYear");if(d)d.apply(a.input?a.input[0]:null,[a.selectedYear,a.selectedMonth+1,a])},_getNumberOfMonths:function(a){a=this._get(a,"numberOfMonths");return a==null?[1,1]:typeof a=="number"?[1,a]:a},_getMinMaxDate:function(a,d){return this._determineDate(a,this._get(a,d+"Date"),null)},_getDaysInMonth:function(a,d){return 32-(new Date(a,d,32)).getDate()},_getFirstDayOfMonth:function(a,d){return(new Date(a,d,1)).getDay()},_canAdjustMonth:function(a,d,h,i){var j=this._getNumberOfMonths(a); +h=this._daylightSavingAdjust(new Date(h,i+(d<0?d:j[0]*j[1]),1));d<0&&h.setDate(this._getDaysInMonth(h.getFullYear(),h.getMonth()));return this._isInRange(a,h)},_isInRange:function(a,d){var h=this._getMinMaxDate(a,"min");a=this._getMinMaxDate(a,"max");return(!h||d.getTime()>=h.getTime())&&(!a||d.getTime()<=a.getTime())},_getFormatConfig:function(a){var d=this._get(a,"shortYearCutoff");d=typeof d!="string"?d:(new Date).getFullYear()%100+parseInt(d,10);return{shortYearCutoff:d,dayNamesShort:this._get(a, +"dayNamesShort"),dayNames:this._get(a,"dayNames"),monthNamesShort:this._get(a,"monthNamesShort"),monthNames:this._get(a,"monthNames")}},_formatDate:function(a,d,h,i){if(!d){a.currentDay=a.selectedDay;a.currentMonth=a.selectedMonth;a.currentYear=a.selectedYear}d=d?typeof d=="object"?d:this._daylightSavingAdjust(new Date(i,h,d)):this._daylightSavingAdjust(new Date(a.currentYear,a.currentMonth,a.currentDay));return this.formatDate(this._get(a,"dateFormat"),d,this._getFormatConfig(a))}});b.fn.datepicker= +function(a){if(!b.datepicker.initialized){b(document).mousedown(b.datepicker._checkExternalClick).find("body").append(b.datepicker.dpDiv);b.datepicker.initialized=true}var d=Array.prototype.slice.call(arguments,1);if(typeof a=="string"&&(a=="isDisabled"||a=="getDate"||a=="widget"))return b.datepicker["_"+a+"Datepicker"].apply(b.datepicker,[this[0]].concat(d));if(a=="option"&&arguments.length==2&&typeof arguments[1]=="string")return b.datepicker["_"+a+"Datepicker"].apply(b.datepicker,[this[0]].concat(d)); +return this.each(function(){typeof a=="string"?b.datepicker["_"+a+"Datepicker"].apply(b.datepicker,[this].concat(d)):b.datepicker._attachDatepicker(this,a)})};b.datepicker=new f;b.datepicker.initialized=false;b.datepicker.uuid=(new Date).getTime();b.datepicker.version="1.8.8";window["DP_jQuery_"+e]=b})(jQuery); +(function(b,c){var f={buttons:true,height:true,maxHeight:true,maxWidth:true,minHeight:true,minWidth:true,width:true},g={maxHeight:true,maxWidth:true,minHeight:true,minWidth:true};b.widget("ui.dialog",{options:{autoOpen:true,buttons:{},closeOnEscape:true,closeText:"close",dialogClass:"",draggable:true,hide:null,height:"auto",maxHeight:false,maxWidth:false,minHeight:150,minWidth:150,modal:false,position:{my:"center",at:"center",collision:"fit",using:function(e){var a=b(this).css(e).offset().top;a<0&& +b(this).css("top",e.top-a)}},resizable:true,show:null,stack:true,title:"",width:300,zIndex:1E3},_create:function(){this.originalTitle=this.element.attr("title");if(typeof this.originalTitle!=="string")this.originalTitle="";this.options.title=this.options.title||this.originalTitle;var e=this,a=e.options,d=a.title||" ",h=b.ui.dialog.getTitleId(e.element),i=(e.uiDialog=b("
    ")).appendTo(document.body).hide().addClass("ui-dialog ui-widget ui-widget-content ui-corner-all "+a.dialogClass).css({zIndex:a.zIndex}).attr("tabIndex", +-1).css("outline",0).keydown(function(q){if(a.closeOnEscape&&q.keyCode&&q.keyCode===b.ui.keyCode.ESCAPE){e.close(q);q.preventDefault()}}).attr({role:"dialog","aria-labelledby":h}).mousedown(function(q){e.moveToTop(false,q)});e.element.show().removeAttr("title").addClass("ui-dialog-content ui-widget-content").appendTo(i);var j=(e.uiDialogTitlebar=b("
    ")).addClass("ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix").prependTo(i),n=b('').addClass("ui-dialog-titlebar-close ui-corner-all").attr("role", +"button").hover(function(){n.addClass("ui-state-hover")},function(){n.removeClass("ui-state-hover")}).focus(function(){n.addClass("ui-state-focus")}).blur(function(){n.removeClass("ui-state-focus")}).click(function(q){e.close(q);return false}).appendTo(j);(e.uiDialogTitlebarCloseText=b("")).addClass("ui-icon ui-icon-closethick").text(a.closeText).appendTo(n);b("").addClass("ui-dialog-title").attr("id",h).html(d).prependTo(j);if(b.isFunction(a.beforeclose)&&!b.isFunction(a.beforeClose))a.beforeClose= +a.beforeclose;j.find("*").add(j).disableSelection();a.draggable&&b.fn.draggable&&e._makeDraggable();a.resizable&&b.fn.resizable&&e._makeResizable();e._createButtons(a.buttons);e._isOpen=false;b.fn.bgiframe&&i.bgiframe()},_init:function(){this.options.autoOpen&&this.open()},destroy:function(){var e=this;e.overlay&&e.overlay.destroy();e.uiDialog.hide();e.element.unbind(".dialog").removeData("dialog").removeClass("ui-dialog-content ui-widget-content").hide().appendTo("body");e.uiDialog.remove();e.originalTitle&& +e.element.attr("title",e.originalTitle);return e},widget:function(){return this.uiDialog},close:function(e){var a=this,d,h;if(false!==a._trigger("beforeClose",e)){a.overlay&&a.overlay.destroy();a.uiDialog.unbind("keypress.ui-dialog");a._isOpen=false;if(a.options.hide)a.uiDialog.hide(a.options.hide,function(){a._trigger("close",e)});else{a.uiDialog.hide();a._trigger("close",e)}b.ui.dialog.overlay.resize();if(a.options.modal){d=0;b(".ui-dialog").each(function(){if(this!==a.uiDialog[0]){h=b(this).css("z-index"); +isNaN(h)||(d=Math.max(d,h))}});b.ui.dialog.maxZ=d}return a}},isOpen:function(){return this._isOpen},moveToTop:function(e,a){var d=this,h=d.options;if(h.modal&&!e||!h.stack&&!h.modal)return d._trigger("focus",a);if(h.zIndex>b.ui.dialog.maxZ)b.ui.dialog.maxZ=h.zIndex;if(d.overlay){b.ui.dialog.maxZ+=1;d.overlay.$el.css("z-index",b.ui.dialog.overlay.maxZ=b.ui.dialog.maxZ)}e={scrollTop:d.element.attr("scrollTop"),scrollLeft:d.element.attr("scrollLeft")};b.ui.dialog.maxZ+=1;d.uiDialog.css("z-index",b.ui.dialog.maxZ); +d.element.attr(e);d._trigger("focus",a);return d},open:function(){if(!this._isOpen){var e=this,a=e.options,d=e.uiDialog;e.overlay=a.modal?new b.ui.dialog.overlay(e):null;e._size();e._position(a.position);d.show(a.show);e.moveToTop(true);a.modal&&d.bind("keypress.ui-dialog",function(h){if(h.keyCode===b.ui.keyCode.TAB){var i=b(":tabbable",this),j=i.filter(":first");i=i.filter(":last");if(h.target===i[0]&&!h.shiftKey){j.focus(1);return false}else if(h.target===j[0]&&h.shiftKey){i.focus(1);return false}}}); +b(e.element.find(":tabbable").get().concat(d.find(".ui-dialog-buttonpane :tabbable").get().concat(d.get()))).eq(0).focus();e._isOpen=true;e._trigger("open");return e}},_createButtons:function(e){var a=this,d=false,h=b("
    ").addClass("ui-dialog-buttonpane ui-widget-content ui-helper-clearfix"),i=b("
    ").addClass("ui-dialog-buttonset").appendTo(h);a.uiDialog.find(".ui-dialog-buttonpane").remove();typeof e==="object"&&e!==null&&b.each(e,function(){return!(d=true)});if(d){b.each(e,function(j, +n){n=b.isFunction(n)?{click:n,text:j}:n;j=b('').attr(n,true).unbind("click").click(function(){n.click.apply(a.element[0],arguments)}).appendTo(i);b.fn.button&&j.button()});h.appendTo(a.uiDialog)}},_makeDraggable:function(){function e(j){return{position:j.position,offset:j.offset}}var a=this,d=a.options,h=b(document),i;a.uiDialog.draggable({cancel:".ui-dialog-content, .ui-dialog-titlebar-close",handle:".ui-dialog-titlebar",containment:"document",start:function(j,n){i= +d.height==="auto"?"auto":b(this).height();b(this).height(b(this).height()).addClass("ui-dialog-dragging");a._trigger("dragStart",j,e(n))},drag:function(j,n){a._trigger("drag",j,e(n))},stop:function(j,n){d.position=[n.position.left-h.scrollLeft(),n.position.top-h.scrollTop()];b(this).removeClass("ui-dialog-dragging").height(i);a._trigger("dragStop",j,e(n));b.ui.dialog.overlay.resize()}})},_makeResizable:function(e){function a(j){return{originalPosition:j.originalPosition,originalSize:j.originalSize, +position:j.position,size:j.size}}e=e===c?this.options.resizable:e;var d=this,h=d.options,i=d.uiDialog.css("position");e=typeof e==="string"?e:"n,e,s,w,se,sw,ne,nw";d.uiDialog.resizable({cancel:".ui-dialog-content",containment:"document",alsoResize:d.element,maxWidth:h.maxWidth,maxHeight:h.maxHeight,minWidth:h.minWidth,minHeight:d._minHeight(),handles:e,start:function(j,n){b(this).addClass("ui-dialog-resizing");d._trigger("resizeStart",j,a(n))},resize:function(j,n){d._trigger("resize",j,a(n))},stop:function(j, +n){b(this).removeClass("ui-dialog-resizing");h.height=b(this).height();h.width=b(this).width();d._trigger("resizeStop",j,a(n));b.ui.dialog.overlay.resize()}}).css("position",i).find(".ui-resizable-se").addClass("ui-icon ui-icon-grip-diagonal-se")},_minHeight:function(){var e=this.options;return e.height==="auto"?e.minHeight:Math.min(e.minHeight,e.height)},_position:function(e){var a=[],d=[0,0],h;if(e){if(typeof e==="string"||typeof e==="object"&&"0"in e){a=e.split?e.split(" "):[e[0],e[1]];if(a.length=== +1)a[1]=a[0];b.each(["left","top"],function(i,j){if(+a[i]===a[i]){d[i]=a[i];a[i]=j}});e={my:a.join(" "),at:a.join(" "),offset:d.join(" ")}}e=b.extend({},b.ui.dialog.prototype.options.position,e)}else e=b.ui.dialog.prototype.options.position;(h=this.uiDialog.is(":visible"))||this.uiDialog.show();this.uiDialog.css({top:0,left:0}).position(b.extend({of:window},e));h||this.uiDialog.hide()},_setOptions:function(e){var a=this,d={},h=false;b.each(e,function(i,j){a._setOption(i,j);if(i in f)h=true;if(i in +g)d[i]=j});h&&this._size();this.uiDialog.is(":data(resizable)")&&this.uiDialog.resizable("option",d)},_setOption:function(e,a){var d=this,h=d.uiDialog;switch(e){case "beforeclose":e="beforeClose";break;case "buttons":d._createButtons(a);break;case "closeText":d.uiDialogTitlebarCloseText.text(""+a);break;case "dialogClass":h.removeClass(d.options.dialogClass).addClass("ui-dialog ui-widget ui-widget-content ui-corner-all "+a);break;case "disabled":a?h.addClass("ui-dialog-disabled"):h.removeClass("ui-dialog-disabled"); +break;case "draggable":var i=h.is(":data(draggable)");i&&!a&&h.draggable("destroy");!i&&a&&d._makeDraggable();break;case "position":d._position(a);break;case "resizable":(i=h.is(":data(resizable)"))&&!a&&h.resizable("destroy");i&&typeof a==="string"&&h.resizable("option","handles",a);!i&&a!==false&&d._makeResizable(a);break;case "title":b(".ui-dialog-title",d.uiDialogTitlebar).html(""+(a||" "));break}b.Widget.prototype._setOption.apply(d,arguments)},_size:function(){var e=this.options,a,d,h= +this.uiDialog.is(":visible");this.element.show().css({width:"auto",minHeight:0,height:0});if(e.minWidth>e.width)e.width=e.minWidth;a=this.uiDialog.css({height:"auto",width:e.width}).height();d=Math.max(0,e.minHeight-a);if(e.height==="auto")if(b.support.minHeight)this.element.css({minHeight:d,height:"auto"});else{this.uiDialog.show();e=this.element.css("height","auto").height();h||this.uiDialog.hide();this.element.height(Math.max(e,d))}else this.element.height(Math.max(e.height-a,0));this.uiDialog.is(":data(resizable)")&& +this.uiDialog.resizable("option","minHeight",this._minHeight())}});b.extend(b.ui.dialog,{version:"1.8.8",uuid:0,maxZ:0,getTitleId:function(e){e=e.attr("id");if(!e){this.uuid+=1;e=this.uuid}return"ui-dialog-title-"+e},overlay:function(e){this.$el=b.ui.dialog.overlay.create(e)}});b.extend(b.ui.dialog.overlay,{instances:[],oldInstances:[],maxZ:0,events:b.map("focus,mousedown,mouseup,keydown,keypress,click".split(","),function(e){return e+".dialog-overlay"}).join(" "),create:function(e){if(this.instances.length=== +0){setTimeout(function(){b.ui.dialog.overlay.instances.length&&b(document).bind(b.ui.dialog.overlay.events,function(d){if(b(d.target).zIndex()").addClass("ui-widget-overlay")).appendTo(document.body).css({width:this.width(), +height:this.height()});b.fn.bgiframe&&a.bgiframe();this.instances.push(a);return a},destroy:function(e){var a=b.inArray(e,this.instances);a!=-1&&this.oldInstances.push(this.instances.splice(a,1)[0]);this.instances.length===0&&b([document,window]).unbind(".dialog-overlay");e.remove();var d=0;b.each(this.instances,function(){d=Math.max(d,this.css("z-index"))});this.maxZ=d},height:function(){var e,a;if(b.browser.msie&&b.browser.version<7){e=Math.max(document.documentElement.scrollHeight,document.body.scrollHeight); +a=Math.max(document.documentElement.offsetHeight,document.body.offsetHeight);return e0?a.left-h:Math.max(a.left-d.collisionPosition.left,a.left)},top:function(a,d){var h=b(window);h=d.collisionPosition.top+d.collisionHeight-h.height()-h.scrollTop();a.top=h>0?a.top-h:Math.max(a.top-d.collisionPosition.top,a.top)}},flip:{left:function(a,d){if(d.at[0]!=="center"){var h=b(window);h=d.collisionPosition.left+d.collisionWidth-h.width()-h.scrollLeft();var i=d.my[0]==="left"?-d.elemWidth:d.my[0]==="right"?d.elemWidth:0,j=d.at[0]==="left"?d.targetWidth:-d.targetWidth,n=-2*d.offset[0];a.left+= +d.collisionPosition.left<0?i+j+n:h>0?i+j+n:0}},top:function(a,d){if(d.at[1]!=="center"){var h=b(window);h=d.collisionPosition.top+d.collisionHeight-h.height()-h.scrollTop();var i=d.my[1]==="top"?-d.elemHeight:d.my[1]==="bottom"?d.elemHeight:0,j=d.at[1]==="top"?d.targetHeight:-d.targetHeight,n=-2*d.offset[1];a.top+=d.collisionPosition.top<0?i+j+n:h>0?i+j+n:0}}}};if(!b.offset.setOffset){b.offset.setOffset=function(a,d){if(/static/.test(b.curCSS(a,"position")))a.style.position="relative";var h=b(a), +i=h.offset(),j=parseInt(b.curCSS(a,"top",true),10)||0,n=parseInt(b.curCSS(a,"left",true),10)||0;i={top:d.top-i.top+j,left:d.left-i.left+n};"using"in d?d.using.call(a,i):h.css(i)};b.fn.offset=function(a){var d=this[0];if(!d||!d.ownerDocument)return null;if(a)return this.each(function(){b.offset.setOffset(this,a)});return e.call(this)}}})(jQuery); +(function(b,c){b.widget("ui.progressbar",{options:{value:0,max:100},min:0,_create:function(){this.element.addClass("ui-progressbar ui-widget ui-widget-content ui-corner-all").attr({role:"progressbar","aria-valuemin":this.min,"aria-valuemax":this.options.max,"aria-valuenow":this._value()});this.valueDiv=b("
    ").appendTo(this.element);this.oldValue=this._value();this._refreshValue()},destroy:function(){this.element.removeClass("ui-progressbar ui-widget ui-widget-content ui-corner-all").removeAttr("role").removeAttr("aria-valuemin").removeAttr("aria-valuemax").removeAttr("aria-valuenow"); +this.valueDiv.remove();b.Widget.prototype.destroy.apply(this,arguments)},value:function(f){if(f===c)return this._value();this._setOption("value",f);return this},_setOption:function(f,g){if(f==="value"){this.options.value=g;this._refreshValue();this._value()===this.options.max&&this._trigger("complete")}b.Widget.prototype._setOption.apply(this,arguments)},_value:function(){var f=this.options.value;if(typeof f!=="number")f=0;return Math.min(this.options.max,Math.max(this.min,f))},_percentage:function(){return 100* +this._value()/this.options.max},_refreshValue:function(){var f=this.value(),g=this._percentage();if(this.oldValue!==f){this.oldValue=f;this._trigger("change")}this.valueDiv.toggleClass("ui-corner-right",f===this.options.max).width(g.toFixed(0)+"%");this.element.attr("aria-valuenow",f)}});b.extend(b.ui.progressbar,{version:"1.8.8"})})(jQuery); +(function(b){b.widget("ui.slider",b.ui.mouse,{widgetEventPrefix:"slide",options:{animate:false,distance:0,max:100,min:0,orientation:"horizontal",range:false,step:1,value:0,values:null},_create:function(){var c=this,f=this.options;this._mouseSliding=this._keySliding=false;this._animateOff=true;this._handleIndex=null;this._detectOrientation();this._mouseInit();this.element.addClass("ui-slider ui-slider-"+this.orientation+" ui-widget ui-widget-content ui-corner-all");f.disabled&&this.element.addClass("ui-slider-disabled ui-disabled"); +this.range=b([]);if(f.range){if(f.range===true){this.range=b("
    ");if(!f.values)f.values=[this._valueMin(),this._valueMin()];if(f.values.length&&f.values.length!==2)f.values=[f.values[0],f.values[0]]}else this.range=b("
    ");this.range.appendTo(this.element).addClass("ui-slider-range");if(f.range==="min"||f.range==="max")this.range.addClass("ui-slider-range-"+f.range);this.range.addClass("ui-widget-header")}b(".ui-slider-handle",this.element).length===0&&b("").appendTo(this.element).addClass("ui-slider-handle"); +if(f.values&&f.values.length)for(;b(".ui-slider-handle",this.element).length").appendTo(this.element).addClass("ui-slider-handle");this.handles=b(".ui-slider-handle",this.element).addClass("ui-state-default ui-corner-all");this.handle=this.handles.eq(0);this.handles.add(this.range).filter("a").click(function(g){g.preventDefault()}).hover(function(){f.disabled||b(this).addClass("ui-state-hover")},function(){b(this).removeClass("ui-state-hover")}).focus(function(){if(f.disabled)b(this).blur(); +else{b(".ui-slider .ui-state-focus").removeClass("ui-state-focus");b(this).addClass("ui-state-focus")}}).blur(function(){b(this).removeClass("ui-state-focus")});this.handles.each(function(g){b(this).data("index.ui-slider-handle",g)});this.handles.keydown(function(g){var e=true,a=b(this).data("index.ui-slider-handle"),d,h,i;if(!c.options.disabled){switch(g.keyCode){case b.ui.keyCode.HOME:case b.ui.keyCode.END:case b.ui.keyCode.PAGE_UP:case b.ui.keyCode.PAGE_DOWN:case b.ui.keyCode.UP:case b.ui.keyCode.RIGHT:case b.ui.keyCode.DOWN:case b.ui.keyCode.LEFT:e= +false;if(!c._keySliding){c._keySliding=true;b(this).addClass("ui-state-active");d=c._start(g,a);if(d===false)return}break}i=c.options.step;d=c.options.values&&c.options.values.length?(h=c.values(a)):(h=c.value());switch(g.keyCode){case b.ui.keyCode.HOME:h=c._valueMin();break;case b.ui.keyCode.END:h=c._valueMax();break;case b.ui.keyCode.PAGE_UP:h=c._trimAlignValue(d+(c._valueMax()-c._valueMin())/5);break;case b.ui.keyCode.PAGE_DOWN:h=c._trimAlignValue(d-(c._valueMax()-c._valueMin())/5);break;case b.ui.keyCode.UP:case b.ui.keyCode.RIGHT:if(d=== +c._valueMax())return;h=c._trimAlignValue(d+i);break;case b.ui.keyCode.DOWN:case b.ui.keyCode.LEFT:if(d===c._valueMin())return;h=c._trimAlignValue(d-i);break}c._slide(g,a,h);return e}}).keyup(function(g){var e=b(this).data("index.ui-slider-handle");if(c._keySliding){c._keySliding=false;c._stop(g,e);c._change(g,e);b(this).removeClass("ui-state-active")}});this._refreshValue();this._animateOff=false},destroy:function(){this.handles.remove();this.range.remove();this.element.removeClass("ui-slider ui-slider-horizontal ui-slider-vertical ui-slider-disabled ui-widget ui-widget-content ui-corner-all").removeData("slider").unbind(".slider"); +this._mouseDestroy();return this},_mouseCapture:function(c){var f=this.options,g,e,a,d,h;if(f.disabled)return false;this.elementSize={width:this.element.outerWidth(),height:this.element.outerHeight()};this.elementOffset=this.element.offset();g=this._normValueFromMouse({x:c.pageX,y:c.pageY});e=this._valueMax()-this._valueMin()+1;d=this;this.handles.each(function(i){var j=Math.abs(g-d.values(i));if(e>j){e=j;a=b(this);h=i}});if(f.range===true&&this.values(1)===f.min){h+=1;a=b(this.handles[h])}if(this._start(c, +h)===false)return false;this._mouseSliding=true;d._handleIndex=h;a.addClass("ui-state-active").focus();f=a.offset();this._clickOffset=!b(c.target).parents().andSelf().is(".ui-slider-handle")?{left:0,top:0}:{left:c.pageX-f.left-a.width()/2,top:c.pageY-f.top-a.height()/2-(parseInt(a.css("borderTopWidth"),10)||0)-(parseInt(a.css("borderBottomWidth"),10)||0)+(parseInt(a.css("marginTop"),10)||0)};this.handles.hasClass("ui-state-hover")||this._slide(c,h,g);return this._animateOff=true},_mouseStart:function(){return true}, +_mouseDrag:function(c){var f=this._normValueFromMouse({x:c.pageX,y:c.pageY});this._slide(c,this._handleIndex,f);return false},_mouseStop:function(c){this.handles.removeClass("ui-state-active");this._mouseSliding=false;this._stop(c,this._handleIndex);this._change(c,this._handleIndex);this._clickOffset=this._handleIndex=null;return this._animateOff=false},_detectOrientation:function(){this.orientation=this.options.orientation==="vertical"?"vertical":"horizontal"},_normValueFromMouse:function(c){var f; +if(this.orientation==="horizontal"){f=this.elementSize.width;c=c.x-this.elementOffset.left-(this._clickOffset?this._clickOffset.left:0)}else{f=this.elementSize.height;c=c.y-this.elementOffset.top-(this._clickOffset?this._clickOffset.top:0)}f=c/f;if(f>1)f=1;if(f<0)f=0;if(this.orientation==="vertical")f=1-f;c=this._valueMax()-this._valueMin();return this._trimAlignValue(this._valueMin()+f*c)},_start:function(c,f){var g={handle:this.handles[f],value:this.value()};if(this.options.values&&this.options.values.length){g.value= +this.values(f);g.values=this.values()}return this._trigger("start",c,g)},_slide:function(c,f,g){var e;if(this.options.values&&this.options.values.length){e=this.values(f?0:1);if(this.options.values.length===2&&this.options.range===true&&(f===0&&g>e||f===1&&g1){this.options.values[c]=this._trimAlignValue(f);this._refreshValue();this._change(null,c)}if(arguments.length)if(b.isArray(arguments[0])){g=this.options.values;e=arguments[0];for(a=0;a=this._valueMax())return this._valueMax();var f=this.options.step>0?this.options.step:1,g=(c-this._valueMin())%f;alignValue=c-g;if(Math.abs(g)*2>=f)alignValue+=g>0?f:-f;return parseFloat(alignValue.toFixed(5))},_valueMin:function(){return this.options.min},_valueMax:function(){return this.options.max}, +_refreshValue:function(){var c=this.options.range,f=this.options,g=this,e=!this._animateOff?f.animate:false,a,d={},h,i,j,n;if(this.options.values&&this.options.values.length)this.handles.each(function(q){a=(g.values(q)-g._valueMin())/(g._valueMax()-g._valueMin())*100;d[g.orientation==="horizontal"?"left":"bottom"]=a+"%";b(this).stop(1,1)[e?"animate":"css"](d,f.animate);if(g.options.range===true)if(g.orientation==="horizontal"){if(q===0)g.range.stop(1,1)[e?"animate":"css"]({left:a+"%"},f.animate); +if(q===1)g.range[e?"animate":"css"]({width:a-h+"%"},{queue:false,duration:f.animate})}else{if(q===0)g.range.stop(1,1)[e?"animate":"css"]({bottom:a+"%"},f.animate);if(q===1)g.range[e?"animate":"css"]({height:a-h+"%"},{queue:false,duration:f.animate})}h=a});else{i=this.value();j=this._valueMin();n=this._valueMax();a=n!==j?(i-j)/(n-j)*100:0;d[g.orientation==="horizontal"?"left":"bottom"]=a+"%";this.handle.stop(1,1)[e?"animate":"css"](d,f.animate);if(c==="min"&&this.orientation==="horizontal")this.range.stop(1, +1)[e?"animate":"css"]({width:a+"%"},f.animate);if(c==="max"&&this.orientation==="horizontal")this.range[e?"animate":"css"]({width:100-a+"%"},{queue:false,duration:f.animate});if(c==="min"&&this.orientation==="vertical")this.range.stop(1,1)[e?"animate":"css"]({height:a+"%"},f.animate);if(c==="max"&&this.orientation==="vertical")this.range[e?"animate":"css"]({height:100-a+"%"},{queue:false,duration:f.animate})}}});b.extend(b.ui.slider,{version:"1.8.8"})})(jQuery); +(function(b,c){function f(){return++e}function g(){return++a}var e=0,a=0;b.widget("ui.tabs",{options:{add:null,ajaxOptions:null,cache:false,cookie:null,collapsible:false,disable:null,disabled:[],enable:null,event:"click",fx:null,idPrefix:"ui-tabs-",load:null,panelTemplate:"
    ",remove:null,select:null,show:null,spinner:"Loading…",tabTemplate:"
  • #{label}
    • "},_create:function(){this._tabify(true)},_setOption:function(d,h){if(d=="selected")this.options.collapsible&& +h==this.options.selected||this.select(h);else{this.options[d]=h;this._tabify()}},_tabId:function(d){return d.title&&d.title.replace(/\s/g,"_").replace(/[^\w\u00c0-\uFFFF-]/g,"")||this.options.idPrefix+f()},_sanitizeSelector:function(d){return d.replace(/:/g,"\\:")},_cookie:function(){var d=this.cookie||(this.cookie=this.options.cookie.name||"ui-tabs-"+g());return b.cookie.apply(null,[d].concat(b.makeArray(arguments)))},_ui:function(d,h){return{tab:d,panel:h,index:this.anchors.index(d)}},_cleanup:function(){this.lis.filter(".ui-state-processing").removeClass("ui-state-processing").find("span:data(label.tabs)").each(function(){var d= +b(this);d.html(d.data("label.tabs")).removeData("label.tabs")})},_tabify:function(d){function h(r,u){r.css("display","");!b.support.opacity&&u.opacity&&r[0].style.removeAttribute("filter")}var i=this,j=this.options,n=/^#.+/;this.list=this.element.find("ol,ul").eq(0);this.lis=b(" > li:has(a[href])",this.list);this.anchors=this.lis.map(function(){return b("a",this)[0]});this.panels=b([]);this.anchors.each(function(r,u){var v=b(u).attr("href"),w=v.split("#")[0],y;if(w&&(w===location.toString().split("#")[0]|| +(y=b("base")[0])&&w===y.href)){v=u.hash;u.href=v}if(n.test(v))i.panels=i.panels.add(i.element.find(i._sanitizeSelector(v)));else if(v&&v!=="#"){b.data(u,"href.tabs",v);b.data(u,"load.tabs",v.replace(/#.*$/,""));v=i._tabId(u);u.href="#"+v;u=i.element.find("#"+v);if(!u.length){u=b(j.panelTemplate).attr("id",v).addClass("ui-tabs-panel ui-widget-content ui-corner-bottom").insertAfter(i.panels[r-1]||i.list);u.data("destroy.tabs",true)}i.panels=i.panels.add(u)}else j.disabled.push(r)});if(d){this.element.addClass("ui-tabs ui-widget ui-widget-content ui-corner-all"); +this.list.addClass("ui-tabs-nav ui-helper-reset ui-helper-clearfix ui-widget-header ui-corner-all");this.lis.addClass("ui-state-default ui-corner-top");this.panels.addClass("ui-tabs-panel ui-widget-content ui-corner-bottom");if(j.selected===c){location.hash&&this.anchors.each(function(r,u){if(u.hash==location.hash){j.selected=r;return false}});if(typeof j.selected!=="number"&&j.cookie)j.selected=parseInt(i._cookie(),10);if(typeof j.selected!=="number"&&this.lis.filter(".ui-tabs-selected").length)j.selected= +this.lis.index(this.lis.filter(".ui-tabs-selected"));j.selected=j.selected||(this.lis.length?0:-1)}else if(j.selected===null)j.selected=-1;j.selected=j.selected>=0&&this.anchors[j.selected]||j.selected<0?j.selected:0;j.disabled=b.unique(j.disabled.concat(b.map(this.lis.filter(".ui-state-disabled"),function(r){return i.lis.index(r)}))).sort();b.inArray(j.selected,j.disabled)!=-1&&j.disabled.splice(b.inArray(j.selected,j.disabled),1);this.panels.addClass("ui-tabs-hide");this.lis.removeClass("ui-tabs-selected ui-state-active"); +if(j.selected>=0&&this.anchors.length){i.element.find(i._sanitizeSelector(i.anchors[j.selected].hash)).removeClass("ui-tabs-hide");this.lis.eq(j.selected).addClass("ui-tabs-selected ui-state-active");i.element.queue("tabs",function(){i._trigger("show",null,i._ui(i.anchors[j.selected],i.element.find(i._sanitizeSelector(i.anchors[j.selected].hash))))});this.load(j.selected)}b(window).bind("unload",function(){i.lis.add(i.anchors).unbind(".tabs");i.lis=i.anchors=i.panels=null})}else j.selected=this.lis.index(this.lis.filter(".ui-tabs-selected")); +this.element[j.collapsible?"addClass":"removeClass"]("ui-tabs-collapsible");j.cookie&&this._cookie(j.selected,j.cookie);d=0;for(var q;q=this.lis[d];d++)b(q)[b.inArray(d,j.disabled)!=-1&&!b(q).hasClass("ui-tabs-selected")?"addClass":"removeClass"]("ui-state-disabled");j.cache===false&&this.anchors.removeData("cache.tabs");this.lis.add(this.anchors).unbind(".tabs");if(j.event!=="mouseover"){var l=function(r,u){u.is(":not(.ui-state-disabled)")&&u.addClass("ui-state-"+r)},k=function(r,u){u.removeClass("ui-state-"+ +r)};this.lis.bind("mouseover.tabs",function(){l("hover",b(this))});this.lis.bind("mouseout.tabs",function(){k("hover",b(this))});this.anchors.bind("focus.tabs",function(){l("focus",b(this).closest("li"))});this.anchors.bind("blur.tabs",function(){k("focus",b(this).closest("li"))})}var m,o;if(j.fx)if(b.isArray(j.fx)){m=j.fx[0];o=j.fx[1]}else m=o=j.fx;var p=o?function(r,u){b(r).closest("li").addClass("ui-tabs-selected ui-state-active");u.hide().removeClass("ui-tabs-hide").animate(o,o.duration||"normal", +function(){h(u,o);i._trigger("show",null,i._ui(r,u[0]))})}:function(r,u){b(r).closest("li").addClass("ui-tabs-selected ui-state-active");u.removeClass("ui-tabs-hide");i._trigger("show",null,i._ui(r,u[0]))},s=m?function(r,u){u.animate(m,m.duration||"normal",function(){i.lis.removeClass("ui-tabs-selected ui-state-active");u.addClass("ui-tabs-hide");h(u,m);i.element.dequeue("tabs")})}:function(r,u){i.lis.removeClass("ui-tabs-selected ui-state-active");u.addClass("ui-tabs-hide");i.element.dequeue("tabs")}; +this.anchors.bind(j.event+".tabs",function(){var r=this,u=b(r).closest("li"),v=i.panels.filter(":not(.ui-tabs-hide)"),w=i.element.find(i._sanitizeSelector(r.hash));if(u.hasClass("ui-tabs-selected")&&!j.collapsible||u.hasClass("ui-state-disabled")||u.hasClass("ui-state-processing")||i.panels.filter(":animated").length||i._trigger("select",null,i._ui(this,w[0]))===false){this.blur();return false}j.selected=i.anchors.index(this);i.abort();if(j.collapsible)if(u.hasClass("ui-tabs-selected")){j.selected= +-1;j.cookie&&i._cookie(j.selected,j.cookie);i.element.queue("tabs",function(){s(r,v)}).dequeue("tabs");this.blur();return false}else if(!v.length){j.cookie&&i._cookie(j.selected,j.cookie);i.element.queue("tabs",function(){p(r,w)});i.load(i.anchors.index(this));this.blur();return false}j.cookie&&i._cookie(j.selected,j.cookie);if(w.length){v.length&&i.element.queue("tabs",function(){s(r,v)});i.element.queue("tabs",function(){p(r,w)});i.load(i.anchors.index(this))}else throw"jQuery UI Tabs: Mismatching fragment identifier."; +b.browser.msie&&this.blur()});this.anchors.bind("click.tabs",function(){return false})},_getIndex:function(d){if(typeof d=="string")d=this.anchors.index(this.anchors.filter("[href$="+d+"]"));return d},destroy:function(){var d=this.options;this.abort();this.element.unbind(".tabs").removeClass("ui-tabs ui-widget ui-widget-content ui-corner-all ui-tabs-collapsible").removeData("tabs");this.list.removeClass("ui-tabs-nav ui-helper-reset ui-helper-clearfix ui-widget-header ui-corner-all");this.anchors.each(function(){var h= +b.data(this,"href.tabs");if(h)this.href=h;var i=b(this).unbind(".tabs");b.each(["href","load","cache"],function(j,n){i.removeData(n+".tabs")})});this.lis.unbind(".tabs").add(this.panels).each(function(){b.data(this,"destroy.tabs")?b(this).remove():b(this).removeClass("ui-state-default ui-corner-top ui-tabs-selected ui-state-active ui-state-hover ui-state-focus ui-state-disabled ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide")});d.cookie&&this._cookie(null,d.cookie);return this},add:function(d, +h,i){if(i===c)i=this.anchors.length;var j=this,n=this.options;h=b(n.tabTemplate.replace(/#\{href\}/g,d).replace(/#\{label\}/g,h));d=!d.indexOf("#")?d.replace("#",""):this._tabId(b("a",h)[0]);h.addClass("ui-state-default ui-corner-top").data("destroy.tabs",true);var q=j.element.find("#"+d);q.length||(q=b(n.panelTemplate).attr("id",d).data("destroy.tabs",true));q.addClass("ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide");if(i>=this.lis.length){h.appendTo(this.list);q.appendTo(this.list[0].parentNode)}else{h.insertBefore(this.lis[i]); +q.insertBefore(this.panels[i])}n.disabled=b.map(n.disabled,function(l){return l>=i?++l:l});this._tabify();if(this.anchors.length==1){n.selected=0;h.addClass("ui-tabs-selected ui-state-active");q.removeClass("ui-tabs-hide");this.element.queue("tabs",function(){j._trigger("show",null,j._ui(j.anchors[0],j.panels[0]))});this.load(0)}this._trigger("add",null,this._ui(this.anchors[i],this.panels[i]));return this},remove:function(d){d=this._getIndex(d);var h=this.options,i=this.lis.eq(d).remove(),j=this.panels.eq(d).remove(); +if(i.hasClass("ui-tabs-selected")&&this.anchors.length>1)this.select(d+(d+1=d?--n:n});this._tabify();this._trigger("remove",null,this._ui(i.find("a")[0],j[0]));return this},enable:function(d){d=this._getIndex(d);var h=this.options;if(b.inArray(d,h.disabled)!=-1){this.lis.eq(d).removeClass("ui-state-disabled");h.disabled=b.grep(h.disabled,function(i){return i!=d});this._trigger("enable",null, +this._ui(this.anchors[d],this.panels[d]));return this}},disable:function(d){d=this._getIndex(d);var h=this.options;if(d!=h.selected){this.lis.eq(d).addClass("ui-state-disabled");h.disabled.push(d);h.disabled.sort();this._trigger("disable",null,this._ui(this.anchors[d],this.panels[d]))}return this},select:function(d){d=this._getIndex(d);if(d==-1)if(this.options.collapsible&&this.options.selected!=-1)d=this.options.selected;else return this;this.anchors.eq(d).trigger(this.options.event+".tabs");return this}, +load:function(d){d=this._getIndex(d);var h=this,i=this.options,j=this.anchors.eq(d)[0],n=b.data(j,"load.tabs");this.abort();if(!n||this.element.queue("tabs").length!==0&&b.data(j,"cache.tabs"))this.element.dequeue("tabs");else{this.lis.eq(d).addClass("ui-state-processing");if(i.spinner){var q=b("span",j);q.data("label.tabs",q.html()).html(i.spinner)}this.xhr=b.ajax(b.extend({},i.ajaxOptions,{url:n,success:function(l,k){h.element.find(h._sanitizeSelector(j.hash)).html(l);h._cleanup();i.cache&&b.data(j, +"cache.tabs",true);h._trigger("load",null,h._ui(h.anchors[d],h.panels[d]));try{i.ajaxOptions.success(l,k)}catch(m){}},error:function(l,k){h._cleanup();h._trigger("load",null,h._ui(h.anchors[d],h.panels[d]));try{i.ajaxOptions.error(l,k,d,j)}catch(m){}}}));h.element.dequeue("tabs");return this}},abort:function(){this.element.queue([]);this.panels.stop(false,true);this.element.queue("tabs",this.element.queue("tabs").splice(-2,2));if(this.xhr){this.xhr.abort();delete this.xhr}this._cleanup();return this}, +url:function(d,h){this.anchors.eq(d).removeData("cache.tabs").data("load.tabs",h);return this},length:function(){return this.anchors.length}});b.extend(b.ui.tabs,{version:"1.8.8"});b.extend(b.ui.tabs.prototype,{rotation:null,rotate:function(d,h){var i=this,j=this.options,n=i._rotate||(i._rotate=function(q){clearTimeout(i.rotation);i.rotation=setTimeout(function(){var l=j.selected;i.select(++l bulk ) + { + break; + } + } + if( t ) { + setTimeout(arguments.callee, delay); + } + else + { + end(); + } + + })(); +} + +// opts.delay : (default 10) delay between async call in ms +// opts.bulk : (default 500) delay during which the loop can continue synchronously without yielding the CPU +// opts.loop : (default empty) function to call in the each loop part, signature: function(index, value) this = value +// opts.end : (default empty) function to call at the end of the each loop +$.eachAsync = function(array, loop, opts) +{ + opts = opts || {}; + var i = 0, l = array.length; + + $.whileAsync( + function() { + var val = array[i]; + return loop.call(val, i++, val); + }, + $.extend(opts, { + test: function(){ return i < l; } + }) + ); +} + +$.fn.eachAsync = function(opts) +{ + $.eachAsync(this, opts); + return this; +} + +})(jQuery) \ No newline at end of file diff --git a/backoffice/static/js/jquery.cycle.lite.min.js b/backoffice/static/js/jquery.cycle.lite.min.js new file mode 100644 index 0000000..3488ff3 --- /dev/null +++ b/backoffice/static/js/jquery.cycle.lite.min.js @@ -0,0 +1,11 @@ +/* + * jQuery Cycle Lite Plugin + * http://malsup.com/jquery/cycle/lite/ + * Copyright (c) 2008 M. Alsup + * Version: 1.0 (06/08/2008) + * Dual licensed under the MIT and GPL licenses: + * http://www.opensource.org/licenses/mit-license.php + * http://www.gnu.org/licenses/gpl.html + * Requires: jQuery v1.2.3 or later + */ +;(function(D){var A="Lite-1.0";D.fn.cycle=function(E){return this.each(function(){E=E||{};if(this.cycleTimeout){clearTimeout(this.cycleTimeout)}this.cycleTimeout=0;this.cyclePause=0;var I=D(this);var J=E.slideExpr?D(E.slideExpr,this):I.children();var G=J.get();if(G.length<2){if(window.console&&window.console.log){window.console.log("terminating; too few slides: "+G.length)}return }var H=D.extend({},D.fn.cycle.defaults,E||{},D.metadata?I.metadata():D.meta?I.data():{});H.before=H.before?[H.before]:[];H.after=H.after?[H.after]:[];H.after.unshift(function(){H.busy=0});var F=this.className;H.width=parseInt((F.match(/w:(\d+)/)||[])[1])||H.width;H.height=parseInt((F.match(/h:(\d+)/)||[])[1])||H.height;H.timeout=parseInt((F.match(/t:(\d+)/)||[])[1])||H.timeout;if(I.css("position")=="static"){I.css("position","relative")}if(H.width){I.width(H.width)}if(H.height&&H.height!="auto"){I.height(H.height)}var K=0;J.css({position:"absolute",top:0,left:0}).hide().each(function(M){D(this).css("z-index",G.length-M)});D(G[K]).css("opacity",1).show();if(D.browser.msie){G[K].style.removeAttribute("filter")}if(H.fit&&H.width){J.width(H.width)}if(H.fit&&H.height&&H.height!="auto"){J.height(H.height)}if(H.pause){I.hover(function(){this.cyclePause=1},function(){this.cyclePause=0})}D.fn.cycle.transitions.fade(I,J,H);J.each(function(){var M=D(this);this.cycleH=(H.fit&&H.height)?H.height:M.height();this.cycleW=(H.fit&&H.width)?H.width:M.width()});J.not(":eq("+K+")").css({opacity:0});if(H.cssFirst){D(J[K]).css(H.cssFirst)}if(H.timeout){if(H.speed.constructor==String){H.speed={slow:600,fast:200}[H.speed]||400}if(!H.sync){H.speed=H.speed/2}while((H.timeout-H.speed)<250){H.timeout+=H.speed}}H.speedIn=H.speed;H.speedOut=H.speed;H.slideCount=G.length;H.currSlide=K;H.nextSlide=1;var L=J[K];if(H.before.length){H.before[0].apply(L,[L,L,H,true])}if(H.after.length>1){H.after[1].apply(L,[L,L,H,true])}if(H.click&&!H.next){H.next=H.click}if(H.next){D(H.next).bind("click",function(){return C(G,H,H.rev?-1:1)})}if(H.prev){D(H.prev).bind("click",function(){return C(G,H,H.rev?1:-1)})}if(H.timeout){this.cycleTimeout=setTimeout(function(){B(G,H,0,!H.rev)},H.timeout+(H.delay||0))}})};function B(J,E,I,K){if(E.busy){return }var H=J[0].parentNode,M=J[E.currSlide],L=J[E.nextSlide];if(H.cycleTimeout===0&&!I){return }if(I||!H.cyclePause){if(E.before.length){D.each(E.before,function(N,O){O.apply(L,[M,L,E,K])})}var F=function(){if(D.browser.msie){this.style.removeAttribute("filter")}D.each(E.after,function(N,O){O.apply(L,[M,L,E,K])})};if(E.nextSlide!=E.currSlide){E.busy=1;D.fn.cycle.custom(M,L,E,F)}var G=(E.nextSlide+1)==J.length;E.nextSlide=G?0:E.nextSlide+1;E.currSlide=G?J.length-1:E.nextSlide-1}if(E.timeout){H.cycleTimeout=setTimeout(function(){B(J,E,0,!E.rev)},E.timeout)}}function C(E,F,I){var H=E[0].parentNode,G=H.cycleTimeout;if(G){clearTimeout(G);H.cycleTimeout=0}F.nextSlide=F.currSlide+I;if(F.nextSlide<0){F.nextSlide=E.length-1}else{if(F.nextSlide>=E.length){F.nextSlide=0}}B(E,F,1,I>=0);return false}D.fn.cycle.custom=function(K,H,I,E){var J=D(K),G=D(H);G.css({opacity:0});var F=function(){G.animate({opacity:1},I.speedIn,I.easeIn,E)};J.animate({opacity:0},I.speedOut,I.easeOut,function(){J.css({display:"none"});if(!I.sync){F()}});if(I.sync){F()}};D.fn.cycle.transitions={fade:function(F,G,E){G.not(":eq(0)").css("opacity",0);E.before.push(function(){D(this).show()})}};D.fn.cycle.ver=function(){return A};D.fn.cycle.defaults={timeout:4000,speed:1000,next:null,prev:null,before:null,after:null,height:"auto",sync:1,fit:0,pause:0,delay:0,slideExpr:null}})(jQuery); diff --git a/backoffice/static/mysql_import.gz b/backoffice/static/mysql_import.gz new file mode 100644 index 0000000..18e12df Binary files /dev/null and b/backoffice/static/mysql_import.gz differ diff --git a/backoffice/static/mysql_import.zip b/backoffice/static/mysql_import.zip new file mode 100644 index 0000000..19b5f57 Binary files /dev/null and b/backoffice/static/mysql_import.zip differ diff --git a/backoffice/static/scripts.js b/backoffice/static/scripts.js new file mode 100644 index 0000000..0e1b8e4 --- /dev/null +++ b/backoffice/static/scripts.js @@ -0,0 +1,16 @@ +/*! + * jQuery JavaScript Library v1.4.1 + * http://jquery.com/ + * + * Copyright 2010, John Resig + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * Includes Sizzle.js + * http://sizzlejs.com/ + * Copyright 2010, The Dojo Foundation + * Released under the MIT, BSD, and GPL Licenses. + * + * Date: Mon Jan 25 19:43:33 2010 -0500 + */ +(function(z,v){function la(){if(!c.isReady){try{r.documentElement.doScroll("left")}catch(a){setTimeout(la,1);return}c.ready()}}function Ma(a,b){b.src?c.ajax({url:b.src,async:false,dataType:"script"}):c.globalEval(b.text||b.textContent||b.innerHTML||"");b.parentNode&&b.parentNode.removeChild(b)}function X(a,b,d,f,e,i){var j=a.length;if(typeof b==="object"){for(var n in b)X(a,n,b[n],f,e,d);return a}if(d!==v){f=!i&&f&&c.isFunction(d);for(n=0;n-1){i=j.data;i.beforeFilter&&i.beforeFilter[a.type]&&!i.beforeFilter[a.type](a)||f.push(j.selector)}else delete x[o]}i=c(a.target).closest(f,a.currentTarget);m=0;for(s=i.length;m)[^>]*$|^#([\w-]+)$/,Qa=/^.[^:#\[\.,]*$/,Ra=/\S/,Sa=/^(\s|\u00A0)+|(\s|\u00A0)+$/g,Ta=/^<(\w+)\s*\/?>(?:<\/\1>)?$/,O=navigator.userAgent,va=false,P=[],L,$=Object.prototype.toString,aa=Object.prototype.hasOwnProperty,ba=Array.prototype.push,Q=Array.prototype.slice,wa=Array.prototype.indexOf;c.fn=c.prototype={init:function(a,b){var d,f;if(!a)return this;if(a.nodeType){this.context=this[0]=a;this.length=1;return this}if(typeof a==="string")if((d=Pa.exec(a))&&(d[1]||!b))if(d[1]){f=b?b.ownerDocument||b:r;if(a=Ta.exec(a))if(c.isPlainObject(b)){a=[r.createElement(a[1])];c.fn.attr.call(a,b,true)}else a=[f.createElement(a[1])];else{a=ra([d[1]],[f]);a=(a.cacheable?a.fragment.cloneNode(true):a.fragment).childNodes}}else{if(b=r.getElementById(d[2])){if(b.id!==d[2])return S.find(a);this.length=1;this[0]=b}this.context=r;this.selector=a;return this}else if(!b&&/^\w+$/.test(a)){this.selector=a;this.context=r;a=r.getElementsByTagName(a)}else return!b||b.jquery?(b||S).find(a):c(b).find(a);else if(c.isFunction(a))return S.ready(a);if(a.selector!==v){this.selector=a.selector;this.context=a.context}return c.isArray(a)?this.setArray(a):c.makeArray(a,this)},selector:"",jquery:"1.4.1",length:0,size:function(){return this.length},toArray:function(){return Q.call(this,0)},get:function(a){return a==null?this.toArray():a<0?this.slice(a)[0]:this[a]},pushStack:function(a,b,d){a=c(a||null);a.prevObject=this;a.context=this.context;if(b==="find")a.selector=this.selector+(this.selector?" ":"")+d;else if(b)a.selector=this.selector+"."+b+"("+d+")";return a},setArray:function(a){this.length=0;ba.apply(this,a);return this},each:function(a,b){return c.each(this,a,b)},ready:function(a){c.bindReady();if(c.isReady)a.call(r,c);else P&&P.push(a);return this},eq:function(a){return a===-1?this.slice(a):this.slice(a,+a+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(Q.apply(this,arguments),"slice",Q.call(arguments).join(","))},map:function(a){return this.pushStack(c.map(this,function(b,d){return a.call(b,d,b)}))},end:function(){return this.prevObject||c(null)},push:ba,sort:[].sort,splice:[].splice};c.fn.init.prototype=c.fn;c.extend=c.fn.extend=function(){var a=arguments[0]||{},b=1,d=arguments.length,f=false,e,i,j,n;if(typeof a==="boolean"){f=a;a=arguments[1]||{};b=2}if(typeof a!=="object"&&!c.isFunction(a))a={};if(d===b){a=this;--b}for(;b
    a";var e=d.getElementsByTagName("*"),i=d.getElementsByTagName("a")[0];if(!(!e||!e.length||!i)){c.support={leadingWhitespace:d.firstChild.nodeType===3,tbody:!d.getElementsByTagName("tbody").length,htmlSerialize:!!d.getElementsByTagName("link").length,style:/red/.test(i.getAttribute("style")),hrefNormalized:i.getAttribute("href")==="/a",opacity:/^0.55$/.test(i.style.opacity),cssFloat:!!i.style.cssFloat,checkOn:d.getElementsByTagName("input")[0].value==="on",optSelected:r.createElement("select").appendChild(r.createElement("option")).selected,checkClone:false,scriptEval:false,noCloneEvent:true,boxModel:null};b.type="text/javascript";try{b.appendChild(r.createTextNode("window."+f+"=1;"))}catch(j){}a.insertBefore(b,a.firstChild);if(z[f]){c.support.scriptEval=true;delete z[f]}a.removeChild(b);if(d.attachEvent&&d.fireEvent){d.attachEvent("onclick",function n(){c.support.noCloneEvent=false;d.detachEvent("onclick",n)});d.cloneNode(true).fireEvent("onclick")}d=r.createElement("div");d.innerHTML="";a=r.createDocumentFragment();a.appendChild(d.firstChild);c.support.checkClone=a.cloneNode(true).cloneNode(true).lastChild.checked;c(function(){var n=r.createElement("div");n.style.width=n.style.paddingLeft="1px";r.body.appendChild(n);c.boxModel=c.support.boxModel=n.offsetWidth===2;r.body.removeChild(n).style.display="none"});a=function(n){var o=r.createElement("div");n="on"+n;var m=n in o;if(!m){o.setAttribute(n,"return;");m=typeof o[n]==="function"}return m};c.support.submitBubbles=a("submit");c.support.changeBubbles=a("change");a=b=d=e=i=null}})();c.props={"for":"htmlFor","class":"className",readonly:"readOnly",maxlength:"maxLength",cellspacing:"cellSpacing",rowspan:"rowSpan",colspan:"colSpan",tabindex:"tabIndex",usemap:"useMap",frameborder:"frameBorder"};var G="jQuery"+J(),Ua=0,xa={},Va={};c.extend({cache:{},expando:G,noData:{embed:true,object:true,applet:true},data:function(a,b,d){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==z?xa:a;var f=a[G],e=c.cache;if(!b&&!f)return null;f||(f=++Ua);if(typeof b==="object"){a[G]=f;e=e[f]=c.extend(true,{},b)}else e=e[f]?e[f]:typeof d==="undefined"?Va:(e[f]={});if(d!==v){a[G]=f;e[b]=d}return typeof b==="string"?e[b]:e}},removeData:function(a,b){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==z?xa:a;var d=a[G],f=c.cache,e=f[d];if(b){if(e){delete e[b];c.isEmptyObject(e)&&c.removeData(a)}}else{try{delete a[G]}catch(i){a.removeAttribute&&a.removeAttribute(G)}delete f[d]}}}});c.fn.extend({data:function(a,b){if(typeof a==="undefined"&&this.length)return c.data(this[0]);else if(typeof a==="object")return this.each(function(){c.data(this,a)});var d=a.split(".");d[1]=d[1]?"."+d[1]:"";if(b===v){var f=this.triggerHandler("getData"+d[1]+"!",[d[0]]);if(f===v&&this.length)f=c.data(this[0],a);return f===v&&d[1]?this.data(d[0]):f}else return this.trigger("setData"+d[1]+"!",[d[0],b]).each(function(){c.data(this,a,b)})},removeData:function(a){return this.each(function(){c.removeData(this,a)})}});c.extend({queue:function(a,b,d){if(a){b=(b||"fx")+"queue";var f=c.data(a,b);if(!d)return f||[];if(!f||c.isArray(d))f=c.data(a,b,c.makeArray(d));else f.push(d);return f}},dequeue:function(a,b){b=b||"fx";var d=c.queue(a,b),f=d.shift();if(f==="inprogress")f=d.shift();if(f){b==="fx"&&d.unshift("inprogress");f.call(a,function(){c.dequeue(a,b)})}}});c.fn.extend({queue:function(a,b){if(typeof a!=="string"){b=a;a="fx"}if(b===v)return c.queue(this[0],a);return this.each(function(){var d=c.queue(this,a,b);a==="fx"&&d[0]!=="inprogress"&&c.dequeue(this,a)})},dequeue:function(a){return this.each(function(){c.dequeue(this,a)})},delay:function(a,b){a=c.fx?c.fx.speeds[a]||a:a;b=b||"fx";return this.queue(b,function(){var d=this;setTimeout(function(){c.dequeue(d,b)},a)})},clearQueue:function(a){return this.queue(a||"fx",[])}});var ya=/[\n\t]/g,ca=/\s+/,Wa=/\r/g,Xa=/href|src|style/,Ya=/(button|input)/i,Za=/(button|input|object|select|textarea)/i,$a=/^(a|area)$/i,za=/radio|checkbox/;c.fn.extend({attr:function(a,b){return X(this,a,b,true,c.attr)},removeAttr:function(a){return this.each(function(){c.attr(this,a,"");this.nodeType===1&&this.removeAttribute(a)})},addClass:function(a){if(c.isFunction(a))return this.each(function(o){var m=c(this);m.addClass(a.call(this,o,m.attr("class")))});if(a&&typeof a==="string")for(var b=(a||"").split(ca),d=0,f=this.length;d-1)return true;return false},val:function(a){if(a===v){var b=this[0];if(b){if(c.nodeName(b,"option"))return(b.attributes.value||{}).specified?b.value:b.text;if(c.nodeName(b,"select")){var d=b.selectedIndex,f=[],e=b.options;b=b.type==="select-one";if(d<0)return null;var i=b?d:0;for(d=b?d+1:e.length;i=0;else if(c.nodeName(this,"select")){var x=c.makeArray(s);c("option",this).each(function(){this.selected=c.inArray(c(this).val(),x)>=0});if(!x.length)this.selectedIndex=-1}else this.value=s}})}});c.extend({attrFn:{val:true,css:true,html:true,text:true,data:true,width:true,height:true,offset:true},attr:function(a,b,d,f){if(!a||a.nodeType===3||a.nodeType===8)return v;if(f&&b in c.attrFn)return c(a)[b](d);f=a.nodeType!==1||!c.isXMLDoc(a);var e=d!==v;b=f&&c.props[b]||b;if(a.nodeType===1){var i=Xa.test(b);if(b in a&&f&&!i){if(e){b==="type"&&Ya.test(a.nodeName)&&a.parentNode&&c.error("type property can't be changed");a[b]=d}if(c.nodeName(a,"form")&&a.getAttributeNode(b))return a.getAttributeNode(b).nodeValue;if(b==="tabIndex")return(b=a.getAttributeNode("tabIndex"))&&b.specified?b.value:Za.test(a.nodeName)||$a.test(a.nodeName)&&a.href?0:v;return a[b]}if(!c.support.style&&f&&b==="style"){if(e)a.style.cssText=""+d;return a.style.cssText}e&&a.setAttribute(b,""+d);a=!c.support.hrefNormalized&&f&&i?a.getAttribute(b,2):a.getAttribute(b);return a===null?v:a}return c.style(a,b,d)}});var ab=function(a){return a.replace(/[^\w\s\.\|`]/g,function(b){return"\\"+b})};c.event={add:function(a,b,d,f){if(!(a.nodeType===3||a.nodeType===8)){if(a.setInterval&&a!==z&&!a.frameElement)a=z;if(!d.guid)d.guid=c.guid++;if(f!==v){d=c.proxy(d);d.data=f}var e=c.data(a,"events")||c.data(a,"events",{}),i=c.data(a,"handle"),j;if(!i){j=function(){return typeof c!=="undefined"&&!c.event.triggered?c.event.handle.apply(j.elem,arguments):v};i=c.data(a,"handle",j)}if(i){i.elem=a;b=b.split(/\s+/);for(var n,o=0;n=b[o++];){var m=n.split(".");n=m.shift();if(o>1){d=c.proxy(d);if(f!==v)d.data=f}d.type=m.slice(0).sort().join(".");var s=e[n],x=this.special[n]||{};if(!s){s=e[n]={};if(!x.setup||x.setup.call(a,f,m,d)===false)if(a.addEventListener)a.addEventListener(n,i,false);else a.attachEvent&&a.attachEvent("on"+n,i)}if(x.add)if((m=x.add.call(a,d,f,m,s))&&c.isFunction(m)){m.guid=m.guid||d.guid;m.data=m.data||d.data;m.type=m.type||d.type;d=m}s[d.guid]=d;this.global[n]=true}a=null}}},global:{},remove:function(a,b,d){if(!(a.nodeType===3||a.nodeType===8)){var f=c.data(a,"events"),e,i,j;if(f){if(b===v||typeof b==="string"&&b.charAt(0)===".")for(i in f)this.remove(a,i+(b||""));else{if(b.type){d=b.handler;b=b.type}b=b.split(/\s+/);for(var n=0;i=b[n++];){var o=i.split(".");i=o.shift();var m=!o.length,s=c.map(o.slice(0).sort(),ab);s=new RegExp("(^|\\.)"+s.join("\\.(?:.*\\.)?")+"(\\.|$)");var x=this.special[i]||{};if(f[i]){if(d){j=f[i][d.guid];delete f[i][d.guid]}else for(var A in f[i])if(m||s.test(f[i][A].type))delete f[i][A];x.remove&&x.remove.call(a,o,j);for(e in f[i])break;if(!e){if(!x.teardown||x.teardown.call(a,o)===false)if(a.removeEventListener)a.removeEventListener(i,c.data(a,"handle"),false);else a.detachEvent&&a.detachEvent("on"+i,c.data(a,"handle"));e=null;delete f[i]}}}}for(e in f)break;if(!e){if(A=c.data(a,"handle"))A.elem=null;c.removeData(a,"events");c.removeData(a,"handle")}}}},trigger:function(a,b,d,f){var e=a.type||a;if(!f){a=typeof a==="object"?a[G]?a:c.extend(c.Event(e),a):c.Event(e);if(e.indexOf("!")>=0){a.type=e=e.slice(0,-1);a.exclusive=true}if(!d){a.stopPropagation();this.global[e]&&c.each(c.cache,function(){this.events&&this.events[e]&&c.event.trigger(a,b,this.handle.elem)})}if(!d||d.nodeType===3||d.nodeType===8)return v;a.result=v;a.target=d;b=c.makeArray(b);b.unshift(a)}a.currentTarget=d;(f=c.data(d,"handle"))&&f.apply(d,b);f=d.parentNode||d.ownerDocument;try{if(!(d&&d.nodeName&&c.noData[d.nodeName.toLowerCase()]))if(d["on"+e]&&d["on"+e].apply(d,b)===false)a.result=false}catch(i){}if(!a.isPropagationStopped()&&f)c.event.trigger(a,b,f,true);else if(!a.isDefaultPrevented()){d=a.target;var j;if(!(c.nodeName(d,"a")&&e==="click")&&!(d&&d.nodeName&&c.noData[d.nodeName.toLowerCase()])){try{if(d[e]){if(j=d["on"+e])d["on"+e]=null;this.triggered=true;d[e]()}}catch(n){}if(j)d["on"+e]=j;this.triggered=false}}},handle:function(a){var b,d;a=arguments[0]=c.event.fix(a||z.event);a.currentTarget=this;d=a.type.split(".");a.type=d.shift();b=!d.length&&!a.exclusive;var f=new RegExp("(^|\\.)"+d.slice(0).sort().join("\\.(?:.*\\.)?")+"(\\.|$)");d=(c.data(this,"events")||{})[a.type];for(var e in d){var i=d[e];if(b||f.test(i.type)){a.handler=i;a.data=i.data;i=i.apply(this,arguments);if(i!==v){a.result=i;if(i===false){a.preventDefault();a.stopPropagation()}}if(a.isImmediatePropagationStopped())break}}return a.result},props:"altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode layerX layerY metaKey newValue offsetX offsetY originalTarget pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target toElement view wheelDelta which".split(" "),fix:function(a){if(a[G])return a;var b=a;a=c.Event(b);for(var d=this.props.length,f;d;){f=this.props[--d];a[f]=b[f]}if(!a.target)a.target=a.srcElement||r;if(a.target.nodeType===3)a.target=a.target.parentNode;if(!a.relatedTarget&&a.fromElement)a.relatedTarget=a.fromElement===a.target?a.toElement:a.fromElement;if(a.pageX==null&&a.clientX!=null){b=r.documentElement;d=r.body;a.pageX=a.clientX+(b&&b.scrollLeft||d&&d.scrollLeft||0)-(b&&b.clientLeft||d&&d.clientLeft||0);a.pageY=a.clientY+(b&&b.scrollTop||d&&d.scrollTop||0)-(b&&b.clientTop||d&&d.clientTop||0)}if(!a.which&&(a.charCode||a.charCode===0?a.charCode:a.keyCode))a.which=a.charCode||a.keyCode;if(!a.metaKey&&a.ctrlKey)a.metaKey=a.ctrlKey;if(!a.which&&a.button!==v)a.which=a.button&1?1:a.button&2?3:a.button&4?2:0;return a},guid:1E8,proxy:c.proxy,special:{ready:{setup:c.bindReady,teardown:c.noop},live:{add:function(a,b){c.extend(a,b||{});a.guid+=b.selector+b.live;b.liveProxy=a;c.event.add(this,b.live,na,b)},remove:function(a){if(a.length){var b=0,d=new RegExp("(^|\\.)"+a[0]+"(\\.|$)");c.each(c.data(this,"events").live||{},function(){d.test(this.type)&&b++});b<1&&c.event.remove(this,a[0],na)}},special:{}},beforeunload:{setup:function(a,b,d){if(this.setInterval)this.onbeforeunload=d;return false},teardown:function(a,b){if(this.onbeforeunload===b)this.onbeforeunload=null}}}};c.Event=function(a){if(!this.preventDefault)return new c.Event(a);if(a&&a.type){this.originalEvent=a;this.type=a.type}else this.type=a;this.timeStamp=J();this[G]=true};c.Event.prototype={preventDefault:function(){this.isDefaultPrevented=Z;var a=this.originalEvent;if(a){a.preventDefault&&a.preventDefault();a.returnValue=false}},stopPropagation:function(){this.isPropagationStopped=Z;var a=this.originalEvent;if(a){a.stopPropagation&&a.stopPropagation();a.cancelBubble=true}},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=Z;this.stopPropagation()},isDefaultPrevented:Y,isPropagationStopped:Y,isImmediatePropagationStopped:Y};var Aa=function(a){for(var b=a.relatedTarget;b&&b!==this;)try{b=b.parentNode}catch(d){break}if(b!==this){a.type=a.data;c.event.handle.apply(this,arguments)}},Ba=function(a){a.type=a.data;c.event.handle.apply(this,arguments)};c.each({mouseenter:"mouseover",mouseleave:"mouseout"},function(a,b){c.event.special[a]={setup:function(d){c.event.add(this,b,d&&d.selector?Ba:Aa,a)},teardown:function(d){c.event.remove(this,b,d&&d.selector?Ba:Aa)}}});if(!c.support.submitBubbles)c.event.special.submit={setup:function(a,b,d){if(this.nodeName.toLowerCase()!=="form"){c.event.add(this,"click.specialSubmit."+d.guid,function(f){var e=f.target,i=e.type;if((i==="submit"||i==="image")&&c(e).closest("form").length)return ma("submit",this,arguments)});c.event.add(this,"keypress.specialSubmit."+d.guid,function(f){var e=f.target,i=e.type;if((i==="text"||i==="password")&&c(e).closest("form").length&&f.keyCode===13)return ma("submit",this,arguments)})}else return false},remove:function(a,b){c.event.remove(this,"click.specialSubmit"+(b?"."+b.guid:""));c.event.remove(this,"keypress.specialSubmit"+(b?"."+b.guid:""))}};if(!c.support.changeBubbles){var da=/textarea|input|select/i;function Ca(a){var b=a.type,d=a.value;if(b==="radio"||b==="checkbox")d=a.checked;else if(b==="select-multiple")d=a.selectedIndex>-1?c.map(a.options,function(f){return f.selected}).join("-"):"";else if(a.nodeName.toLowerCase()==="select")d=a.selectedIndex;return d}function ea(a,b){var d=a.target,f,e;if(!(!da.test(d.nodeName)||d.readOnly)){f=c.data(d,"_change_data");e=Ca(d);if(a.type!=="focusout"||d.type!=="radio")c.data(d,"_change_data",e);if(!(f===v||e===f))if(f!=null||e){a.type="change";return c.event.trigger(a,b,d)}}}c.event.special.change={filters:{focusout:ea,click:function(a){var b=a.target,d=b.type;if(d==="radio"||d==="checkbox"||b.nodeName.toLowerCase()==="select")return ea.call(this,a)},keydown:function(a){var b=a.target,d=b.type;if(a.keyCode===13&&b.nodeName.toLowerCase()!=="textarea"||a.keyCode===32&&(d==="checkbox"||d==="radio")||d==="select-multiple")return ea.call(this,a)},beforeactivate:function(a){a=a.target;a.nodeName.toLowerCase()==="input"&&a.type==="radio"&&c.data(a,"_change_data",Ca(a))}},setup:function(a,b,d){for(var f in T)c.event.add(this,f+".specialChange."+d.guid,T[f]);return da.test(this.nodeName)},remove:function(a,b){for(var d in T)c.event.remove(this,d+".specialChange"+(b?"."+b.guid:""),T[d]);return da.test(this.nodeName)}};var T=c.event.special.change.filters}r.addEventListener&&c.each({focus:"focusin",blur:"focusout"},function(a,b){function d(f){f=c.event.fix(f);f.type=b;return c.event.handle.call(this,f)}c.event.special[b]={setup:function(){this.addEventListener(a,d,true)},teardown:function(){this.removeEventListener(a,d,true)}}});c.each(["bind","one"],function(a,b){c.fn[b]=function(d,f,e){if(typeof d==="object"){for(var i in d)this[b](i,f,d[i],e);return this}if(c.isFunction(f)){e=f;f=v}var j=b==="one"?c.proxy(e,function(n){c(this).unbind(n,j);return e.apply(this,arguments)}):e;return d==="unload"&&b!=="one"?this.one(d,f,e):this.each(function(){c.event.add(this,d,j,f)})}});c.fn.extend({unbind:function(a,b){if(typeof a==="object"&&!a.preventDefault){for(var d in a)this.unbind(d,a[d]);return this}return this.each(function(){c.event.remove(this,a,b)})},trigger:function(a,b){return this.each(function(){c.event.trigger(a,b,this)})},triggerHandler:function(a,b){if(this[0]){a=c.Event(a);a.preventDefault();a.stopPropagation();c.event.trigger(a,b,this[0]);return a.result}},toggle:function(a){for(var b=arguments,d=1;d0){y=t;break}}t=t[g]}l[q]=y}}}var f=/((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^[\]]*\]|['"][^'"]*['"]|[^[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g,e=0,i=Object.prototype.toString,j=false,n=true;[0,0].sort(function(){n=false;return 0});var o=function(g,h,k,l){k=k||[];var q=h=h||r;if(h.nodeType!==1&&h.nodeType!==9)return[];if(!g||typeof g!=="string")return k;for(var p=[],u,t,y,R,H=true,M=w(h),I=g;(f.exec(""),u=f.exec(I))!==null;){I=u[3];p.push(u[1]);if(u[2]){R=u[3];break}}if(p.length>1&&s.exec(g))if(p.length===2&&m.relative[p[0]])t=fa(p[0]+p[1],h);else for(t=m.relative[p[0]]?[h]:o(p.shift(),h);p.length;){g=p.shift();if(m.relative[g])g+=p.shift();t=fa(g,t)}else{if(!l&&p.length>1&&h.nodeType===9&&!M&&m.match.ID.test(p[0])&&!m.match.ID.test(p[p.length-1])){u=o.find(p.shift(),h,M);h=u.expr?o.filter(u.expr,u.set)[0]:u.set[0]}if(h){u=l?{expr:p.pop(),set:A(l)}:o.find(p.pop(),p.length===1&&(p[0]==="~"||p[0]==="+")&&h.parentNode?h.parentNode:h,M);t=u.expr?o.filter(u.expr,u.set):u.set;if(p.length>0)y=A(t);else H=false;for(;p.length;){var D=p.pop();u=D;if(m.relative[D])u=p.pop();else D="";if(u==null)u=h;m.relative[D](y,u,M)}}else y=[]}y||(y=t);y||o.error(D||g);if(i.call(y)==="[object Array]")if(H)if(h&&h.nodeType===1)for(g=0;y[g]!=null;g++){if(y[g]&&(y[g]===true||y[g].nodeType===1&&E(h,y[g])))k.push(t[g])}else for(g=0;y[g]!=null;g++)y[g]&&y[g].nodeType===1&&k.push(t[g]);else k.push.apply(k,y);else A(y,k);if(R){o(R,q,k,l);o.uniqueSort(k)}return k};o.uniqueSort=function(g){if(C){j=n;g.sort(C);if(j)for(var h=1;h":function(g,h){var k=typeof h==="string";if(k&&!/\W/.test(h)){h=h.toLowerCase();for(var l=0,q=g.length;l=0))k||l.push(u);else if(k)h[p]=false;return false},ID:function(g){return g[1].replace(/\\/g,"")},TAG:function(g){return g[1].toLowerCase()},CHILD:function(g){if(g[1]==="nth"){var h=/(-?)(\d*)n((?:\+|-)?\d*)/.exec(g[2]==="even"&&"2n"||g[2]==="odd"&&"2n+1"||!/\D/.test(g[2])&&"0n+"+g[2]||g[2]);g[2]=h[1]+(h[2]||1)-0;g[3]=h[3]-0}g[0]=e++;return g},ATTR:function(g,h,k,l,q,p){h=g[1].replace(/\\/g,"");if(!p&&m.attrMap[h])g[1]=m.attrMap[h];if(g[2]==="~=")g[4]=" "+g[4]+" ";return g},PSEUDO:function(g,h,k,l,q){if(g[1]==="not")if((f.exec(g[3])||"").length>1||/^\w/.test(g[3]))g[3]=o(g[3],null,null,h);else{g=o.filter(g[3],h,k,true^q);k||l.push.apply(l,g);return false}else if(m.match.POS.test(g[0])||m.match.CHILD.test(g[0]))return true;return g},POS:function(g){g.unshift(true);return g}},filters:{enabled:function(g){return g.disabled===false&&g.type!=="hidden"},disabled:function(g){return g.disabled===true},checked:function(g){return g.checked===true},selected:function(g){return g.selected===true},parent:function(g){return!!g.firstChild},empty:function(g){return!g.firstChild},has:function(g,h,k){return!!o(k[3],g).length},header:function(g){return/h\d/i.test(g.nodeName)},text:function(g){return"text"===g.type},radio:function(g){return"radio"===g.type},checkbox:function(g){return"checkbox"===g.type},file:function(g){return"file"===g.type},password:function(g){return"password"===g.type},submit:function(g){return"submit"===g.type},image:function(g){return"image"===g.type},reset:function(g){return"reset"===g.type},button:function(g){return"button"===g.type||g.nodeName.toLowerCase()==="button"},input:function(g){return/input|select|textarea|button/i.test(g.nodeName)}},setFilters:{first:function(g,h){return h===0},last:function(g,h,k,l){return h===l.length-1},even:function(g,h){return h%2===0},odd:function(g,h){return h%2===1},lt:function(g,h,k){return hk[3]-0},nth:function(g,h,k){return k[3]-0===h},eq:function(g,h,k){return k[3]-0===h}},filter:{PSEUDO:function(g,h,k,l){var q=h[1],p=m.filters[q];if(p)return p(g,k,h,l);else if(q==="contains")return(g.textContent||g.innerText||a([g])||"").indexOf(h[3])>=0;else if(q==="not"){h=h[3];k=0;for(l=h.length;k=0}},ID:function(g,h){return g.nodeType===1&&g.getAttribute("id")===h},TAG:function(g,h){return h==="*"&&g.nodeType===1||g.nodeName.toLowerCase()===h},CLASS:function(g,h){return(" "+(g.className||g.getAttribute("class"))+" ").indexOf(h)>-1},ATTR:function(g,h){var k=h[1];g=m.attrHandle[k]?m.attrHandle[k](g):g[k]!=null?g[k]:g.getAttribute(k);k=g+"";var l=h[2];h=h[4];return g==null?l==="!=":l==="="?k===h:l==="*="?k.indexOf(h)>=0:l==="~="?(" "+k+" ").indexOf(h)>=0:!h?k&&g!==false:l==="!="?k!==h:l==="^="?k.indexOf(h)===0:l==="$="?k.substr(k.length-h.length)===h:l==="|="?k===h||k.substr(0,h.length+1)===h+"-":false},POS:function(g,h,k,l){var q=m.setFilters[h[2]];if(q)return q(g,k,h,l)}}},s=m.match.POS;for(var x in m.match){m.match[x]=new RegExp(m.match[x].source+/(?![^\[]*\])(?![^\(]*\))/.source);m.leftMatch[x]=new RegExp(/(^(?:.|\r|\n)*?)/.source+m.match[x].source.replace(/\\(\d+)/g,function(g,h){return"\\"+(h-0+1)}))}var A=function(g,h){g=Array.prototype.slice.call(g,0);if(h){h.push.apply(h,g);return h}return g};try{Array.prototype.slice.call(r.documentElement.childNodes,0)}catch(B){A=function(g,h){h=h||[];if(i.call(g)==="[object Array]")Array.prototype.push.apply(h,g);else if(typeof g.length==="number")for(var k=0,l=g.length;k";var k=r.documentElement;k.insertBefore(g,k.firstChild);if(r.getElementById(h)){m.find.ID=function(l,q,p){if(typeof q.getElementById!=="undefined"&&!p)return(q=q.getElementById(l[1]))?q.id===l[1]||typeof q.getAttributeNode!=="undefined"&&q.getAttributeNode("id").nodeValue===l[1]?[q]:v:[]};m.filter.ID=function(l,q){var p=typeof l.getAttributeNode!=="undefined"&&l.getAttributeNode("id");return l.nodeType===1&&p&&p.nodeValue===q}}k.removeChild(g);k=g=null})();(function(){var g=r.createElement("div");g.appendChild(r.createComment(""));if(g.getElementsByTagName("*").length>0)m.find.TAG=function(h,k){k=k.getElementsByTagName(h[1]);if(h[1]==="*"){h=[];for(var l=0;k[l];l++)k[l].nodeType===1&&h.push(k[l]);k=h}return k};g.innerHTML="";if(g.firstChild&&typeof g.firstChild.getAttribute!=="undefined"&&g.firstChild.getAttribute("href")!=="#")m.attrHandle.href=function(h){return h.getAttribute("href",2)};g=null})();r.querySelectorAll&&function(){var g=o,h=r.createElement("div");h.innerHTML="

    ";if(!(h.querySelectorAll&&h.querySelectorAll(".TEST").length===0)){o=function(l,q,p,u){q=q||r;if(!u&&q.nodeType===9&&!w(q))try{return A(q.querySelectorAll(l),p)}catch(t){}return g(l,q,p,u)};for(var k in g)o[k]=g[k];h=null}}();(function(){var g=r.createElement("div");g.innerHTML="
    ";if(!(!g.getElementsByClassName||g.getElementsByClassName("e").length===0)){g.lastChild.className="e";if(g.getElementsByClassName("e").length!==1){m.order.splice(1,0,"CLASS");m.find.CLASS=function(h,k,l){if(typeof k.getElementsByClassName!=="undefined"&&!l)return k.getElementsByClassName(h[1])};g=null}}})();var E=r.compareDocumentPosition?function(g,h){return g.compareDocumentPosition(h)&16}:function(g,h){return g!==h&&(g.contains?g.contains(h):true)},w=function(g){return(g=(g?g.ownerDocument||g:0).documentElement)?g.nodeName!=="HTML":false},fa=function(g,h){var k=[],l="",q;for(h=h.nodeType?[h]:h;q=m.match.PSEUDO.exec(g);){l+=q[0];g=g.replace(m.match.PSEUDO,"")}g=m.relative[g]?g+"*":g;q=0;for(var p=h.length;q=0===d})};c.fn.extend({find:function(a){for(var b=this.pushStack("","find",a),d=0,f=0,e=this.length;f0)for(var i=d;i0},closest:function(a,b){if(c.isArray(a)){var d=[],f=this[0],e,i={},j;if(f&&a.length){e=0;for(var n=a.length;e-1:c(f).is(e)){d.push({selector:j,elem:f});delete i[j]}}f=f.parentNode}}return d}var o=c.expr.match.POS.test(a)?c(a,b||this.context):null;return this.map(function(m,s){for(;s&&s.ownerDocument&&s!==b;){if(o?o.index(s)>-1:c(s).is(a))return s;s=s.parentNode}return null})},index:function(a){if(!a||typeof a==="string")return c.inArray(this[0],a?c(a):this.parent().children());return c.inArray(a.jquery?a[0]:a,this)},add:function(a,b){a=typeof a==="string"?c(a,b||this.context):c.makeArray(a);b=c.merge(this.get(),a);return this.pushStack(pa(a[0])||pa(b[0])?b:c.unique(b))},andSelf:function(){return this.add(this.prevObject)}});c.each({parent:function(a){return(a=a.parentNode)&&a.nodeType!==11?a:null},parents:function(a){return c.dir(a,"parentNode")},parentsUntil:function(a,b,d){return c.dir(a,"parentNode",d)},next:function(a){return c.nth(a,2,"nextSibling")},prev:function(a){return c.nth(a,2,"previousSibling")},nextAll:function(a){return c.dir(a,"nextSibling")},prevAll:function(a){return c.dir(a,"previousSibling")},nextUntil:function(a,b,d){return c.dir(a,"nextSibling",d)},prevUntil:function(a,b,d){return c.dir(a,"previousSibling",d)},siblings:function(a){return c.sibling(a.parentNode.firstChild,a)},children:function(a){return c.sibling(a.firstChild)},contents:function(a){return c.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:c.makeArray(a.childNodes)}},function(a,b){c.fn[a]=function(d,f){var e=c.map(this,b,d);bb.test(a)||(f=d);if(f&&typeof f==="string")e=c.filter(f,e);e=this.length>1?c.unique(e):e;if((this.length>1||db.test(f))&&cb.test(a))e=e.reverse();return this.pushStack(e,a,Q.call(arguments).join(","))}});c.extend({filter:function(a,b,d){if(d)a=":not("+a+")";return c.find.matches(a,b)},dir:function(a,b,d){var f=[];for(a=a[b];a&&a.nodeType!==9&&(d===v||a.nodeType!==1||!c(a).is(d));){a.nodeType===1&&f.push(a);a=a[b]}return f},nth:function(a,b,d){b=b||1;for(var f=0;a;a=a[d])if(a.nodeType===1&&++f===b)break;return a},sibling:function(a,b){for(var d=[];a;a=a.nextSibling)a.nodeType===1&&a!==b&&d.push(a);return d}});var Fa=/ jQuery\d+="(?:\d+|null)"/g,V=/^\s+/,Ga=/(<([\w:]+)[^>]*?)\/>/g,eb=/^(?:area|br|col|embed|hr|img|input|link|meta|param)$/i,Ha=/<([\w:]+)/,fb=/"},F={option:[1,""],legend:[1,"
    ","
    "],thead:[1,"","
    "],tr:[2,"","
    "],td:[3,"","
    "],col:[2,"","
    "],area:[1,"",""],_default:[0,"",""]};F.optgroup=F.option;F.tbody=F.tfoot=F.colgroup=F.caption=F.thead;F.th=F.td;if(!c.support.htmlSerialize)F._default=[1,"div
    ","
    "];c.fn.extend({text:function(a){if(c.isFunction(a))return this.each(function(b){var d=c(this);d.text(a.call(this,b,d.text()))});if(typeof a!=="object"&&a!==v)return this.empty().append((this[0]&&this[0].ownerDocument||r).createTextNode(a));return c.getText(this)},wrapAll:function(a){if(c.isFunction(a))return this.each(function(d){c(this).wrapAll(a.call(this,d))});if(this[0]){var b=c(a,this[0].ownerDocument).eq(0).clone(true);this[0].parentNode&&b.insertBefore(this[0]);b.map(function(){for(var d=this;d.firstChild&&d.firstChild.nodeType===1;)d=d.firstChild;return d}).append(this)}return this},wrapInner:function(a){if(c.isFunction(a))return this.each(function(b){c(this).wrapInner(a.call(this,b))});return this.each(function(){var b=c(this),d=b.contents();d.length?d.wrapAll(a):b.append(a)})},wrap:function(a){return this.each(function(){c(this).wrapAll(a)})},unwrap:function(){return this.parent().each(function(){c.nodeName(this,"body")||c(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.appendChild(a)})},prepend:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.insertBefore(a,this.firstChild)})},before:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this)});else if(arguments.length){var a=c(arguments[0]);a.push.apply(a,this.toArray());return this.pushStack(a,"before",arguments)}},after:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this.nextSibling)});else if(arguments.length){var a=this.pushStack(this,"after",arguments);a.push.apply(a,c(arguments[0]).toArray());return a}},clone:function(a){var b=this.map(function(){if(!c.support.noCloneEvent&&!c.isXMLDoc(this)){var d=this.outerHTML,f=this.ownerDocument;if(!d){d=f.createElement("div");d.appendChild(this.cloneNode(true));d=d.innerHTML}return c.clean([d.replace(Fa,"").replace(V,"")],f)[0]}else return this.cloneNode(true)});if(a===true){qa(this,b);qa(this.find("*"),b.find("*"))}return b},html:function(a){if(a===v)return this[0]&&this[0].nodeType===1?this[0].innerHTML.replace(Fa,""):null;else if(typeof a==="string"&&!/ + + + {% block extrahead %} + + + {% endblock %} + + + + + +
    +
    + + + +
    +{% block content %} +{% endblock %} +
    + + + +
    +
    + + + + diff --git a/backoffice/templates/beta_invitations.html b/backoffice/templates/beta_invitations.html new file mode 100644 index 0000000..694f71b --- /dev/null +++ b/backoffice/templates/beta_invitations.html @@ -0,0 +1,66 @@ +{% extends "base.html" %} +{% load custom_tags %} +{% load humanize %} +{% block content %} +
    +
    +
    +
    +

    Beta Invitations

    +
    +
    +
    +
    +
    +
    + + + + + + + + + + {% for beta_invitation in invitations %} + + + + + + + {% empty %} + {% endfor %} + + + + + + + + +
    + CUSTOMER + + PASSWORD + + ACCOUNT? + + DATE +
    + {% if beta_invitation.beta_requester %}{{ beta_invitation.beta_requester.email }}{% else %}{{ beta_invitation.assigned_customer }}{% endif %} + + {{ beta_invitation.password }} (link) + + {% if beta_invitation.account %}{{ beta_invitation.account.user.email }}{% else %}NOT YET{% endif %} + + {{ beta_invitation.invitation_date|date:'Y/m/d H:i'}} +
    + New Invitee: {{ form.requesting_customer }} +
    +
    +
    +
    +
    + +{% endblock %} \ No newline at end of file diff --git a/backoffice/templates/beta_requests.html b/backoffice/templates/beta_requests.html new file mode 100644 index 0000000..b0a5e6b --- /dev/null +++ b/backoffice/templates/beta_requests.html @@ -0,0 +1,67 @@ +{% extends "base.html" %} +{% load custom_tags %} +{% load humanize %} +{% block content %} +
    +
    +
    +
    +

    Beta Requests

    +
    +
    +
    +
    +
    +
    + + + + + + + + + + + {% for beta_request in requests %} + + + + + + + + + + {% empty %} + {% endfor %} + +
    + E-MAIL + + SITE + + INTENDED USE + + REQUESTED + + SIGN UP + TOTAL: {{ requests.count }}
    + {{ beta_request.email }} + + {{ beta_request.site_url }} + + {{ beta_request.summary }} + + {{ beta_request.request_date|date:"M j, gA" }} + + {{ beta_request.invitation.all.0.account.creation_time|date:"M j, gA" }} + + {% if beta_request.invitation.all %}Invited on {{ beta_request.invitation.all.0.invitation_date|date:"M j, gA" }}{% else %}Invite{% endif %} +
    +
    +
    +
    +
    + +{% endblock %} diff --git a/backoffice/templates/biz_stats.html b/backoffice/templates/biz_stats.html new file mode 100644 index 0000000..ed4c6b5 --- /dev/null +++ b/backoffice/templates/biz_stats.html @@ -0,0 +1,94 @@ + + + + Biz + + + + + + + +
    +

    Twitter History

    +
    +
    +
    +

    Subscriptions Detail

    +
    +

    Revenue

    +
    +
    +
    +

    Queries per day

    +
    + (-Free)   |   + (*Factual)   |   +
    +
    +
    +
    +

    Accounts History

    +
    +
    +
    +

    API History

    +
    +
    +

    Accounts Detail

    +
    + (-Free)   |   + (*Factual)   |   + (-Inactive)   |   + Reset   |   +
    +
    +
    +
    +

    Index detail

    +
    +
    +
    +

    QoS

    +

    GET API Time [ms]: Account / HTTP method / GMT hour

    +
    +

    Logged API Errors: Account / GMT day

    +
    +

    GET API Time [ms]: Account / 95% / GMT month

    +
    +
    +
    +

    Activity

    +
    +
    +
    +
    +

    Adoption

    +

    Monthly

    +
    +

    Weekly

    +
    +

    Daily

    +
    +

    Activation

    +

    Weekly, accounts that created an index and stored at least one document

    +
    +

    Retention

    +

    In the last four days, accounts that had GET and PUT activity

    +
    +
    + + + diff --git a/backoffice/templates/control_panel.html b/backoffice/templates/control_panel.html new file mode 100644 index 0000000..9597d37 --- /dev/null +++ b/backoffice/templates/control_panel.html @@ -0,0 +1,74 @@ +{% extends "base.html" %} +{% load custom_tags %} +{% load humanize %} +{% block content %} +
    +
    +
    +
    +

    IndexTank Control Panel

    +
    +
    +
    +
    +
    + {#

    Manage your indexes

    #} +
    + + + + + + + + + + {% for worker in workers %} + + + + + + + + + + + + {% empty %} + {% endfor %} + + + + +
    + ID + + INSTANCE NAME + + WAN_DNS + + DEPLOY COUNT + + XMX +
    + {{ worker.id }} + + {{ worker.instance_name }} + + {{ worker.wan_dns }} + + {{ worker.deploys.all|length }} + + {{ worker.get_used_ram }}M / {{ worker.ram }}M + + Check Stats +
    + There is a total of {{ workers|length }} workers running. +
    +
    +
    +
    +
    + +{% endblock %} diff --git a/backoffice/templates/index_history.html b/backoffice/templates/index_history.html new file mode 100644 index 0000000..6eb7b52 --- /dev/null +++ b/backoffice/templates/index_history.html @@ -0,0 +1,128 @@ +{% extends "base.html" %} +{% load custom_tags %} +{% load humanize %} + + +{% block extramenu %} +
  • |
  • Back to Worker '{{ worker.instance_name }}'
    • +{% endblock %} + +{% block content %} + + + + + + + + +
    +
    +
    +
    +

    Index {{ index.code }} stats

    +

    + Check the memory and disk usage history of the Index.
    +

    +
    +
    +
    +
    +
    +
    + +
    + + + + + + + + + + + + + + + +
    + Memory Usage + +   +
    +
    +
    +   +
    +
    + + +
    + + + + + + + + + + + + + + + +
    + Disk Usage + +   +
    +
    +
    +   +
    +
    + + + + + +{% endblock %} \ No newline at end of file diff --git a/backoffice/templates/load_history.html b/backoffice/templates/load_history.html new file mode 100644 index 0000000..e927600 --- /dev/null +++ b/backoffice/templates/load_history.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} +{% load custom_tags %} +{% load humanize %} +{% block content %} + + + + + + + + +
    +
    + +
    +
    +

    Worker {{ worker.id }} ({{ worker.wan_dns }} - {{ worker.instance_name }}), load stats

    +

    + Check the load average history for the worker.
    +

    +
    +
    +

    +
    +
    +
    +
    + +
    + + + + + + + + + + + + + + + +
    + Load Average History + +   +
    +
    +
    +   +
    +
    + + + + + +{% endblock %} \ No newline at end of file diff --git a/backoffice/templates/login.html b/backoffice/templates/login.html new file mode 100644 index 0000000..949de52 --- /dev/null +++ b/backoffice/templates/login.html @@ -0,0 +1,46 @@ +{% extends "base.html" %} +{% load custom_tags %} + +{% block content %} +
    +
    +

    Log in to IndexTank!

    + +

    Just enter your email address and password.

    +
    + +
    + + + {% for f in login_form %} + + + + + {% endfor %} + {% if login_message %} + + + + + {% else %} + {% endif %} + + + + + +
    {{ f.label_tag }}{{ f }} +
    {% for e in f.errors %}{{e}} {% endfor %}
    +
    + {{ login_message }} +
    + LOGIN +
    +
    +
    +
    + +
    +{% endblock %} + diff --git a/backoffice/templates/manage_worker.html b/backoffice/templates/manage_worker.html new file mode 100644 index 0000000..b6abb93 --- /dev/null +++ b/backoffice/templates/manage_worker.html @@ -0,0 +1,121 @@ +{% extends "base.html" %} +{% load custom_tags %} +{% load humanize %} +{% block content %} +
    +
    +
    +
    +

    CPU Stats Control panel

    +

    + Check the stats of the worker to prevent performance issues.
    + Worker {{ worker.id }} - {{ worker.wan_dns }} - {{ worker.instance_name }}
    + Current Load Average: {{ load_info.load_average }} (Check history) +

    +
    +
    +

    +
    +
    +
    +
    +
    +

    Filesystem

    +
    + + + + + + + + + {% for info in mount_infos %} + + + + + + + {% empty %} + {% endfor %} + + + + + +
    + MOUNT + + AVAILABLE SPACE + + USED SPACE + +   +
    + {{ info.mount }} + + {{ info.available }} + + {{ info.used }} ({{ mount_percentages|get:info.mount|floatformat:2 }}%) + + History +
    + There is a total of {{ mount_infos|length }} devices. +
    +
    +
    +
    +

    Indexes

    +
    + + + + + + + + + {% for index_info in indexes_infos %} + + + + + + + {% empty %} + {% endfor %} + + + + + + +
    + INDEX + + MEMORY USED + + DISK USED + +   +
    + "{{ index_info.deploy.index.name }}" [{{ index_info.deploy.index.code }}] ({{ index_info.deploy.index.account.user.email }}) + + {{ index_info.used_mem }} Mb + + {{ index_info.used_disk }} Mb + + History +
    + Total: + + {{ used_mem_total }} Mb + + {{ used_disk_total }} Mb +
    +
    +
    +
    +
    +{% endblock %} \ No newline at end of file diff --git a/backoffice/templates/mount_history.html b/backoffice/templates/mount_history.html new file mode 100644 index 0000000..b715408 --- /dev/null +++ b/backoffice/templates/mount_history.html @@ -0,0 +1,81 @@ +{% extends "base.html" %} +{% load custom_tags %} +{% load humanize %} +{% block extramenu %} +
  • |
  • Back to Worker '{{ worker.instance_name }}'
    • +{% endblock %} + +{% block content %} + + + + + + + + +
    +
    + +
    +
    +

    Worker {{ worker.id }} ({{ worker.wan_dns }} - {{ worker.instance_name }}), mount '{{ mount }}' stats

    +

    + Check the space availability history for the device.
    +

    +
    +
    +

    +
    +
    +
    +
    +
    + + + + + + + + + + + + + + + +
    + Mount '{{ mount }}' history + +   +
    +
    +
    +   +
    +
    + + + + + +{% endblock %} \ No newline at end of file diff --git a/backoffice/templates/operations/db.js b/backoffice/templates/operations/db.js new file mode 100644 index 0000000..bf234d1 --- /dev/null +++ b/backoffice/templates/operations/db.js @@ -0,0 +1,110 @@ + function DB(name, rfields) { + rfields = rfields || []; + var db = { + name: name, + rel_fields: rfields, + rels: {}, + count: 0, + all: {}, + add: function(o) { + if (o.id in this.all) return; + this.count++; + this.all[o.id] = o; + $.each(this.rels, function(r,db) { + db.get(o[r]).add(o); + }); + }, + remove: function(k) { + var db = this; + if (k in this.all) { + var o = this.all[k]; + $.each(db.rel_fields, function (i,r) { + var rdb = db.rels[r]; + rdb.get(o[r]).remove(k); + }); + delete this.all[k]; + this.count--; + } + }, + load: function(ls) { + var cl = eval(name); + var db = this; + var ndb = DB(name, rfields); + $.each(this.rel_fields, function (i,r) { + ndb.rels[r] = RelDB(name, r); + }); + $.each(ls, function(i, n) { + ndb.add(n); + }); + if (db.count) { + var old = db.all; + setTimeout(function() { + db.all = ndb.all; + db.count = ndb.count; + db.rels = ndb.rels; + }, 0); + $.each(ndb.all, function(k,v) { + if (k in old) { + var o = old[k]; + if (!equals(o,v)) { + setTimeout(function() { cl.changed(k, o, v); }, 0); + } + } else { + setTimeout(function() { cl.created(k, v); }, 0); + } + }); + $.each(old, function(k,v) { + if (!(k in ndb.all)) { + cl.deleted(k, v); + } + }); + } else { + db.all = ndb.all; + db.count = ndb.count; + db.rels = ndb.rels; + } + }, + get: function(k) { return this.all[k] }, + update: function (a, b) { + var db = this; + var cl = eval(db.name); + $.each(b, function(k,v) { + if (k in a) { + var o = a[k]; + if (!equals(o,v)) { + setTimeout(function() { cl.changed(k, o, v); }, 0); + db.remove(k); + db.add(v); + } + } else { + setTimeout(function() { cl.created(k, v); }, 0); + db.add(v); + } + }); + $.each(a, function(k,v) { + if (!(k in b)) { + cl.deleted(k, v); + db.remove(k); + } + }); + }, + }; + $.each(db.rel_fields, function (i,r) { + db.rels[r] = RelDB(name, r); + db['for_' + r] = function(k) { + return db.rels[r].get(k); + } + }); + return db; + } + function RelDB(parent, child) { + return { + get: function(k) { + //if ('id' in k) k = k.id; + if (!(k in this)) { + this[k] = DB(parent+'['+k+'].'+child); + } + return this[k]; + }, + }; + } \ No newline at end of file diff --git a/backoffice/templates/operations/index.html b/backoffice/templates/operations/index.html new file mode 100644 index 0000000..38ae5bd --- /dev/null +++ b/backoffice/templates/operations/index.html @@ -0,0 +1,641 @@ + + +{% load custom_tags %} +{% load humanize %} + + + + + + + + + + + + +
    + + +
    + + +
    +
    +
    +
    + +
    +
    +
    + {% include "operations/templates.html" %} +
    + + diff --git a/backoffice/templates/operations/listings.js b/backoffice/templates/operations/listings.js new file mode 100644 index 0000000..6fab76b --- /dev/null +++ b/backoffice/templates/operations/listings.js @@ -0,0 +1,168 @@ +function load_accounts(target, filter, append) { + filter = filter || function() { return true; }; + if (!append) target.html(''); + var x = $("
    ").appendTo(target).html(t('wait', {})); + var c = $("
    "); + var accs = []; + $.each(Account.objects.all, function(id, a) { + if (filter(a)) accs.push([dcount(a),a]); + }); + if (accs.length) { + var h = t('account', {code:'code', email:'Account Email', documents: 'docs', indexes: 'idxs'}).addClass('header').appendTo(c); + if (!append) h.css('margin-top', '-25px').css('position', 'fixed'); + accs.sort(function(a,b) { return b[0]-a[0]; }); + $.eachAsync(accs, function(i,o) { Account.link(o[1]).appendTo(c); }, { end: function() { x.html(c); } }); + } else { + x.html(''); + } + return accs.length; +} + +function load_indexes(target, filter, append) { + filter = filter || function() { return true; }; + if (!append) target.html(''); + var x = $("
    ").appendTo(target).html(t('wait', {})); + var c = $("
    "); + var idxs = []; + $.each(Index.objects.all, function(id, i) { if (filter(i)) idxs.push(i); }); + if (idxs.length) { + var h = t('index', {code:'code', name:'Index Name', documents: '# docs'}).addClass('header').appendTo(c); + if (!append) h.css('margin-top', '-25px').css('position', 'fixed'); + idxs.sort(function(a,b) { return b.docs-a.docs; }); + $.eachAsync(idxs, function(i,o) { Index.link(o).appendTo(c); }, { end: function() { x.html(c); } }); + } else { + x.html(''); + } + return idxs.length; +} + +function load_workers(target, filter, append) { + filter = filter || function() { return true; }; + if (!append) target.html(''); + var x = $("
    ").appendTo(target).html(t('wait', {})); + var c = $("
    "); + var wkrs = []; + $.each(Worker.objects.all, function(id, i) { if (filter(i)) wkrs.push(i); }); + if (wkrs.length) { + var h = t('worker', {id:'id', depcount:'Worker', used: 'used ', ram: 'total '}).addClass('header').appendTo(c); + if (!append) h.css('margin-top', '-25px').css('position', 'fixed'); + wkrs.sort(function(a,b) { return b.id-a.id; }); + $.eachAsync(wkrs, function(i,o) { Worker.link(o).appendTo(c); }, { end: function() { x.html(c); } }); + } else { + x.html(''); + } + return wkrs.length; +} + +function load_packages(target, filter, append) { + filter = filter || function() { return true; }; + if (!append) target.html(''); + var x = $("
    ").appendTo(target).html(t('wait', {})); + var c = $("
    "); + var objs = []; + $.each(Package.objects.all, function(id, o) { if (filter(o)) objs.push(o); }); + if (objs.length) { + var h = t('package', {id:'id', name:'Package Name', documents: 'docs', price: 'price'}).addClass('header').appendTo(c); + if (!append) h.css('margin-top', '-25px').css('position', 'fixed'); + objs.sort(function(a,b) { return b.price-a.price; }); + $.eachAsync(objs, function(i,o) { Package.link(o).appendTo(c); }, { end: function() { x.html(c); } }); + } else { + x.html(''); + } + return objs.length; +} + +function load_configs(target, filter, append) { + filter = filter || function() { return true; }; + if (!append) target.html(''); + var x = $("
    ").appendTo(target).html(t('wait', {})); + var c = $("
    "); + var objs = []; + $.each(Config.objects.all, function(id, o) { if (filter(o)) objs.push(o); }); + if (objs.length) { + var h = t('config', {id:'id', name:'Config Description'}).addClass('header').appendTo(c); + if (!append) h.css('margin-top', '-25px').css('position', 'fixed'); + objs.sort(function(a,b) { return b.id-a.id; }); + $.eachAsync(objs, function(i,o) { Config.link(o).appendTo(c); }, { end: function() { x.html(c); } }); + } else { + x.html(''); + } + return objs.length; +} + +function load_deploys(target, filter, append) { + filter = filter || function() { return true; }; + if (!append) target.html(''); + var x = $("
    ").appendTo(target).html(t('wait', {})); + var c = $("
    "); + var objs = []; + $.each(Deploy.objects.all, function(id, o) { if (filter(o)) objs.push(o); }); + if (objs.length) { + var h = t('deploy', {id:'id', workerid:'_id', base_port:'port', effective_bdb:'#', effective_xmx: '#'}).addClass('header').appendTo(c); + if (!append) h.css('margin-top', '-25px').css('position', 'fixed'); + objs.sort(function(a,b) { return (a.status < b.status) ? 1 : ((a.status > b.status) ? -1 : b.id-a.id); }); + $.eachAsync(objs, function(i,o) { Deploy.link(o).appendTo(c); }, { end: function() { x.html(c); } }); + } else { + x.html(''); + } + return objs.length; +} + + function idx_good(id) { + var n = 0; var controllable = false; + $.each(_dep.for_index(id), function (k,v) { + n++; controllable = v.status == 'CONTROLLABLE'; + }); + return (n == 1) && controllable; + } + function dep_listing(d) { + var dep = ap(t('deploy', d), { 'workerid': d.worker }).attr('pk',d.id); + dep.find('.shortstatus').css('color', d.status == 'CONTROLLABLE' ? '#3B3' : '#33B' ); + dep.find('.status').addClass(d['status']); + return dep; + } + function wdep_listing(d) { + var i = _ind.get(d.index); + var a = _acc.get(i.account); + var opts = { + ram: d.effective_bdb + d.effective_xmx, + name: i.name, + email: a.email, + docs: human(i.docs) + }; + var dep = ap(t('wdeploy', d), opts).attr('pk',d.id); + dep.find('.shortstatus').css('color', d.status == 'CONTROLLABLE' ? '#3B3' : '#33B' ); + return dep; + } + function idx_listing(i) { + var idx = ap(t('index', i), { documents: human(i.docs) }).attr('pk',i.id); + idx.find('.shortstatus').css('color', idx_good(i.id) ? 'red' : '#3B3'); + return idx; + } + function acc_listing(a) { + var acc = ap(t('account', a), { indexes: human(icount(a)), documents: human(dcount(a)) }).attr('pk',a.id); + var p = _pkg.get(a.package); + var usage = parseInt(100.0 * dcount(a) / p.docs); + acc.find('.usage').css('color', usage > 100 ? 'red' : '#3B3'); + return acc; + } + function cfg_listing(c) { + return t('config', { id: c.id, name: c.description }).attr('pk',c.id); + } + function pkg_listing(p) { + return ap(t('package', p), { documents: human(p.docs) }).attr('pk',p.id); + } + function wkr_listing(w) { + var wkr = t('worker', w).attr('pk',w.id); + var used = 0; + var depcount = _dep.for_worker(w.id).count; + $.each(_dep.for_worker(w.id).all, function(k,v) { + used += v.effective_xmx; + used += v.effective_bdb; + }); + var usage = parseInt(100 * used / w.ram); + ap(wkr, { usage: usage, used: used, depcount: depcount }); + wkr.find('.shortstatus').css('color', w.status == 'CONTROLLABLE' ? '#3B3' : '#B33' ); + wkr.find('.usagebar').css('width', usage+'%'); + return wkr; + } diff --git a/backoffice/templates/operations/renderers.js b/backoffice/templates/operations/renderers.js new file mode 100644 index 0000000..60179db --- /dev/null +++ b/backoffice/templates/operations/renderers.js @@ -0,0 +1,305 @@ +function push(node) { + var id = node.attr('id'); + if (id) { + $('#'+id).slideUp(400); + } + var hidden = node.find(".header"); + hidden.hide(); + slideDown(node.prependTo('.maincontent'), function() { + hidden.show(); + }); +} +function change_data(cl, id, action, header, olddata, newdata, extra) { + var extra = extra || function() {}; + var ch = t('change', {}); + ch.attr('pk', id).attr('type', cl); + ch.append("
    " + cl + " " + id + " [" + action + "]
    "); + ch.append("
    "); + $.each(header, function(k,v) { + ch.append("

    " + k + " " + v + "

    "); + }); + if (!($.isEmptyObject(newdata) && $.isEmptyObject(olddata))) { + ch.append("
    "); + $.each(newdata, function(k,v) { + ch.append("

    " + k + " " + v + "

    "); + var node = $('#' + cl + '_' + id + ' .' + k).add('.listing.' + cl.toLowerCase() + '[pk=' + id + '] .' + k); + var bg = node.css('backgroundColor'); + var col = node.css('color'); + node.animate({ backgroundColor: '#F00', color: '#FFF' }, 300, function() { + setTimeout(function() { + node.css('backgroundColor', '#FF0'); + node.css('color', col); + node.text(v); + node.animate({ backgroundColor: bg }, 1000, function() { node.removeAttr('style'); }); + }, 200); + }); + extra(node); + }); + $.each(olddata, function(k,v) { + ch.append("

    " + k + " " + v + "

    "); + }); + } + if (action == 'deleted') { + var node = $('#' + cl + '_' + id).add('.' + cl.toLowerCase() + '[pk=' + id + ']'); + deleted_data(node); + } + change(ch); +} +function created_data(node) { + slideDown(node.hide()); + var bg = node.css('backgroundColor'); + node.css('backgroundColor', '#F00').animate({ backgroundColor: bg }, 1500, function() { node.removeAttr('style'); }); +} +function deleted_data(node) { + node.slideUp(400, function() { node.remove(); }); +} +function build_item(pk, type) { + var item = t('item', {}); + item.attr('id', type+'_'+pk); + item.attr('pk', pk); + item.attr('type', type); + return item; +} + +function render_index(pk) { + var item = build_item(pk, 'Index'); + var i = _ind.get(pk); + var a = _acc.get(i.account); + var p = _pkg.get(a.package); + var cfg = _cfg.get(i.configuration); + var usage = parseInt(100.0 * i.docs / p.docs); + var opts = { + id: i.id, + status: i.status, + name: i.name, + code: i.code, + documents: i.docs, + usage: usage, + maxdocs: human(p.docs), + configdesc: cfg.description, + xmx: cfg.data.xmx, + bdb: cfg.data.bdb_cache || 0, + package: p.name, + maxdocs: human(p.docs), + docs: human(i.docs), + usage: usage, + created: new Date(i.creation_time).format('yyyy/mm/dd HH:MM:ss') + }; + var index = t('index_det', opts); + var rels = index.find('.rels'); + acc_listing(a).appendTo(rels); + cfg_listing(cfg).appendTo(rels); + $.each(_dep.for_index(pk).all, function(k,v) { + dep_listing(v).appendTo(rels); + }); + index.appendTo(item.find('.content')); + return item; +} +function render_deploy(pk) { + var item = build_item(pk, 'Deploy'); + var d = _dep.get(pk); + var i = _ind.get(d.index); + var w = _wkr.get(d.worker); + var opts = { + worker_dns: w.wan_dns, + index_code: i.code, + created: new Date(d.timestamp).format('yyyy/mm/dd HH:MM:ss') + }; + var deploy = ap(t('deploy_det', d), opts); + var rels = deploy.find('.rels'); + idx_listing(i).appendTo(rels); + wkr_listing(w).appendTo(rels); + deploy.appendTo(item.find('.content')); + return item; +} +function render_worker(pk) { + var item = build_item(pk, 'Worker'); + var w = _wkr.get(pk); + var used = 0; + var deps = [] + $.each(_dep.for_worker(pk).all, function(k,v) { + deps.push([_ind.get(v.index).docs,v]); + used += v.effective_xmx; + used += v.effective_bdb; + }); + var usage = parseInt(100 * used / w.ram); + var worker = ap(t('worker_det', w), { used_ram: used, usage: usage }); + var rels = worker.find('.rels'); + deps.sort(function(a,b) { return b[0]-a[0]; }); + t('wdeploy', {id:'id', email:'Account Email', name:'Index Name', docs:'#', ram: 'ram ', status: 'status'}).addClass('header').css('position', 'absolute').css('margin-top', '-20px').appendTo(rels); + $.each(deps, function(k,v) { + wdep_listing(v[1]).appendTo(rels); + }); + worker.appendTo(item.find('.content')); + return item; +} +function render_config(pk) { + var item = build_item(pk, 'Config'); + var c = _cfg.get(pk); + var cc = $('
    '); + cc.append('

    ' + c.description + '

    ') + var pkgs = _pkg.for_configuration(pk); + var right = $('
    '); + $.each(pkgs.all, function(k,v) { + pkg_listing(v).appendTo(right); + }); + if (pkgs.count == 0) { + right.append('
    THIS CONFIG IS CUSTOM (No package)
    '); + $.each(_acc.for_configuration(pk).all, function(k,v) { + acc_listing(v).appendTo(right); + }); + $.each(_ind.for_configuration(pk).all, function(k,v) { + idx_listing(v).appendTo(right); + }); + } + right.appendTo(cc) + cc.append('

    Created ' + new Date(c.creation_date).format('yyyy/mm/dd')); + cc.append('

    Accounts ' + _acc.for_configuration(pk).count + ' accounts'); + cc.append('

    Indexes ' + _ind.for_configuration(pk).count + ' indexes'); + cc.append('

    '); + var pairs = []; + $.each(c.data, function(k,v) { + pairs.push([k,v]); + }) + pairs.sort(); + $.each(pairs, function(i,pair) { + cc.append('

    ' + pair[0] + ' ' + pair[1]); + }) + cc.append('

    '); + cc.append('
    Choose Config
    '); + cc.append('
    '); + cc.appendTo(item.find('.content')); + return item; +} +function render_package(pk) { + var item = build_item(pk, 'Package'); + var p = _pkg.get(pk); + var c = _cfg.get(p.configuration); + var package = ap(t('package_det', p), { accounts: _acc.for_package(pk).count }); + var rels = package.find('.rels'); + cfg_listing(c).appendTo(rels); + /*var relslong = package.find('.relslong'); + $.each(_acc.for_package(pk).all, function(k,v) { + acc_listing(v).appendTo(relslong); + });*/ + package.appendTo(item.find('.content')); + return item; +} +function render_deploy_stats(pk) { + var item = build_item(pk, 'DeployStats'); + var c = item.find('.content'); + t('load',{}).appendTo(c); + var dep = Deploy.objects.get(pk); + var ind = Index.objects.get(dep.index); + var acc = Account.objects.get(ind.account); + $.ajax({ + url: 'operations?level=stats&id=' + pk, + success: function (d) { + var cc = $('
    '); + var pairs = []; + $.each(d, function(k,v) { + pairs.push([k,v]); + }) + pairs.sort(); + $.each(pairs, function(i,pair) { + var val = pair[1]; + if (!isNaN(val)) { + if (Math.abs(val - 1300000000) < 315360000) { + // within 10 years of a recent date + val = new Date(val * 1000).format('yyyy/mm/dd HH:MM:ss'); + } else if (val >= 1000) { + val = val + ' (' + human(val) + ')'; + } + } + cc.append('

    ' + pair[0] + ' ' + val); + }) + c.html(cc); + $('

    ').append(dep_listing(dep)).prependTo(c); + c.prepend("

    stats for deploy " + pk + "

    "); + }, + dataType: 'json' + }); + return item; +} +function render_deploy_config(pk) { + var item = build_item(pk, 'DeployConfigFile'); + var c = item.find('.content'); + t('load',{}).appendTo(c); + var cc = $('
    '); + $.ajax({ + url: 'operations?level=log&file=indexengine_config&id=' + pk, + success: function (d) { + try { + var cfg = $.parseJSON(d); + $.each(cfg, function(k,v) { + $('

    ' + k + ' ' + v + '

    ').appendTo(cc); + }); + } catch(e) { + cc.append(d); + } + c.html(cc); + }, + dataType: 'json' + }); + return item; +} +function render_deploy_log(pk) { + var item = build_item(pk, 'DeployLog'); + var c = item.find('.content'); + t('load',{}).appendTo(c); + $.ajax({ + url: 'operations?level=log&file=logs/indextank.log&id=' + pk, + success: function (d) { + d = d.split('\n'); + var cc = $('
    '); + $.each(d, function(i,line) { + $('
    ').text(line).appendTo(cc); + }); + c.html(cc); + cc.animate({ scrollTop: cc.attr("scrollHeight") }, 1000); + }, + dataType: 'json' + }); + return item; +} +function render_deploy_gclog(pk) { + var item = build_item(pk, 'DeployGCLog'); + var c = item.find('.content'); + t('load',{}).appendTo(c); + $.ajax({ + url: 'operations?level=log&file=logs/gc.log&id=' + pk, + success: function (d) { + d = d.split('\n'); + var cc = $('
    '); + $.each(d, function(i,line) { + $('
    ').text(line).appendTo(cc); + }); + c.html(cc); + cc.animate({ scrollTop: cc.attr("scrollHeight") }, 1000); + }, + dataType: 'json' + }); + return item; +} +function render_account(pk) { + var item = build_item(pk, 'Account'); + var a = _acc.get(pk); + var p = _pkg.get(a.package); + var c = _cfg.get(a.configuration); + var pc = _cfg.get(p.configuration); + var usage = parseInt(100.0 * dcount(a) / p.docs); + var account = ap(t('account_det', a), { package: p.name, maxdocs: human(p.docs), maxidxs: human(p.indexes), usage:usage, created: new Date(a.creation_time).format('yyyy/mm/dd HH:MM:ss') }); + var rels = account.find('.rels'); + pkg_listing(p).appendTo(rels); + cfg_listing(c).appendTo(rels); + var idxs = []; + $.each(_ind.for_account(pk).all, function(k,v) { + idxs.push(v); + }); + idxs.sort(function(a,b) { return b.docs-a.docs; }); + $.each(idxs, function(i,o) { + idx_listing(o).appendTo(rels); + }); + account.appendTo(item.find('.content')); + return item; +} diff --git a/backoffice/templates/operations/styles.css b/backoffice/templates/operations/styles.css new file mode 100644 index 0000000..4fd9bb3 --- /dev/null +++ b/backoffice/templates/operations/styles.css @@ -0,0 +1,119 @@ +{% load custom_tags %} + +html, body, .all { height: 100%; } +body { margin: 0; padding: 0; font-family: arial; font-size: 11px; color: #333; background: #444; position:relative; z-index: -3; } + +.fit { z-index: -2; display: block; position:absolute; height:auto; bottom:0; top:0; left:0; right:0; } +.fitleft { z-index: -1; display: block; position:absolute; height:auto; bottom:0; top:0; left:0; } +.fitright { z-index: -1; display: block; position:absolute; height:auto; bottom:0; top:0; right:0; } + +.tbtn { float: left; border: solid 1px gray; background: white; -webkit-border-radius: 6px; padding: 4px 8px; background: #444; color: white; cursor: pointer; } +.tbtn:hover { background: white; color: gray; } +.tab { float: left; border: solid 1px gray; background: white; -webkit-border-radius: 6px; padding: 4px 8px; background: #444; color: white; cursor: pointer; } +.tab:hover { background: white; color: gray; } +.tab.active { background: yellow; color: black } + +.listing { width: 230px; border: solid 1px green; -webkit-border-radius: 4px; font-family: arial; font-size: 11px; color: #333; padding: 2px 5px; margin-bottom: 3px; background: white; } +.listing .code, .listing .id { font-weight: bold; color: green; float: left; } +.ldata { margin-left: 4px; float: left; text-align: left; } +.rdata { margin-right: 4px; float: right; text-align: right; } +.hid { overflow-x: hidden; } + +.account .email { white-space: nowrap; width: 135px ; } +.account .info { white-space: nowrap; width: 50px; margin-right: -3px; } +.account.header .email { white-space: nowrap; width: 85px; } +.account.header .info { white-space: nowrap; width: 85px; } + +.package .name { white-space: nowrap; width: 115px; } +.package .info { white-space: nowrap; width: 70px; margin-right: -3px; } + +.config .name { white-space: nowrap; width: 185px; } + +.listing { cursor: pointer; } +.listing:hover { background: green; color: white; } +.listing:hover .code, .listing:hover .id { color: yellow; } +.listing.header { background: #444; color: white; cursor: default; border-color: white; } +.listing.header .code, .listing.header .id { color: yellow; } + +.listing .under { + height: 10px; + background: #555; + margin: 2px -5px -2px -5px; + padding: 1px 4px; + color: white; + text-align: right; + font-size: 10px; + font-weight: bold; + -webkit-border-bottom-left-radius: 2px; -webkit-border-bottom-right-radius: 2px; +} +.item .listing .under { + margin-left: -78px; +} +.listing.header .under { display: none; } + +.item { -webkit-border-radius: 6px; margin-bottom: 15px; margin-right: 15px; background: white; box-shadow: 3px 3px 10px #333; } +.item .iheader { -webkit-border-top-left-radius: 5px; -webkit-border-top-right-radius: 5px; background: darkgreen; color: white; padding: 3px 8px; font-size: 13px; height: 4px; } +.item .iheader .type, .item .iheader .id { float: left; margin-right: 4px; } +.item .iheader .close { position: absolute; right: 28px; border: solid 1px white; background: #B33; -webkit-border-radius: 6px; padding: 2px 4px; color: white; cursor: pointer; text-transform: uppercase; } +.item .iheader .reload { display: none; position: absolute; right: 50px; border: solid 1px white; background: #3b3; -webkit-border-radius: 6px; padding: 2px 4px; color: white; cursor: pointer; text-transform: uppercase; } +.item .iheader .reload.active { background: url({% static 'images/refreshing.gif' %}) no-repeat 50% 50% #000; width: 12px; text-indent: -9999px; } +.item .iheader .end { clear: both; } +.item .content { padding: 10px; } +.item .iheader .close:hover, .item .iheader .reload:hover { background-color: black; } + +.item[type=Index] .iheader .reload { display: block; } + +.nosb .listing { padding-left: 78px; } +.nosb .listing:before { float: left; background: #3B3; padding: 2px 2px; color: white; -webkit-border-top-left-radius: 2px; -webkit-border-bottom-left-radius: 2px; margin: -2px 5px -2px -78px; font-weight: bold; text-align: center; width: 70px; } +.nosb .listing:hover:before { background: #444; } +.nosb .account:before { content: 'Account'; background: #33B; } +.nosb .index:before { content: 'Index'; } +.nosb .deploy:before { content: 'Deploy'; } +.nosb .wdeploy:before { content: 'Deploy'; } +.nosb .package:before { content: 'Package'; background: #888; } +.nosb .config:before { content: 'Config'; background: #993; } +.nosb .worker:before { content: 'Worker'; background: #399; } +.nosb .header.listing:before { display: none; } +.nosb .header.listing { border-color: silver; } + +.item h1 { font-size: 15px; margin: 0 0 10px 0; color: #555; } +.item h2 { font-size: 14px; margin: 7px 0 4px; color: #555; } +.item h3 { font-size: 13px; margin: 7px 0 4px; color: #555; } +.item p { font-size: 11px; margin: 0 0 4px; color: #777; display: block; } +.item p em { float: left; width: 55px; font-style: italic; color: #EC7F15; border-right: solid 1px #EC7F15; margin-right: 5px; } +.item hr { border: none; border-bottom: solid 1px #EC7F15; } +.account_det h1:before {content: 'account'; font-style: italic; color: #00B800; margin-right: 15px } +.index_det h1:before {content: 'index'; font-style: italic; color: #00B800; margin-right: 15px } +.deploy_det h1:before {content: 'deploy'; font-style: italic; color: #00B800; margin-right: 15px } +.worker_det h1:before {content: 'worker'; font-style: italic; color: #00B800; margin-right: 15px } +.package_det h1:before {content: 'package'; font-style: italic; color: #00B800; margin-right: 15px } +.item[type=Config] h1:before {content: 'config'; font-style: italic; color: #00B800; margin-right: 15px } +.item .statsdict p em { width: 180px; } +.item .log { max-height: 250px; overflow: auto; white-space: nowrap; font-family: "Andale Mono", "courier new"; font-size: 11px; } + +.btn { float: left; border: solid 1px #444; background: #888; -webkit-border-radius: 6px; padding: 4px 8px; color: white; cursor: pointer; font-weight: bold; text-transform: uppercase; margin-right: 5px; margin-top: 5px; } +.btn:hover { background: #3b3; } +.tab:hover { background: white; color: gray; } + +.change p { font-size: 11px; margin: 0 0 4px; color: #777; display: block; } +.change p.new { color: #3b3; font-weight: bold; } +.change p.old { color: #b33; font-weight: bold; } +.change p em { float: left; width: 55px; font-style: italic; color: #EC7F15; border-right: solid 1px #EC7F15; margin-right: 5px; font-weight: normal; } +.change hr { border: none; border-bottom: solid 1px #EC7F15; } + +.change:hover { color: #777; background: yellow; } + +.deploy .status:after { float: right; background: black; -webkit-border-bottom-right-radius: 2px; margin-right: -15px; width: 10px; height: 11px; content: ''; } +.deploy .status { padding-right: 15px; } +.deploy .status.CONTROLLABLE:after { background: #0f0; } +.deploy .status.CREATED:after { background: #f00; } +.deploy .status.MOVE_REQUESTED:after { background: #0FF; } +.deploy .status.MOVING:after { background: #00f; } +.deploy .status.INITIALIZING:after { background: #f80; } +.deploy .status.RECOVERING:after { background: #ff0; } + +.worker .under { background: silver; } + + + +#templates { display: none }; diff --git a/backoffice/templates/operations/templates.html b/backoffice/templates/operations/templates.html new file mode 100644 index 0000000..f75993f --- /dev/null +++ b/backoffice/templates/operations/templates.html @@ -0,0 +1,182 @@ +{% load custom_tags %} +{% load humanize %} + +
    +
    +
    x
    +
    r
    +
    +
    +
    +
    + +
    +

    ()

    +
    +
    +
    +

    id

    +

    status

    +

    created

    +

    docs (% of )

    +

    config => xmx: M - bdb:

    +
    +
    +
    Redeploy
    +
    Set Config
    +
    +
    +
    +

    ()

    +
    +
    +
    +

    created

    +

    ssh indextank@

    +

    cd /data/indexes/-

    +

    +

    Get Stats
    +
    Show Log
    +
    Show GC Log
    +
    Show IE Config
    +

    +
    +
    +
    +
    +

    +
    +
    +

    aws name

    +

    lan dns

    +

    status

    +

    ram % M of M

    +
    +
    +
    +
    +
    Decommission
    +
    Delete
    +
    +
    +
    +

    ()

    +
    +
    +
    +

    price $ / month

    +

    max docs

    +

    max idxs

    +

    accounts accounts

    +
    +
    +
    +
    +
    Choose Package
    + {#
    Set Config
    #} +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + / $ +
    +
    +
    + +
    +
    +
    +
    + ● +
    +
    +
    +
    +
    +
    + W: +
    +
    + bdb:M ● +
    +
    + xmx:M +
    +
    +
    +
    +
    +
    +
    + +
    +
    + +
    +
    + ● +
    +
    + M +
    +
    + docs +
    +
    +
    +
    +
    +
    deploys
    +
    + M + of + M ● +
    +
    +
    +
    + % +
    +
    +
    +
    +
    + +
    +
    + +
    +
    + +
    diff --git a/backoffice/templates/operations/templating.js b/backoffice/templates/operations/templating.js new file mode 100644 index 0000000..82e7d29 --- /dev/null +++ b/backoffice/templates/operations/templating.js @@ -0,0 +1,13 @@ + + function ap(n, opts) { + $.each(opts, function (k,v) { + var f = n.find('.'+k); + if (f != 'undefined') f.text(v); + }); + return n; + } + + function t(temp, opts) { + var n = $('#templates .' + temp).clone(); + return ap(n, opts); + } diff --git a/backoffice/templates/operations/utils.js b/backoffice/templates/operations/utils.js new file mode 100644 index 0000000..c3bdb23 --- /dev/null +++ b/backoffice/templates/operations/utils.js @@ -0,0 +1,267 @@ +slideRight = function(node) { + node.show(); + var css = node.css('width'); + var w = node.width(); + node.hide().css('width', 0); + node.show().animate({ width: w }, { duration: 400, complete: function() { node.css('width', css); } }); +} +slideLeft = function(node) { + node.show().animate({ width: 0 }, { duration: 400, complete: function() { node.remove(); } }); +} +slideDown = function(node, complete) { + complete = complete || function() {}; + node.show(); + var h = node.height(); + node.hide().css('height', 0); + node.show().animate({ height: h }, { duration: 400, complete: function() { node.css('height', ''); complete() } }); +} + + + equals = function(x,y) { + if (x == y) { return true; } + if (typeof(x) != typeof(y)) { return false; } + + for(p in x) { + // x contained in y + if(typeof(y[p])=='undefined') {return false;} + } + for(p in y) { + if (x[p] == y[p]) continue; + switch(typeof(x[p])) { + case 'undefined': + // y not contained in x + return false; + case 'object': + if (!equals(x[p],y[p])) return false; + break; + case 'function': + if (x[p].toString() != y[p].toString()) return false; + default: + return false; + } + } + return true; + }; + +var dateFormat = function () { + + var token = /d{1,4}|m{1,4}|yy(?:yy)?|([HhMsTt])\1?|[LloSZ]|"[^"]*"|'[^']*'/g, + + timezone = /\b(?:[PMCEA][SDP]T|(?:Pacific|Mountain|Central|Eastern|Atlantic) (?:Standard|Daylight|Prevailing) Time|(?:GMT|UTC)(?:[-+]\d{4})?)\b/g, + + timezoneClip = /[^-+\dA-Z]/g, + + pad = function (val, len) { + + val = String(val); + + len = len || 2; + + while (val.length < len) val = "0" + val; + + return val; + + }; + + + + // Regexes and supporting functions are cached through closure + + return function (date, mask, utc) { + + var dF = dateFormat; + + + + // You can't provide utc if you skip other args (use the "UTC:" mask prefix) + + if (arguments.length == 1 && (typeof date == "string" || date instanceof String) && !/\d/.test(date)) { + + mask = date; + + date = undefined; + + } + + + + // Passing date through Date applies Date.parse, if necessary + + date = date ? new Date(date) : new Date(); + + if (isNaN(date)) throw new SyntaxError("invalid date"); + + + + mask = String(dF.masks[mask] || mask || dF.masks["default"]); + + + + // Allow setting the utc argument via the mask + + if (mask.slice(0, 4) == "UTC:") { + + mask = mask.slice(4); + + utc = true; + + } + + + + var _ = utc ? "getUTC" : "get", + + d = date[_ + "Date"](), + + D = date[_ + "Day"](), + + m = date[_ + "Month"](), + + y = date[_ + "FullYear"](), + + H = date[_ + "Hours"](), + + M = date[_ + "Minutes"](), + + s = date[_ + "Seconds"](), + + L = date[_ + "Milliseconds"](), + + o = utc ? 0 : date.getTimezoneOffset(), + + flags = { + + d: d, + + dd: pad(d), + + ddd: dF.i18n.dayNames[D], + + dddd: dF.i18n.dayNames[D + 7], + + m: m + 1, + + mm: pad(m + 1), + + mmm: dF.i18n.monthNames[m], + + mmmm: dF.i18n.monthNames[m + 12], + + yy: String(y).slice(2), + + yyyy: y, + + h: H % 12 || 12, + + hh: pad(H % 12 || 12), + + H: H, + + HH: pad(H), + + M: M, + + MM: pad(M), + + s: s, + + ss: pad(s), + + l: pad(L, 3), + + L: pad(L > 99 ? Math.round(L / 10) : L), + + t: H < 12 ? "a" : "p", + + tt: H < 12 ? "am" : "pm", + + T: H < 12 ? "A" : "P", + + TT: H < 12 ? "AM" : "PM", + + Z: utc ? "UTC" : (String(date).match(timezone) || [""]).pop().replace(timezoneClip, ""), + + o: (o > 0 ? "-" : "+") + pad(Math.floor(Math.abs(o) / 60) * 100 + Math.abs(o) % 60, 4), + + S: ["th", "st", "nd", "rd"][d % 10 > 3 ? 0 : (d % 100 - d % 10 != 10) * d % 10] + + }; + + + + return mask.replace(token, function ($0) { + + return $0 in flags ? flags[$0] : $0.slice(1, $0.length - 1); + + }); + + }; + +}(); + + + +// Some common format strings + +dateFormat.masks = { + + "default": "ddd mmm dd yyyy HH:MM:ss", + + shortDate: "m/d/yy", + + mediumDate: "mmm d, yyyy", + + longDate: "mmmm d, yyyy", + + fullDate: "dddd, mmmm d, yyyy", + + shortTime: "h:MM TT", + + mediumTime: "h:MM:ss TT", + + longTime: "h:MM:ss TT Z", + + isoDate: "yyyy-mm-dd", + + isoTime: "HH:MM:ss", + + isoDateTime: "yyyy-mm-dd'T'HH:MM:ss", + + isoUtcDateTime: "UTC:yyyy-mm-dd'T'HH:MM:ss'Z'" + +}; + + + +// Internationalization strings + +dateFormat.i18n = { + + dayNames: [ + + "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", + + "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" + + ], + + monthNames: [ + + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + + "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December" + + ] + +}; + + + +// For convenience... + +Date.prototype.format = function (mask, utc) { + + return dateFormat(this, mask, utc); + +}; + diff --git a/backoffice/templates/resource_map.html b/backoffice/templates/resource_map.html new file mode 100644 index 0000000..96569ae --- /dev/null +++ b/backoffice/templates/resource_map.html @@ -0,0 +1,301 @@ +{% load custom_tags %} +{% load humanize %} + + + + RESOURCE MAP + + + + + + + {% for w in workers %} + +
    + + {% endfor %} + + +
    + + +
    + +
    + + +
    + + diff --git a/backoffice/templates/worker_resource_map.html b/backoffice/templates/worker_resource_map.html new file mode 100644 index 0000000..2ecd508 --- /dev/null +++ b/backoffice/templates/worker_resource_map.html @@ -0,0 +1,54 @@ +{% load custom_tags %} +{% load humanize %} + +{% with worker as w %} + +
    +
    +
    {{ w.used }}M / {{ w.ram }}M
    + REFRESH + +
    +

    Worker {{ w.id }} ({{ w.instance_name }}) - {{ w.wan_dns }}

    + {% for d in w.sorted_deploys %} + {% with d.index.configuration.get_data.ram as max_ram %} + +
    {{ d.id }} {{ d.index.name }}
    +
    +
    +
    +
    {{ d.total_ram }}M / {{ max_ram }}M
    +
    + +
    +
    +
    {{ d.index.current_docs_number|intcomma }} / {{ d.index.account.package.index_max_size|intcomma }}
    +
    + +
    +
    +

    Index {{ d.index.name }} - code: {{ d.index.code }} [clear]

    +

    {{ d.index.account.package.code }}: {{ d.index.account.package }}

    +

    {{ d.index.account.user.email }}

    +

    {{ d.index.account.get_private_apiurl }}

    +

    Deploys

    +
      + {% for dd in d.index.deploys.all %} +
    • {{ dd.id }} : Worker {{ dd.worker.id }} @ {{ dd.base_port }} - {{ dd.status }} - ram: {{ dd.total_ram }}
      • + {% endfor %} +
    +

    Configuration {{ d.index.configuration.id }} - {{ d.index.configuration.description }}

    + + {% for k,v in d.index.configuration.get_data.items %} + + {% endfor %} +
    {{k}}:{{v}}
    +

    + REDEPLOY +

    +
    + + {% endwith %} + {% endfor %} + +{% endwith %} diff --git a/backoffice/templatetags/__init__.py b/backoffice/templatetags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backoffice/templatetags/custom_tags.py b/backoffice/templatetags/custom_tags.py new file mode 100644 index 0000000..bb83572 --- /dev/null +++ b/backoffice/templatetags/custom_tags.py @@ -0,0 +1,56 @@ +from django import template +from django.template import Node + +from django.conf import settings + +register = template.Library() + +@register.filter(name='fdivide') +def fdivide(value,arg): + return float(value) / float(arg) +@register.filter(name='fmultiply') +def fmultiply(value,arg): + return float(value) * float(arg) + +def var_tag_compiler(params, defaults, name, node_class, parser, token): + "Returns a template.Node subclass." + bits = token.split_contents()[1:] + return node_class(map(parser.compile_filter, bits)) + +def simple_var_tag(func): + params, xx, xxx, defaults = template.getargspec(func) + + class SimpleNode(Node): + def __init__(self, vars_to_resolve): + self.vars_to_resolve = vars_to_resolve + + def render(self, context): + resolved_vars = [var.resolve(context, True) for var in self.vars_to_resolve] + return func(*resolved_vars) + + compile_func = template.curry(var_tag_compiler, params, defaults, getattr(func, "_decorated_function", func).__name__, SimpleNode) + compile_func.__doc__ = func.__doc__ + register.tag(getattr(func, "_decorated_function", func).__name__, compile_func) + return func + +@simple_var_tag +def static(*parts): + path = ''.join(parts) + urls = settings.STATIC_URLS + size = len(urls) + h = hash(path) % size + if h < 0: + h += size + return urls[h] + '/' + path + +@register.filter(name='get') +def doget(value,arg): + return dict(value).get(arg) or '' + +@register.filter(name='a1000times') +def a1000times(value): + return value * 1000 + +@register.filter(name='range') +def rangefilter(value): + return xrange(int(value)) diff --git a/backoffice/templatetags/macros.py b/backoffice/templatetags/macros.py new file mode 100644 index 0000000..2e827c3 --- /dev/null +++ b/backoffice/templatetags/macros.py @@ -0,0 +1,169 @@ +from django import template +from django.template import TemplateSyntaxError + +register = template.Library() + +""" + The MacroRoot node (= %enablemacros% tag) functions quite similar to + the ExtendsNode from django.template.loader_tags. It will capture + everything that follows, and thus should be one of the first tags in + the template. Because %extends% also needs to be the first, if you are + using template inheritance, use %extends_with_macros% instead. + + This whole procedure is necessary because otherwise we would have no + possiblity to access the blocktag referenced by a %repeat% (we could + do it for %macro%, but not for %block%, at least not without patching + the django source). + + So what we do is add a custom attribute to the parser object and store + a reference to the MacroRoot node there, which %repeat% object will + later be able to access when they need to find a block. + + Apart from that, the node doesn't do much, except rendering it's childs. +""" +class MacroRoot(template.Node): + def __init__(self, nodelist=[]): + self.nodelist = nodelist + + def render(self, context): + return self.nodelist.render(context) + + def find(self, block_name, parent_nodelist=None): + # parent_nodelist is internally for recusion, start with root nodelist + if parent_nodelist is None: parent_nodelist = self.nodelist + + from django.template.loader_tags import BlockNode + for node in parent_nodelist: + if isinstance(node, (MacroNode, BlockNode)): + if node.name == block_name: + return node + if hasattr(node, 'nodelist'): + result = self.find(block_name, node.nodelist) + if result: + return result + return None # nothing found + +def do_enablemacros(parser, token): + # check that there are no arguments + bits = token.split_contents() + if len(bits) != 1: + raise TemplateSyntaxError, "'%s' takes no arguments" % bits[0] + # create the Node object now, so we can assign it to the parser + # before we continue with our call to parse(). this enables repeat + # tags that follow later to already enforce at the parsing stage + # that macros are correctly enabled. + parser._macro_root = MacroRoot() + # capture the rest of the template + nodelist = parser.parse() + if nodelist.get_nodes_by_type(MacroRoot): + raise TemplateSyntaxError, "'%s' cannot appear more than once in the same template" % bits[0] + # update the nodelist on the previously created MacroRoot node and + # return it. + parser._macro_root.nodelist = nodelist + return parser._macro_root + +def do_extends_with_macros(parser, token): + from django.template.loader_tags import do_extends + # parse it as an ExtendsNode, but also create a fake MacroRoot node + # and add it to the parser, like we do in do_enablemacros(). + parser._macro_root = MacroRoot() + extendsnode = do_extends(parser, token) + parser._macro_root.nodelist = extendsnode.nodelist + return extendsnode + +""" + %macro% is pretty much exactly like a %block%. Both can be repeated, but + the macro does not output it's content by itself, but *only* if it is + called via a %repeat% tag. +""" + +from django.template.loader_tags import BlockNode, do_block + +class MacroNode(BlockNode): + def render(self, context): + return '' + + # the render that actually works + def repeat(self, context): + return super(MacroNode, self).render(context) + +def do_macro(parser, token): + # let the block parse itself + result = do_block(parser, token) + # "upgrade" the BlockNode to a MacroNode and return it. Yes, I was not + # completely comfortable with it either at first, but Google says it's ok. + result.__class__ = MacroNode + return result + +""" + This (the %repeast%) is the heart of the macro system. It will try to + find the specified %macro% or %block% tag and render it with the most + up-to-date context, including any number of additional parameters passed + to the repeat-tag itself. +""" +class RepeatNode(template.Node): + def __init__(self, block_name, macro_root, extra_context): + self.block_name = block_name + self.macro_root = macro_root + self.extra_context = extra_context + + def render(self, context): + block = self.macro_root.find(self.block_name) + if not block: + # apparently we are not supposed to raise exceptions at rendering + # stage, but this is serious, and we cannot do it while parsing. + # once again, it comes down to being able to support repeating of + # standard blocks. If we would only support our own %macro% tags, + # we would not need the whole %enablemacros% stuff and could do + # things differently. + raise TemplateSyntaxError, "cannot repeat '%s': block or macro not found" % self.block_name + else: + # resolve extra context variables + resolved_context = {} + for key, value in self.extra_context.items(): + resolved_context[key] = value.resolve(context) + # render the block with the new context + context.update(resolved_context) + if isinstance(block, MacroNode): + result = block.repeat(context) + else: + result = block.render(context) + context.pop() + return result + +def do_repeat(parser, token): + # Stolen from django.templatetags.i18n.BlockTranslateParser + # Parses something like "with x as y, i as j", and + # returns it as a context dict. + class RepeatTagParser(template.TokenParser): + def top(self): + extra_context = {} + # first tag is the blockname + try: block_name = self.tag() + except TemplateSyntaxError: + raise TemplateSyntaxError("'%s' requires a block or macro name" % self.tagname) + # read param bindings + while self.more(): + tag = self.tag() + if tag == 'with' or tag == 'and': + value = self.value() + if self.tag() != 'as': + raise TemplateSyntaxError, "variable bindings in %s must be 'with value as variable'" % self.tagname + extra_context[self.tag()] = parser.compile_filter(value) + else: + raise TemplateSyntaxError, "unknown subtag %s for '%s' found" % (tag, self.tagname) + return self.tagname, block_name, extra_context + + # parse arguments + (tag_name, block_name, extra_context) = \ + RepeatTagParser(token.contents).top() + # return as a RepeatNode + if not hasattr(parser, '_macro_root'): + raise TemplateSyntaxError, "'%s' requires macros to be enabled first" % tag_name + return RepeatNode(block_name, parser._macro_root, extra_context) + +# register all our tags +register.tag('repeat', do_repeat) +register.tag('macro', do_macro) +register.tag('enablemacros', do_enablemacros) +register.tag('extends_with_macros', do_extends_with_macros) \ No newline at end of file diff --git a/backoffice/thrift/TSCons.py b/backoffice/thrift/TSCons.py new file mode 100644 index 0000000..2404625 --- /dev/null +++ b/backoffice/thrift/TSCons.py @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from os import path +from SCons.Builder import Builder + +def scons_env(env, add=''): + opath = path.dirname(path.abspath('$TARGET')) + lstr = 'thrift --gen cpp -o ' + opath + ' ' + add + ' $SOURCE' + cppbuild = Builder(action = lstr) + env.Append(BUILDERS = {'ThriftCpp' : cppbuild}) + +def gen_cpp(env, dir, file): + scons_env(env) + suffixes = ['_types.h', '_types.cpp'] + targets = map(lambda s: 'gen-cpp/' + file + s, suffixes) + return env.ThriftCpp(targets, dir+file+'.thrift') diff --git a/backoffice/thrift/TSerialization.py b/backoffice/thrift/TSerialization.py new file mode 100644 index 0000000..b19f98a --- /dev/null +++ b/backoffice/thrift/TSerialization.py @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from protocol import TBinaryProtocol +from transport import TTransport + +def serialize(thrift_object, protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()): + transport = TTransport.TMemoryBuffer() + protocol = protocol_factory.getProtocol(transport) + thrift_object.write(protocol) + return transport.getvalue() + +def deserialize(base, buf, protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()): + transport = TTransport.TMemoryBuffer(buf) + protocol = protocol_factory.getProtocol(transport) + base.read(protocol) + return base + diff --git a/backoffice/thrift/Thrift.py b/backoffice/thrift/Thrift.py new file mode 100644 index 0000000..91728a7 --- /dev/null +++ b/backoffice/thrift/Thrift.py @@ -0,0 +1,133 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys + +class TType: + STOP = 0 + VOID = 1 + BOOL = 2 + BYTE = 3 + I08 = 3 + DOUBLE = 4 + I16 = 6 + I32 = 8 + I64 = 10 + STRING = 11 + UTF7 = 11 + STRUCT = 12 + MAP = 13 + SET = 14 + LIST = 15 + UTF8 = 16 + UTF16 = 17 + +class TMessageType: + CALL = 1 + REPLY = 2 + EXCEPTION = 3 + ONEWAY = 4 + +class TProcessor: + + """Base class for procsessor, which works on two streams.""" + + def process(iprot, oprot): + pass + +class TException(Exception): + + """Base class for all thrift exceptions.""" + + # BaseException.message is deprecated in Python v[2.6,3.0) + if (2,6,0) <= sys.version_info < (3,0): + def _get_message(self): + return self._message + def _set_message(self, message): + self._message = message + message = property(_get_message, _set_message) + + def __init__(self, message=None): + Exception.__init__(self, message) + self.message = message + +class TApplicationException(TException): + + """Application level thrift exceptions.""" + + UNKNOWN = 0 + UNKNOWN_METHOD = 1 + INVALID_MESSAGE_TYPE = 2 + WRONG_METHOD_NAME = 3 + BAD_SEQUENCE_ID = 4 + MISSING_RESULT = 5 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + + def __str__(self): + if self.message: + return self.message + elif self.type == self.UNKNOWN_METHOD: + return 'Unknown method' + elif self.type == self.INVALID_MESSAGE_TYPE: + return 'Invalid message type' + elif self.type == self.WRONG_METHOD_NAME: + return 'Wrong method name' + elif self.type == self.BAD_SEQUENCE_ID: + return 'Bad sequence ID' + elif self.type == self.MISSING_RESULT: + return 'Missing result' + else: + return 'Default (unknown) TApplicationException' + + def read(self, iprot): + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.message = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.type = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + oprot.writeStructBegin('TApplicationException') + if self.message != None: + oprot.writeFieldBegin('message', TType.STRING, 1) + oprot.writeString(self.message) + oprot.writeFieldEnd() + if self.type != None: + oprot.writeFieldBegin('type', TType.I32, 2) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() diff --git a/backoffice/thrift/__init__.py b/backoffice/thrift/__init__.py new file mode 100644 index 0000000..48d659c --- /dev/null +++ b/backoffice/thrift/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['Thrift', 'TSCons'] diff --git a/backoffice/thrift/protocol/TBinaryProtocol.py b/backoffice/thrift/protocol/TBinaryProtocol.py new file mode 100644 index 0000000..50c6aa8 --- /dev/null +++ b/backoffice/thrift/protocol/TBinaryProtocol.py @@ -0,0 +1,259 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TProtocol import * +from struct import pack, unpack + +class TBinaryProtocol(TProtocolBase): + + """Binary implementation of the Thrift protocol driver.""" + + # NastyHaxx. Python 2.4+ on 32-bit machines forces hex constants to be + # positive, converting this into a long. If we hardcode the int value + # instead it'll stay in 32 bit-land. + + # VERSION_MASK = 0xffff0000 + VERSION_MASK = -65536 + + # VERSION_1 = 0x80010000 + VERSION_1 = -2147418112 + + TYPE_MASK = 0x000000ff + + def __init__(self, trans, strictRead=False, strictWrite=True): + TProtocolBase.__init__(self, trans) + self.strictRead = strictRead + self.strictWrite = strictWrite + + def writeMessageBegin(self, name, type, seqid): + if self.strictWrite: + self.writeI32(TBinaryProtocol.VERSION_1 | type) + self.writeString(name) + self.writeI32(seqid) + else: + self.writeString(name) + self.writeByte(type) + self.writeI32(seqid) + + def writeMessageEnd(self): + pass + + def writeStructBegin(self, name): + pass + + def writeStructEnd(self): + pass + + def writeFieldBegin(self, name, type, id): + self.writeByte(type) + self.writeI16(id) + + def writeFieldEnd(self): + pass + + def writeFieldStop(self): + self.writeByte(TType.STOP); + + def writeMapBegin(self, ktype, vtype, size): + self.writeByte(ktype) + self.writeByte(vtype) + self.writeI32(size) + + def writeMapEnd(self): + pass + + def writeListBegin(self, etype, size): + self.writeByte(etype) + self.writeI32(size) + + def writeListEnd(self): + pass + + def writeSetBegin(self, etype, size): + self.writeByte(etype) + self.writeI32(size) + + def writeSetEnd(self): + pass + + def writeBool(self, bool): + if bool: + self.writeByte(1) + else: + self.writeByte(0) + + def writeByte(self, byte): + buff = pack("!b", byte) + self.trans.write(buff) + + def writeI16(self, i16): + buff = pack("!h", i16) + self.trans.write(buff) + + def writeI32(self, i32): + buff = pack("!i", i32) + self.trans.write(buff) + + def writeI64(self, i64): + buff = pack("!q", i64) + self.trans.write(buff) + + def writeDouble(self, dub): + buff = pack("!d", dub) + self.trans.write(buff) + + def writeString(self, str): + self.writeI32(len(str)) + self.trans.write(str) + + def readMessageBegin(self): + sz = self.readI32() + if sz < 0: + version = sz & TBinaryProtocol.VERSION_MASK + if version != TBinaryProtocol.VERSION_1: + raise TProtocolException(type=TProtocolException.BAD_VERSION, message='Bad version in readMessageBegin: %d' % (sz)) + type = sz & TBinaryProtocol.TYPE_MASK + name = self.readString() + seqid = self.readI32() + else: + if self.strictRead: + raise TProtocolException(type=TProtocolException.BAD_VERSION, message='No protocol version header') + name = self.trans.readAll(sz) + type = self.readByte() + seqid = self.readI32() + return (name, type, seqid) + + def readMessageEnd(self): + pass + + def readStructBegin(self): + pass + + def readStructEnd(self): + pass + + def readFieldBegin(self): + type = self.readByte() + if type == TType.STOP: + return (None, type, 0) + id = self.readI16() + return (None, type, id) + + def readFieldEnd(self): + pass + + def readMapBegin(self): + ktype = self.readByte() + vtype = self.readByte() + size = self.readI32() + return (ktype, vtype, size) + + def readMapEnd(self): + pass + + def readListBegin(self): + etype = self.readByte() + size = self.readI32() + return (etype, size) + + def readListEnd(self): + pass + + def readSetBegin(self): + etype = self.readByte() + size = self.readI32() + return (etype, size) + + def readSetEnd(self): + pass + + def readBool(self): + byte = self.readByte() + if byte == 0: + return False + return True + + def readByte(self): + buff = self.trans.readAll(1) + val, = unpack('!b', buff) + return val + + def readI16(self): + buff = self.trans.readAll(2) + val, = unpack('!h', buff) + return val + + def readI32(self): + buff = self.trans.readAll(4) + val, = unpack('!i', buff) + return val + + def readI64(self): + buff = self.trans.readAll(8) + val, = unpack('!q', buff) + return val + + def readDouble(self): + buff = self.trans.readAll(8) + val, = unpack('!d', buff) + return val + + def readString(self): + len = self.readI32() + str = self.trans.readAll(len) + return str + + +class TBinaryProtocolFactory: + def __init__(self, strictRead=False, strictWrite=True): + self.strictRead = strictRead + self.strictWrite = strictWrite + + def getProtocol(self, trans): + prot = TBinaryProtocol(trans, self.strictRead, self.strictWrite) + return prot + + +class TBinaryProtocolAccelerated(TBinaryProtocol): + + """C-Accelerated version of TBinaryProtocol. + + This class does not override any of TBinaryProtocol's methods, + but the generated code recognizes it directly and will call into + our C module to do the encoding, bypassing this object entirely. + We inherit from TBinaryProtocol so that the normal TBinaryProtocol + encoding can happen if the fastbinary module doesn't work for some + reason. (TODO(dreiss): Make this happen sanely in more cases.) + + In order to take advantage of the C module, just use + TBinaryProtocolAccelerated instead of TBinaryProtocol. + + NOTE: This code was contributed by an external developer. + The internal Thrift team has reviewed and tested it, + but we cannot guarantee that it is production-ready. + Please feel free to report bugs and/or success stories + to the public mailing list. + """ + + pass + + +class TBinaryProtocolAcceleratedFactory: + def getProtocol(self, trans): + return TBinaryProtocolAccelerated(trans) diff --git a/backoffice/thrift/protocol/TCompactProtocol.py b/backoffice/thrift/protocol/TCompactProtocol.py new file mode 100644 index 0000000..fbc156a --- /dev/null +++ b/backoffice/thrift/protocol/TCompactProtocol.py @@ -0,0 +1,368 @@ +from TProtocol import * +from struct import pack, unpack + +__all__ = ['TCompactProtocol', 'TCompactProtocolFactory'] + +CLEAR = 0 +FIELD_WRITE = 1 +VALUE_WRITE = 2 +CONTAINER_WRITE = 3 +BOOL_WRITE = 4 +FIELD_READ = 5 +CONTAINER_READ = 6 +VALUE_READ = 7 +BOOL_READ = 8 + +def make_helper(v_from, container): + def helper(func): + def nested(self, *args, **kwargs): + assert self.state in (v_from, container), (self.state, v_from, container) + return func(self, *args, **kwargs) + return nested + return helper +writer = make_helper(VALUE_WRITE, CONTAINER_WRITE) +reader = make_helper(VALUE_READ, CONTAINER_READ) + +def makeZigZag(n, bits): + return (n << 1) ^ (n >> (bits - 1)) + +def fromZigZag(n): + return (n >> 1) ^ -(n & 1) + +def writeVarint(trans, n): + out = [] + while True: + if n & ~0x7f == 0: + out.append(n) + break + else: + out.append((n & 0xff) | 0x80) + n = n >> 7 + trans.write(''.join(map(chr, out))) + +def readVarint(trans): + result = 0 + shift = 0 + while True: + x = trans.readAll(1) + byte = ord(x) + result |= (byte & 0x7f) << shift + if byte >> 7 == 0: + return result + shift += 7 + +class CompactType: + TRUE = 1 + FALSE = 2 + BYTE = 0x03 + I16 = 0x04 + I32 = 0x05 + I64 = 0x06 + DOUBLE = 0x07 + BINARY = 0x08 + LIST = 0x09 + SET = 0x0A + MAP = 0x0B + STRUCT = 0x0C + +CTYPES = {TType.BOOL: CompactType.TRUE, # used for collection + TType.BYTE: CompactType.BYTE, + TType.I16: CompactType.I16, + TType.I32: CompactType.I32, + TType.I64: CompactType.I64, + TType.DOUBLE: CompactType.DOUBLE, + TType.STRING: CompactType.BINARY, + TType.STRUCT: CompactType.STRUCT, + TType.LIST: CompactType.LIST, + TType.SET: CompactType.SET, + TType.MAP: CompactType.MAP, + } + +TTYPES = {} +for k, v in CTYPES.items(): + TTYPES[v] = k +TTYPES[CompactType.FALSE] = TType.BOOL +del k +del v + +class TCompactProtocol(TProtocolBase): + "Compact implementation of the Thrift protocol driver." + + PROTOCOL_ID = 0x82 + VERSION = 1 + VERSION_MASK = 0x1f + TYPE_MASK = 0xe0 + TYPE_SHIFT_AMOUNT = 5 + + def __init__(self, trans): + TProtocolBase.__init__(self, trans) + self.state = CLEAR + self.__last_fid = 0 + self.__bool_fid = None + self.__bool_value = None + self.__structs = [] + self.__containers = [] + + def __writeVarint(self, n): + writeVarint(self.trans, n) + + def writeMessageBegin(self, name, type, seqid): + assert self.state == CLEAR + self.__writeUByte(self.PROTOCOL_ID) + self.__writeUByte(self.VERSION | (type << self.TYPE_SHIFT_AMOUNT)) + self.__writeVarint(seqid) + self.__writeString(name) + self.state = VALUE_WRITE + + def writeMessageEnd(self): + assert self.state == VALUE_WRITE + self.state = CLEAR + + def writeStructBegin(self, name): + assert self.state in (CLEAR, CONTAINER_WRITE, VALUE_WRITE), self.state + self.__structs.append((self.state, self.__last_fid)) + self.state = FIELD_WRITE + self.__last_fid = 0 + + def writeStructEnd(self): + assert self.state == FIELD_WRITE + self.state, self.__last_fid = self.__structs.pop() + + def writeFieldStop(self): + self.__writeByte(0) + + def __writeFieldHeader(self, type, fid): + delta = fid - self.__last_fid + if 0 < delta <= 15: + self.__writeUByte(delta << 4 | type) + else: + self.__writeByte(type) + self.__writeI16(fid) + self.__last_fid = fid + + def writeFieldBegin(self, name, type, fid): + assert self.state == FIELD_WRITE, self.state + if type == TType.BOOL: + self.state = BOOL_WRITE + self.__bool_fid = fid + else: + self.state = VALUE_WRITE + self.__writeFieldHeader(CTYPES[type], fid) + + def writeFieldEnd(self): + assert self.state in (VALUE_WRITE, BOOL_WRITE), self.state + self.state = FIELD_WRITE + + def __writeUByte(self, byte): + self.trans.write(pack('!B', byte)) + + def __writeByte(self, byte): + self.trans.write(pack('!b', byte)) + + def __writeI16(self, i16): + self.__writeVarint(makeZigZag(i16, 16)) + + def __writeSize(self, i32): + self.__writeVarint(i32) + + def writeCollectionBegin(self, etype, size): + assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state + if size <= 14: + self.__writeUByte(size << 4 | CTYPES[etype]) + else: + self.__writeUByte(0xf0 | CTYPES[etype]) + self.__writeSize(size) + self.__containers.append(self.state) + self.state = CONTAINER_WRITE + writeSetBegin = writeCollectionBegin + writeListBegin = writeCollectionBegin + + def writeMapBegin(self, ktype, vtype, size): + assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state + if size == 0: + self.__writeByte(0) + else: + self.__writeSize(size) + self.__writeUByte(CTYPES[ktype] << 4 | CTYPES[vtype]) + self.__containers.append(self.state) + self.state = CONTAINER_WRITE + + def writeCollectionEnd(self): + assert self.state == CONTAINER_WRITE, self.state + self.state = self.__containers.pop() + writeMapEnd = writeCollectionEnd + writeSetEnd = writeCollectionEnd + writeListEnd = writeCollectionEnd + + def writeBool(self, bool): + if self.state == BOOL_WRITE: + self.__writeFieldHeader(types[bool], self.__bool_fid) + elif self.state == CONTAINER_WRITE: + self.__writeByte(int(bool)) + else: + raise AssertetionError, "Invalid state in compact protocol" + + writeByte = writer(__writeByte) + writeI16 = writer(__writeI16) + + @writer + def writeI32(self, i32): + self.__writeVarint(makeZigZag(i32, 32)) + + @writer + def writeI64(self, i64): + self.__writeVarint(makeZigZag(i64, 64)) + + @writer + def writeDouble(self, dub): + self.trans.write(pack('!d', dub)) + + def __writeString(self, s): + self.__writeSize(len(s)) + self.trans.write(s) + writeString = writer(__writeString) + + def readFieldBegin(self): + assert self.state == FIELD_READ, self.state + type = self.__readUByte() + if type & 0x0f == TType.STOP: + return (None, 0, 0) + delta = type >> 4 + if delta == 0: + fid = self.__readI16() + else: + fid = self.__last_fid + delta + self.__last_fid = fid + type = type & 0x0f + if type == CompactType.TRUE: + self.state = BOOL_READ + self.__bool_value = True + elif type == CompactType.FALSE: + self.state = BOOL_READ + self.__bool_value = False + else: + self.state = VALUE_READ + return (None, self.__getTType(type), fid) + + def readFieldEnd(self): + assert self.state in (VALUE_READ, BOOL_READ), self.state + self.state = FIELD_READ + + def __readUByte(self): + result, = unpack('!B', self.trans.readAll(1)) + return result + + def __readByte(self): + result, = unpack('!b', self.trans.readAll(1)) + return result + + def __readVarint(self): + return readVarint(self.trans) + + def __readZigZag(self): + return fromZigZag(self.__readVarint()) + + def __readSize(self): + result = self.__readVarint() + if result < 0: + raise TException("Length < 0") + return result + + def readMessageBegin(self): + assert self.state == CLEAR + proto_id = self.__readUByte() + if proto_id != self.PROTOCOL_ID: + raise TProtocolException(TProtocolException.BAD_VERSION, + 'Bad protocol id in the message: %d' % proto_id) + ver_type = self.__readUByte() + type = (ver_type & self.TYPE_MASK) >> self.TYPE_SHIFT_AMOUNT + version = ver_type & self.VERSION_MASK + if version != self.VERSION: + raise TProtocolException(TProtocolException.BAD_VERSION, + 'Bad version: %d (expect %d)' % (version, self.VERSION)) + seqid = self.__readVarint() + name = self.__readString() + return (name, type, seqid) + + def readMessageEnd(self): + assert self.state == VALUE_READ + assert len(self.__structs) == 0 + self.state = CLEAR + + def readStructBegin(self): + assert self.state in (CLEAR, CONTAINER_READ, VALUE_READ), self.state + self.__structs.append((self.state, self.__last_fid)) + self.state = FIELD_READ + self.__last_fid = 0 + + def readStructEnd(self): + assert self.state == FIELD_READ + self.state, self.__last_fid = self.__structs.pop() + + def readCollectionBegin(self): + assert self.state in (VALUE_READ, CONTAINER_READ), self.state + size_type = self.__readUByte() + size = size_type >> 4 + type = self.__getTType(size_type) + if size == 15: + size = self.__readSize() + self.__containers.append(self.state) + self.state = CONTAINER_READ + return type, size + readSetBegin = readCollectionBegin + readListBegin = readCollectionBegin + + def readMapBegin(self): + assert self.state in (VALUE_READ, CONTAINER_READ), self.state + size = self.__readSize() + types = 0 + if size > 0: + types = self.__readUByte() + vtype = self.__getTType(types) + ktype = self.__getTType(types >> 4) + self.__containers.append(self.state) + self.state = CONTAINER_READ + return (ktype, vtype, size) + + def readCollectionEnd(self): + assert self.state == CONTAINER_READ, self.state + self.state = self.__containers.pop() + readSetEnd = readCollectionEnd + readListEnd = readCollectionEnd + readMapEnd = readCollectionEnd + + def readBool(self): + if self.state == BOOL_READ: + return self.__bool_value + elif self.state == CONTAINER_READ: + return bool(self.__readByte()) + else: + raise AssertionError, "Invalid state in compact protocol: %d" % self.state + + readByte = reader(__readByte) + __readI16 = __readZigZag + readI16 = reader(__readZigZag) + readI32 = reader(__readZigZag) + readI64 = reader(__readZigZag) + + @reader + def readDouble(self): + buff = self.trans.readAll(8) + val, = unpack('!d', buff) + return val + + def __readString(self): + len = self.__readSize() + return self.trans.readAll(len) + readString = reader(__readString) + + def __getTType(self, byte): + return TTYPES[byte & 0x0f] + + +class TCompactProtocolFactory: + def __init__(self): + pass + + def getProtocol(self, trans): + return TCompactProtocol(trans) diff --git a/backoffice/thrift/protocol/TProtocol.py b/backoffice/thrift/protocol/TProtocol.py new file mode 100644 index 0000000..be3cb14 --- /dev/null +++ b/backoffice/thrift/protocol/TProtocol.py @@ -0,0 +1,205 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from thrift.Thrift import * + +class TProtocolException(TException): + + """Custom Protocol Exception class""" + + UNKNOWN = 0 + INVALID_DATA = 1 + NEGATIVE_SIZE = 2 + SIZE_LIMIT = 3 + BAD_VERSION = 4 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + +class TProtocolBase: + + """Base class for Thrift protocol driver.""" + + def __init__(self, trans): + self.trans = trans + + def writeMessageBegin(self, name, type, seqid): + pass + + def writeMessageEnd(self): + pass + + def writeStructBegin(self, name): + pass + + def writeStructEnd(self): + pass + + def writeFieldBegin(self, name, type, id): + pass + + def writeFieldEnd(self): + pass + + def writeFieldStop(self): + pass + + def writeMapBegin(self, ktype, vtype, size): + pass + + def writeMapEnd(self): + pass + + def writeListBegin(self, etype, size): + pass + + def writeListEnd(self): + pass + + def writeSetBegin(self, etype, size): + pass + + def writeSetEnd(self): + pass + + def writeBool(self, bool): + pass + + def writeByte(self, byte): + pass + + def writeI16(self, i16): + pass + + def writeI32(self, i32): + pass + + def writeI64(self, i64): + pass + + def writeDouble(self, dub): + pass + + def writeString(self, str): + pass + + def readMessageBegin(self): + pass + + def readMessageEnd(self): + pass + + def readStructBegin(self): + pass + + def readStructEnd(self): + pass + + def readFieldBegin(self): + pass + + def readFieldEnd(self): + pass + + def readMapBegin(self): + pass + + def readMapEnd(self): + pass + + def readListBegin(self): + pass + + def readListEnd(self): + pass + + def readSetBegin(self): + pass + + def readSetEnd(self): + pass + + def readBool(self): + pass + + def readByte(self): + pass + + def readI16(self): + pass + + def readI32(self): + pass + + def readI64(self): + pass + + def readDouble(self): + pass + + def readString(self): + pass + + def skip(self, type): + if type == TType.STOP: + return + elif type == TType.BOOL: + self.readBool() + elif type == TType.BYTE: + self.readByte() + elif type == TType.I16: + self.readI16() + elif type == TType.I32: + self.readI32() + elif type == TType.I64: + self.readI64() + elif type == TType.DOUBLE: + self.readDouble() + elif type == TType.STRING: + self.readString() + elif type == TType.STRUCT: + name = self.readStructBegin() + while True: + (name, type, id) = self.readFieldBegin() + if type == TType.STOP: + break + self.skip(type) + self.readFieldEnd() + self.readStructEnd() + elif type == TType.MAP: + (ktype, vtype, size) = self.readMapBegin() + for i in range(size): + self.skip(ktype) + self.skip(vtype) + self.readMapEnd() + elif type == TType.SET: + (etype, size) = self.readSetBegin() + for i in range(size): + self.skip(etype) + self.readSetEnd() + elif type == TType.LIST: + (etype, size) = self.readListBegin() + for i in range(size): + self.skip(etype) + self.readListEnd() + +class TProtocolFactory: + def getProtocol(self, trans): + pass diff --git a/backoffice/thrift/protocol/__init__.py b/backoffice/thrift/protocol/__init__.py new file mode 100644 index 0000000..01bfe18 --- /dev/null +++ b/backoffice/thrift/protocol/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TProtocol', 'TBinaryProtocol', 'fastbinary'] diff --git a/backoffice/thrift/protocol/fastbinary.c b/backoffice/thrift/protocol/fastbinary.c new file mode 100644 index 0000000..67b215a --- /dev/null +++ b/backoffice/thrift/protocol/fastbinary.c @@ -0,0 +1,1203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include "cStringIO.h" +#include +#include +#include + +/* Fix endianness issues on Solaris */ +#if defined (__SVR4) && defined (__sun) + #if defined(__i386) && !defined(__i386__) + #define __i386__ + #endif + + #ifndef BIG_ENDIAN + #define BIG_ENDIAN (4321) + #endif + #ifndef LITTLE_ENDIAN + #define LITTLE_ENDIAN (1234) + #endif + + /* I386 is LE, even on Solaris */ + #if !defined(BYTE_ORDER) && defined(__i386__) + #define BYTE_ORDER LITTLE_ENDIAN + #endif +#endif + +// TODO(dreiss): defval appears to be unused. Look into removing it. +// TODO(dreiss): Make parse_spec_args recursive, and cache the output +// permanently in the object. (Malloc and orphan.) +// TODO(dreiss): Why do we need cStringIO for reading, why not just char*? +// Can cStringIO let us work with a BufferedTransport? +// TODO(dreiss): Don't ignore the rv from cwrite (maybe). + +/* ====== BEGIN UTILITIES ====== */ + +#define INIT_OUTBUF_SIZE 128 + +// Stolen out of TProtocol.h. +// It would be a huge pain to have both get this from one place. +typedef enum TType { + T_STOP = 0, + T_VOID = 1, + T_BOOL = 2, + T_BYTE = 3, + T_I08 = 3, + T_I16 = 6, + T_I32 = 8, + T_U64 = 9, + T_I64 = 10, + T_DOUBLE = 4, + T_STRING = 11, + T_UTF7 = 11, + T_STRUCT = 12, + T_MAP = 13, + T_SET = 14, + T_LIST = 15, + T_UTF8 = 16, + T_UTF16 = 17 +} TType; + +#ifndef __BYTE_ORDER +# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN) +# define __BYTE_ORDER BYTE_ORDER +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __BIG_ENDIAN BIG_ENDIAN +# else +# error "Cannot determine endianness" +# endif +#endif + +// Same comment as the enum. Sorry. +#if __BYTE_ORDER == __BIG_ENDIAN +# define ntohll(n) (n) +# define htonll(n) (n) +#elif __BYTE_ORDER == __LITTLE_ENDIAN +# if defined(__GNUC__) && defined(__GLIBC__) +# include +# define ntohll(n) bswap_64(n) +# define htonll(n) bswap_64(n) +# else /* GNUC & GLIBC */ +# define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) ) +# define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) ) +# endif /* GNUC & GLIBC */ +#else /* __BYTE_ORDER */ +# error "Can't define htonll or ntohll!" +#endif + +// Doing a benchmark shows that interning actually makes a difference, amazingly. +#define INTERN_STRING(value) _intern_ ## value + +#define INT_CONV_ERROR_OCCURRED(v) ( ((v) == -1) && PyErr_Occurred() ) +#define CHECK_RANGE(v, min, max) ( ((v) <= (max)) && ((v) >= (min)) ) + +// Py_ssize_t was not defined before Python 2.5 +#if (PY_VERSION_HEX < 0x02050000) +typedef int Py_ssize_t; +#endif + +/** + * A cache of the spec_args for a set or list, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + TType element_type; + PyObject* typeargs; +} SetListTypeArgs; + +/** + * A cache of the spec_args for a map, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + TType ktag; + TType vtag; + PyObject* ktypeargs; + PyObject* vtypeargs; +} MapTypeArgs; + +/** + * A cache of the spec_args for a struct, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + PyObject* klass; + PyObject* spec; +} StructTypeArgs; + +/** + * A cache of the item spec from a struct specification, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + int tag; + TType type; + PyObject* attrname; + PyObject* typeargs; + PyObject* defval; +} StructItemSpec; + +/** + * A cache of the two key attributes of a CReadableTransport, + * so we don't have to keep calling PyObject_GetAttr. + */ +typedef struct { + PyObject* stringiobuf; + PyObject* refill_callable; +} DecodeBuffer; + +/** Pointer to interned string to speed up attribute lookup. */ +static PyObject* INTERN_STRING(cstringio_buf); +/** Pointer to interned string to speed up attribute lookup. */ +static PyObject* INTERN_STRING(cstringio_refill); + +static inline bool +check_ssize_t_32(Py_ssize_t len) { + // error from getting the int + if (INT_CONV_ERROR_OCCURRED(len)) { + return false; + } + if (!CHECK_RANGE(len, 0, INT32_MAX)) { + PyErr_SetString(PyExc_OverflowError, "string size out of range"); + return false; + } + return true; +} + +static inline bool +parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) { + long val = PyInt_AsLong(o); + + if (INT_CONV_ERROR_OCCURRED(val)) { + return false; + } + if (!CHECK_RANGE(val, min, max)) { + PyErr_SetString(PyExc_OverflowError, "int out of range"); + return false; + } + + *ret = (int32_t) val; + return true; +} + + +/* --- FUNCTIONS TO PARSE STRUCT SPECIFICATOINS --- */ + +static bool +parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 2) { + PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args"); + return false; + } + + dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->element_type)) { + return false; + } + + dest->typeargs = PyTuple_GET_ITEM(typeargs, 1); + + return true; +} + +static bool +parse_map_args(MapTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 4) { + PyErr_SetString(PyExc_TypeError, "expecting 4 arguments for typeargs to map"); + return false; + } + + dest->ktag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->ktag)) { + return false; + } + + dest->vtag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2)); + if (INT_CONV_ERROR_OCCURRED(dest->vtag)) { + return false; + } + + dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1); + dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3); + + return true; +} + +static bool +parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 2) { + PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for struct args"); + return false; + } + + dest->klass = PyTuple_GET_ITEM(typeargs, 0); + dest->spec = PyTuple_GET_ITEM(typeargs, 1); + + return true; +} + +static int +parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) { + + // i'd like to use ParseArgs here, but it seems to be a bottleneck. + if (PyTuple_Size(spec_tuple) != 5) { + PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for spec tuple"); + return false; + } + + dest->tag = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->tag)) { + return false; + } + + dest->type = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1)); + if (INT_CONV_ERROR_OCCURRED(dest->type)) { + return false; + } + + dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2); + dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3); + dest->defval = PyTuple_GET_ITEM(spec_tuple, 4); + return true; +} + +/* ====== END UTILITIES ====== */ + + +/* ====== BEGIN WRITING FUNCTIONS ====== */ + +/* --- LOW-LEVEL WRITING FUNCTIONS --- */ + +static void writeByte(PyObject* outbuf, int8_t val) { + int8_t net = val; + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int8_t)); +} + +static void writeI16(PyObject* outbuf, int16_t val) { + int16_t net = (int16_t)htons(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int16_t)); +} + +static void writeI32(PyObject* outbuf, int32_t val) { + int32_t net = (int32_t)htonl(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int32_t)); +} + +static void writeI64(PyObject* outbuf, int64_t val) { + int64_t net = (int64_t)htonll(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int64_t)); +} + +static void writeDouble(PyObject* outbuf, double dub) { + // Unfortunately, bitwise_cast doesn't work in C. Bad C! + union { + double f; + int64_t t; + } transfer; + transfer.f = dub; + writeI64(outbuf, transfer.t); +} + + +/* --- MAIN RECURSIVE OUTPUT FUCNTION -- */ + +static int +output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) { + /* + * Refcounting Strategy: + * + * We assume that elements of the thrift_spec tuple are not going to be + * mutated, so we don't ref count those at all. Other than that, we try to + * keep a reference to all the user-created objects while we work with them. + * output_val assumes that a reference is already held. The *caller* is + * responsible for handling references + */ + + switch (type) { + + case T_BOOL: { + int v = PyObject_IsTrue(value); + if (v == -1) { + return false; + } + + writeByte(output, (int8_t) v); + break; + } + case T_I08: { + int32_t val; + + if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) { + return false; + } + + writeByte(output, (int8_t) val); + break; + } + case T_I16: { + int32_t val; + + if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) { + return false; + } + + writeI16(output, (int16_t) val); + break; + } + case T_I32: { + int32_t val; + + if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) { + return false; + } + + writeI32(output, val); + break; + } + case T_I64: { + int64_t nval = PyLong_AsLongLong(value); + + if (INT_CONV_ERROR_OCCURRED(nval)) { + return false; + } + + if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) { + PyErr_SetString(PyExc_OverflowError, "int out of range"); + return false; + } + + writeI64(output, nval); + break; + } + + case T_DOUBLE: { + double nval = PyFloat_AsDouble(value); + if (nval == -1.0 && PyErr_Occurred()) { + return false; + } + + writeDouble(output, nval); + break; + } + + case T_STRING: { + Py_ssize_t len = PyString_Size(value); + + if (!check_ssize_t_32(len)) { + return false; + } + + writeI32(output, (int32_t) len); + PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len); + break; + } + + case T_LIST: + case T_SET: { + Py_ssize_t len; + SetListTypeArgs parsedargs; + PyObject *item; + PyObject *iterator; + + if (!parse_set_list_args(&parsedargs, typeargs)) { + return false; + } + + len = PyObject_Length(value); + + if (!check_ssize_t_32(len)) { + return false; + } + + writeByte(output, parsedargs.element_type); + writeI32(output, (int32_t) len); + + iterator = PyObject_GetIter(value); + if (iterator == NULL) { + return false; + } + + while ((item = PyIter_Next(iterator))) { + if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) { + Py_DECREF(item); + Py_DECREF(iterator); + return false; + } + Py_DECREF(item); + } + + Py_DECREF(iterator); + + if (PyErr_Occurred()) { + return false; + } + + break; + } + + case T_MAP: { + PyObject *k, *v; + Py_ssize_t pos = 0; + Py_ssize_t len; + + MapTypeArgs parsedargs; + + len = PyDict_Size(value); + if (!check_ssize_t_32(len)) { + return false; + } + + if (!parse_map_args(&parsedargs, typeargs)) { + return false; + } + + writeByte(output, parsedargs.ktag); + writeByte(output, parsedargs.vtag); + writeI32(output, len); + + // TODO(bmaurer): should support any mapping, not just dicts + while (PyDict_Next(value, &pos, &k, &v)) { + // TODO(dreiss): Think hard about whether these INCREFs actually + // turn any unsafe scenarios into safe scenarios. + Py_INCREF(k); + Py_INCREF(v); + + if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs) + || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) { + Py_DECREF(k); + Py_DECREF(v); + return false; + } + Py_DECREF(k); + Py_DECREF(v); + } + break; + } + + // TODO(dreiss): Consider breaking this out as a function + // the way we did for decode_struct. + case T_STRUCT: { + StructTypeArgs parsedargs; + Py_ssize_t nspec; + Py_ssize_t i; + + if (!parse_struct_args(&parsedargs, typeargs)) { + return false; + } + + nspec = PyTuple_Size(parsedargs.spec); + + if (nspec == -1) { + return false; + } + + for (i = 0; i < nspec; i++) { + StructItemSpec parsedspec; + PyObject* spec_tuple; + PyObject* instval = NULL; + + spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i); + if (spec_tuple == Py_None) { + continue; + } + + if (!parse_struct_item_spec (&parsedspec, spec_tuple)) { + return false; + } + + instval = PyObject_GetAttr(value, parsedspec.attrname); + + if (!instval) { + return false; + } + + if (instval == Py_None) { + Py_DECREF(instval); + continue; + } + + writeByte(output, (int8_t) parsedspec.type); + writeI16(output, parsedspec.tag); + + if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) { + Py_DECREF(instval); + return false; + } + + Py_DECREF(instval); + } + + writeByte(output, (int8_t)T_STOP); + break; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return false; + + } + + return true; +} + + +/* --- TOP-LEVEL WRAPPER FOR OUTPUT -- */ + +static PyObject * +encode_binary(PyObject *self, PyObject *args) { + PyObject* enc_obj; + PyObject* type_args; + PyObject* buf; + PyObject* ret = NULL; + + if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) { + return NULL; + } + + buf = PycStringIO->NewOutput(INIT_OUTBUF_SIZE); + if (output_val(buf, enc_obj, T_STRUCT, type_args)) { + ret = PycStringIO->cgetvalue(buf); + } + + Py_DECREF(buf); + return ret; +} + +/* ====== END WRITING FUNCTIONS ====== */ + + +/* ====== BEGIN READING FUNCTIONS ====== */ + +/* --- LOW-LEVEL READING FUNCTIONS --- */ + +static void +free_decodebuf(DecodeBuffer* d) { + Py_XDECREF(d->stringiobuf); + Py_XDECREF(d->refill_callable); +} + +static bool +decode_buffer_from_obj(DecodeBuffer* dest, PyObject* obj) { + dest->stringiobuf = PyObject_GetAttr(obj, INTERN_STRING(cstringio_buf)); + if (!dest->stringiobuf) { + return false; + } + + if (!PycStringIO_InputCheck(dest->stringiobuf)) { + free_decodebuf(dest); + PyErr_SetString(PyExc_TypeError, "expecting stringio input"); + return false; + } + + dest->refill_callable = PyObject_GetAttr(obj, INTERN_STRING(cstringio_refill)); + + if(!dest->refill_callable) { + free_decodebuf(dest); + return false; + } + + if (!PyCallable_Check(dest->refill_callable)) { + free_decodebuf(dest); + PyErr_SetString(PyExc_TypeError, "expecting callable"); + return false; + } + + return true; +} + +static bool readBytes(DecodeBuffer* input, char** output, int len) { + int read; + + // TODO(dreiss): Don't fear the malloc. Think about taking a copy of + // the partial read instead of forcing the transport + // to prepend it to its buffer. + + read = PycStringIO->cread(input->stringiobuf, output, len); + + if (read == len) { + return true; + } else if (read == -1) { + return false; + } else { + PyObject* newiobuf; + + // using building functions as this is a rare codepath + newiobuf = PyObject_CallFunction( + input->refill_callable, "s#i", *output, read, len, NULL); + if (newiobuf == NULL) { + return false; + } + + // must do this *AFTER* the call so that we don't deref the io buffer + Py_CLEAR(input->stringiobuf); + input->stringiobuf = newiobuf; + + read = PycStringIO->cread(input->stringiobuf, output, len); + + if (read == len) { + return true; + } else if (read == -1) { + return false; + } else { + // TODO(dreiss): This could be a valid code path for big binary blobs. + PyErr_SetString(PyExc_TypeError, + "refill claimed to have refilled the buffer, but didn't!!"); + return false; + } + } +} + +static int8_t readByte(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int8_t))) { + return -1; + } + + return *(int8_t*) buf; +} + +static int16_t readI16(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int16_t))) { + return -1; + } + + return (int16_t) ntohs(*(int16_t*) buf); +} + +static int32_t readI32(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int32_t))) { + return -1; + } + return (int32_t) ntohl(*(int32_t*) buf); +} + + +static int64_t readI64(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int64_t))) { + return -1; + } + + return (int64_t) ntohll(*(int64_t*) buf); +} + +static double readDouble(DecodeBuffer* input) { + union { + int64_t f; + double t; + } transfer; + + transfer.f = readI64(input); + if (transfer.f == -1) { + return -1; + } + return transfer.t; +} + +static bool +checkTypeByte(DecodeBuffer* input, TType expected) { + TType got = readByte(input); + if (INT_CONV_ERROR_OCCURRED(got)) { + return false; + } + + if (expected != got) { + PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field"); + return false; + } + return true; +} + +static bool +skip(DecodeBuffer* input, TType type) { +#define SKIPBYTES(n) \ + do { \ + if (!readBytes(input, &dummy_buf, (n))) { \ + return false; \ + } \ + } while(0) + + char* dummy_buf; + + switch (type) { + + case T_BOOL: + case T_I08: SKIPBYTES(1); break; + case T_I16: SKIPBYTES(2); break; + case T_I32: SKIPBYTES(4); break; + case T_I64: + case T_DOUBLE: SKIPBYTES(8); break; + + case T_STRING: { + // TODO(dreiss): Find out if these check_ssize_t32s are really necessary. + int len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + SKIPBYTES(len); + break; + } + + case T_LIST: + case T_SET: { + TType etype; + int len, i; + + etype = readByte(input); + if (etype == -1) { + return false; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + for (i = 0; i < len; i++) { + if (!skip(input, etype)) { + return false; + } + } + break; + } + + case T_MAP: { + TType ktype, vtype; + int len, i; + + ktype = readByte(input); + if (ktype == -1) { + return false; + } + + vtype = readByte(input); + if (vtype == -1) { + return false; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + for (i = 0; i < len; i++) { + if (!(skip(input, ktype) && skip(input, vtype))) { + return false; + } + } + break; + } + + case T_STRUCT: { + while (true) { + TType type; + + type = readByte(input); + if (type == -1) { + return false; + } + + if (type == T_STOP) + break; + + SKIPBYTES(2); // tag + if (!skip(input, type)) { + return false; + } + } + break; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return false; + + } + + return true; + +#undef SKIPBYTES +} + + +/* --- HELPER FUNCTION FOR DECODE_VAL --- */ + +static PyObject* +decode_val(DecodeBuffer* input, TType type, PyObject* typeargs); + +static bool +decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) { + int spec_seq_len = PyTuple_Size(spec_seq); + if (spec_seq_len == -1) { + return false; + } + + while (true) { + TType type; + int16_t tag; + PyObject* item_spec; + PyObject* fieldval = NULL; + StructItemSpec parsedspec; + + type = readByte(input); + if (type == -1) { + return false; + } + if (type == T_STOP) { + break; + } + tag = readI16(input); + if (INT_CONV_ERROR_OCCURRED(tag)) { + return false; + } + if (tag >= 0 && tag < spec_seq_len) { + item_spec = PyTuple_GET_ITEM(spec_seq, tag); + } else { + item_spec = Py_None; + } + + if (item_spec == Py_None) { + if (!skip(input, type)) { + return false; + } else { + continue; + } + } + + if (!parse_struct_item_spec(&parsedspec, item_spec)) { + return false; + } + if (parsedspec.type != type) { + if (!skip(input, type)) { + PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading and can't be skipped"); + return false; + } else { + continue; + } + } + + fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs); + if (fieldval == NULL) { + return false; + } + + if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) { + Py_DECREF(fieldval); + return false; + } + Py_DECREF(fieldval); + } + return true; +} + + +/* --- MAIN RECURSIVE INPUT FUCNTION --- */ + +// Returns a new reference. +static PyObject* +decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) { + switch (type) { + + case T_BOOL: { + int8_t v = readByte(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + + switch (v) { + case 0: Py_RETURN_FALSE; + case 1: Py_RETURN_TRUE; + // Don't laugh. This is a potentially serious issue. + default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL; + } + break; + } + case T_I08: { + int8_t v = readByte(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + + return PyInt_FromLong(v); + } + case T_I16: { + int16_t v = readI16(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + return PyInt_FromLong(v); + } + case T_I32: { + int32_t v = readI32(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + return PyInt_FromLong(v); + } + + case T_I64: { + int64_t v = readI64(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + // TODO(dreiss): Find out if we can take this fastpath always when + // sizeof(long) == sizeof(long long). + if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) { + return PyInt_FromLong((long) v); + } + + return PyLong_FromLongLong(v); + } + + case T_DOUBLE: { + double v = readDouble(input); + if (v == -1.0 && PyErr_Occurred()) { + return false; + } + return PyFloat_FromDouble(v); + } + + case T_STRING: { + Py_ssize_t len = readI32(input); + char* buf; + if (!readBytes(input, &buf, len)) { + return NULL; + } + + return PyString_FromStringAndSize(buf, len); + } + + case T_LIST: + case T_SET: { + SetListTypeArgs parsedargs; + int32_t len; + PyObject* ret = NULL; + int i; + + if (!parse_set_list_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!checkTypeByte(input, parsedargs.element_type)) { + return NULL; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return NULL; + } + + ret = PyList_New(len); + if (!ret) { + return NULL; + } + + for (i = 0; i < len; i++) { + PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs); + if (!item) { + Py_DECREF(ret); + return NULL; + } + PyList_SET_ITEM(ret, i, item); + } + + // TODO(dreiss): Consider biting the bullet and making two separate cases + // for list and set, avoiding this post facto conversion. + if (type == T_SET) { + PyObject* setret; +#if (PY_VERSION_HEX < 0x02050000) + // hack needed for older versions + setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL); +#else + // official version + setret = PySet_New(ret); +#endif + Py_DECREF(ret); + return setret; + } + return ret; + } + + case T_MAP: { + int32_t len; + int i; + MapTypeArgs parsedargs; + PyObject* ret = NULL; + + if (!parse_map_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!checkTypeByte(input, parsedargs.ktag)) { + return NULL; + } + if (!checkTypeByte(input, parsedargs.vtag)) { + return NULL; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + ret = PyDict_New(); + if (!ret) { + goto error; + } + + for (i = 0; i < len; i++) { + PyObject* k = NULL; + PyObject* v = NULL; + k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs); + if (k == NULL) { + goto loop_error; + } + v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs); + if (v == NULL) { + goto loop_error; + } + if (PyDict_SetItem(ret, k, v) == -1) { + goto loop_error; + } + + Py_DECREF(k); + Py_DECREF(v); + continue; + + // Yuck! Destructors, anyone? + loop_error: + Py_XDECREF(k); + Py_XDECREF(v); + goto error; + } + + return ret; + + error: + Py_XDECREF(ret); + return NULL; + } + + case T_STRUCT: { + StructTypeArgs parsedargs; + if (!parse_struct_args(&parsedargs, typeargs)) { + return NULL; + } + + PyObject* ret = PyObject_CallObject(parsedargs.klass, NULL); + if (!ret) { + return NULL; + } + + if (!decode_struct(input, ret, parsedargs.spec)) { + Py_DECREF(ret); + return NULL; + } + + return ret; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return NULL; + } +} + + +/* --- TOP-LEVEL WRAPPER FOR INPUT -- */ + +static PyObject* +decode_binary(PyObject *self, PyObject *args) { + PyObject* output_obj = NULL; + PyObject* transport = NULL; + PyObject* typeargs = NULL; + StructTypeArgs parsedargs; + DecodeBuffer input = {}; + + if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) { + return NULL; + } + + if (!parse_struct_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!decode_buffer_from_obj(&input, transport)) { + return NULL; + } + + if (!decode_struct(&input, output_obj, parsedargs.spec)) { + free_decodebuf(&input); + return NULL; + } + + free_decodebuf(&input); + + Py_RETURN_NONE; +} + +/* ====== END READING FUNCTIONS ====== */ + + +/* -- PYTHON MODULE SETUP STUFF --- */ + +static PyMethodDef ThriftFastBinaryMethods[] = { + + {"encode_binary", encode_binary, METH_VARARGS, ""}, + {"decode_binary", decode_binary, METH_VARARGS, ""}, + + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +PyMODINIT_FUNC +initfastbinary(void) { +#define INIT_INTERN_STRING(value) \ + do { \ + INTERN_STRING(value) = PyString_InternFromString(#value); \ + if(!INTERN_STRING(value)) return; \ + } while(0) + + INIT_INTERN_STRING(cstringio_buf); + INIT_INTERN_STRING(cstringio_refill); +#undef INIT_INTERN_STRING + + PycString_IMPORT; + if (PycStringIO == NULL) return; + + (void) Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods); +} diff --git a/backoffice/thrift/server/THttpServer.py b/backoffice/thrift/server/THttpServer.py new file mode 100644 index 0000000..3047d9c --- /dev/null +++ b/backoffice/thrift/server/THttpServer.py @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import BaseHTTPServer + +from thrift.server import TServer +from thrift.transport import TTransport + +class ResponseException(Exception): + """Allows handlers to override the HTTP response + + Normally, THttpServer always sends a 200 response. If a handler wants + to override this behavior (e.g., to simulate a misconfigured or + overloaded web server during testing), it can raise a ResponseException. + The function passed to the constructor will be called with the + RequestHandler as its only argument. + """ + def __init__(self, handler): + self.handler = handler + + +class THttpServer(TServer.TServer): + """A simple HTTP-based Thrift server + + This class is not very performant, but it is useful (for example) for + acting as a mock version of an Apache-based PHP Thrift endpoint.""" + + def __init__(self, processor, server_address, + inputProtocolFactory, outputProtocolFactory = None, + server_class = BaseHTTPServer.HTTPServer): + """Set up protocol factories and HTTP server. + + See BaseHTTPServer for server_address. + See TServer for protocol factories.""" + + if outputProtocolFactory is None: + outputProtocolFactory = inputProtocolFactory + + TServer.TServer.__init__(self, processor, None, None, None, + inputProtocolFactory, outputProtocolFactory) + + thttpserver = self + + class RequestHander(BaseHTTPServer.BaseHTTPRequestHandler): + def do_POST(self): + # Don't care about the request path. + itrans = TTransport.TFileObjectTransport(self.rfile) + otrans = TTransport.TFileObjectTransport(self.wfile) + itrans = TTransport.TBufferedTransport(itrans, int(self.headers['Content-Length'])) + otrans = TTransport.TMemoryBuffer() + iprot = thttpserver.inputProtocolFactory.getProtocol(itrans) + oprot = thttpserver.outputProtocolFactory.getProtocol(otrans) + try: + thttpserver.processor.process(iprot, oprot) + except ResponseException, exn: + exn.handler(self) + else: + self.send_response(200) + self.send_header("content-type", "application/x-thrift") + self.end_headers() + self.wfile.write(otrans.getvalue()) + + self.httpd = server_class(server_address, RequestHander) + + def serve(self): + self.httpd.serve_forever() diff --git a/backoffice/thrift/server/TNonblockingServer.py b/backoffice/thrift/server/TNonblockingServer.py new file mode 100644 index 0000000..ea348a0 --- /dev/null +++ b/backoffice/thrift/server/TNonblockingServer.py @@ -0,0 +1,310 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +"""Implementation of non-blocking server. + +The main idea of the server is reciving and sending requests +only from main thread. + +It also makes thread pool server in tasks terms, not connections. +""" +import threading +import socket +import Queue +import select +import struct +import logging + +from thrift.transport import TTransport +from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory + +__all__ = ['TNonblockingServer'] + +class Worker(threading.Thread): + """Worker is a small helper to process incoming connection.""" + def __init__(self, queue): + threading.Thread.__init__(self) + self.queue = queue + + def run(self): + """Process queries from task queue, stop if processor is None.""" + while True: + try: + processor, iprot, oprot, otrans, callback = self.queue.get() + if processor is None: + break + processor.process(iprot, oprot) + callback(True, otrans.getvalue()) + except Exception: + logging.exception("Exception while processing request") + callback(False, '') + +WAIT_LEN = 0 +WAIT_MESSAGE = 1 +WAIT_PROCESS = 2 +SEND_ANSWER = 3 +CLOSED = 4 + +def locked(func): + "Decorator which locks self.lock." + def nested(self, *args, **kwargs): + self.lock.acquire() + try: + return func(self, *args, **kwargs) + finally: + self.lock.release() + return nested + +def socket_exception(func): + "Decorator close object on socket.error." + def read(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except socket.error: + self.close() + return read + +class Connection: + """Basic class is represented connection. + + It can be in state: + WAIT_LEN --- connection is reading request len. + WAIT_MESSAGE --- connection is reading request. + WAIT_PROCESS --- connection has just read whole request and + waits for call ready routine. + SEND_ANSWER --- connection is sending answer string (including length + of answer). + CLOSED --- socket was closed and connection should be deleted. + """ + def __init__(self, new_socket, wake_up): + self.socket = new_socket + self.socket.setblocking(False) + self.status = WAIT_LEN + self.len = 0 + self.message = '' + self.lock = threading.Lock() + self.wake_up = wake_up + + def _read_len(self): + """Reads length of request. + + It's really paranoic routine and it may be replaced by + self.socket.recv(4).""" + read = self.socket.recv(4 - len(self.message)) + if len(read) == 0: + # if we read 0 bytes and self.message is empty, it means client close + # connection + if len(self.message) != 0: + logging.error("can't read frame size from socket") + self.close() + return + self.message += read + if len(self.message) == 4: + self.len, = struct.unpack('!i', self.message) + if self.len < 0: + logging.error("negative frame size, it seems client"\ + " doesn't use FramedTransport") + self.close() + elif self.len == 0: + logging.error("empty frame, it's really strange") + self.close() + else: + self.message = '' + self.status = WAIT_MESSAGE + + @socket_exception + def read(self): + """Reads data from stream and switch state.""" + assert self.status in (WAIT_LEN, WAIT_MESSAGE) + if self.status == WAIT_LEN: + self._read_len() + # go back to the main loop here for simplicity instead of + # falling through, even though there is a good chance that + # the message is already available + elif self.status == WAIT_MESSAGE: + read = self.socket.recv(self.len - len(self.message)) + if len(read) == 0: + logging.error("can't read frame from socket (get %d of %d bytes)" % + (len(self.message), self.len)) + self.close() + return + self.message += read + if len(self.message) == self.len: + self.status = WAIT_PROCESS + + @socket_exception + def write(self): + """Writes data from socket and switch state.""" + assert self.status == SEND_ANSWER + sent = self.socket.send(self.message) + if sent == len(self.message): + self.status = WAIT_LEN + self.message = '' + self.len = 0 + else: + self.message = self.message[sent:] + + @locked + def ready(self, all_ok, message): + """Callback function for switching state and waking up main thread. + + This function is the only function witch can be called asynchronous. + + The ready can switch Connection to three states: + WAIT_LEN if request was oneway. + SEND_ANSWER if request was processed in normal way. + CLOSED if request throws unexpected exception. + + The one wakes up main thread. + """ + assert self.status == WAIT_PROCESS + if not all_ok: + self.close() + self.wake_up() + return + self.len = '' + if len(message) == 0: + # it was a oneway request, do not write answer + self.message = '' + self.status = WAIT_LEN + else: + self.message = struct.pack('!i', len(message)) + message + self.status = SEND_ANSWER + self.wake_up() + + @locked + def is_writeable(self): + "Returns True if connection should be added to write list of select." + return self.status == SEND_ANSWER + + # it's not necessary, but... + @locked + def is_readable(self): + "Returns True if connection should be added to read list of select." + return self.status in (WAIT_LEN, WAIT_MESSAGE) + + @locked + def is_closed(self): + "Returns True if connection is closed." + return self.status == CLOSED + + def fileno(self): + "Returns the file descriptor of the associated socket." + return self.socket.fileno() + + def close(self): + "Closes connection" + self.status = CLOSED + self.socket.close() + +class TNonblockingServer: + """Non-blocking server.""" + def __init__(self, processor, lsocket, inputProtocolFactory=None, + outputProtocolFactory=None, threads=10): + self.processor = processor + self.socket = lsocket + self.in_protocol = inputProtocolFactory or TBinaryProtocolFactory() + self.out_protocol = outputProtocolFactory or self.in_protocol + self.threads = int(threads) + self.clients = {} + self.tasks = Queue.Queue() + self._read, self._write = socket.socketpair() + self.prepared = False + + def setNumThreads(self, num): + """Set the number of worker threads that should be created.""" + # implement ThreadPool interface + assert not self.prepared, "You can't change number of threads for working server" + self.threads = num + + def prepare(self): + """Prepares server for serve requests.""" + self.socket.listen() + for _ in xrange(self.threads): + thread = Worker(self.tasks) + thread.setDaemon(True) + thread.start() + self.prepared = True + + def wake_up(self): + """Wake up main thread. + + The server usualy waits in select call in we should terminate one. + The simplest way is using socketpair. + + Select always wait to read from the first socket of socketpair. + + In this case, we can just write anything to the second socket from + socketpair.""" + self._write.send('1') + + def _select(self): + """Does select on open connections.""" + readable = [self.socket.handle.fileno(), self._read.fileno()] + writable = [] + for i, connection in self.clients.items(): + if connection.is_readable(): + readable.append(connection.fileno()) + if connection.is_writeable(): + writable.append(connection.fileno()) + if connection.is_closed(): + del self.clients[i] + return select.select(readable, writable, readable) + + def handle(self): + """Handle requests. + + WARNING! You must call prepare BEFORE calling handle. + """ + assert self.prepared, "You have to call prepare before handle" + rset, wset, xset = self._select() + for readable in rset: + if readable == self._read.fileno(): + # don't care i just need to clean readable flag + self._read.recv(1024) + elif readable == self.socket.handle.fileno(): + client = self.socket.accept().handle + self.clients[client.fileno()] = Connection(client, self.wake_up) + else: + connection = self.clients[readable] + connection.read() + if connection.status == WAIT_PROCESS: + itransport = TTransport.TMemoryBuffer(connection.message) + otransport = TTransport.TMemoryBuffer() + iprot = self.in_protocol.getProtocol(itransport) + oprot = self.out_protocol.getProtocol(otransport) + self.tasks.put([self.processor, iprot, oprot, + otransport, connection.ready]) + for writeable in wset: + self.clients[writeable].write() + for oob in xset: + self.clients[oob].close() + del self.clients[oob] + + def close(self): + """Closes the server.""" + for _ in xrange(self.threads): + self.tasks.put([None, None, None, None, None]) + self.socket.close() + self.prepared = False + + def serve(self): + """Serve forever.""" + self.prepare() + while True: + self.handle() diff --git a/backoffice/thrift/server/TServer.py b/backoffice/thrift/server/TServer.py new file mode 100644 index 0000000..8456e2d --- /dev/null +++ b/backoffice/thrift/server/TServer.py @@ -0,0 +1,274 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import logging +import sys +import os +import traceback +import threading +import Queue + +from thrift.Thrift import TProcessor +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol + +class TServer: + + """Base interface for a server, which must have a serve method.""" + + """ 3 constructors for all servers: + 1) (processor, serverTransport) + 2) (processor, serverTransport, transportFactory, protocolFactory) + 3) (processor, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory)""" + def __init__(self, *args): + if (len(args) == 2): + self.__initArgs__(args[0], args[1], + TTransport.TTransportFactoryBase(), + TTransport.TTransportFactoryBase(), + TBinaryProtocol.TBinaryProtocolFactory(), + TBinaryProtocol.TBinaryProtocolFactory()) + elif (len(args) == 4): + self.__initArgs__(args[0], args[1], args[2], args[2], args[3], args[3]) + elif (len(args) == 6): + self.__initArgs__(args[0], args[1], args[2], args[3], args[4], args[5]) + + def __initArgs__(self, processor, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory): + self.processor = processor + self.serverTransport = serverTransport + self.inputTransportFactory = inputTransportFactory + self.outputTransportFactory = outputTransportFactory + self.inputProtocolFactory = inputProtocolFactory + self.outputProtocolFactory = outputProtocolFactory + + def serve(self): + pass + +class TSimpleServer(TServer): + + """Simple single-threaded server that just pumps around one transport.""" + + def __init__(self, *args): + TServer.__init__(self, *args) + + def serve(self): + self.serverTransport.listen() + while True: + client = self.serverTransport.accept() + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + +class TThreadedServer(TServer): + + """Threaded server that spawns a new thread per each connection.""" + + def __init__(self, *args, **kwargs): + TServer.__init__(self, *args) + self.daemon = kwargs.get("daemon", False) + + def serve(self): + self.serverTransport.listen() + while True: + try: + client = self.serverTransport.accept() + t = threading.Thread(target = self.handle, args=(client,)) + t.setDaemon(self.daemon) + t.start() + except KeyboardInterrupt: + raise + except Exception, x: + logging.exception(x) + + def handle(self, client): + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + +class TThreadPoolServer(TServer): + + """Server with a fixed size pool of threads which service requests.""" + + def __init__(self, *args, **kwargs): + TServer.__init__(self, *args) + self.clients = Queue.Queue() + self.threads = 10 + self.daemon = kwargs.get("daemon", False) + + def setNumThreads(self, num): + """Set the number of worker threads that should be created""" + self.threads = num + + def serveThread(self): + """Loop around getting clients from the shared queue and process them.""" + while True: + try: + client = self.clients.get() + self.serveClient(client) + except Exception, x: + logging.exception(x) + + def serveClient(self, client): + """Process input/output from a client for as long as possible""" + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + + def serve(self): + """Start a fixed number of worker threads and put client into a queue""" + for i in range(self.threads): + try: + t = threading.Thread(target = self.serveThread) + t.setDaemon(self.daemon) + t.start() + except Exception, x: + logging.exception(x) + + # Pump the socket for clients + self.serverTransport.listen() + while True: + try: + client = self.serverTransport.accept() + self.clients.put(client) + except Exception, x: + logging.exception(x) + + +class TForkingServer(TServer): + + """A Thrift server that forks a new process for each request""" + """ + This is more scalable than the threaded server as it does not cause + GIL contention. + + Note that this has different semantics from the threading server. + Specifically, updates to shared variables will no longer be shared. + It will also not work on windows. + + This code is heavily inspired by SocketServer.ForkingMixIn in the + Python stdlib. + """ + + def __init__(self, *args): + TServer.__init__(self, *args) + self.children = [] + + def serve(self): + def try_close(file): + try: + file.close() + except IOError, e: + logging.warning(e, exc_info=True) + + + self.serverTransport.listen() + while True: + client = self.serverTransport.accept() + try: + pid = os.fork() + + if pid: # parent + # add before collect, otherwise you race w/ waitpid + self.children.append(pid) + self.collect_children() + + # Parent must close socket or the connection may not get + # closed promptly + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + try_close(itrans) + try_close(otrans) + else: + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + + ecode = 0 + try: + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, e: + logging.exception(e) + ecode = 1 + finally: + try_close(itrans) + try_close(otrans) + + os._exit(ecode) + + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + + def collect_children(self): + while self.children: + try: + pid, status = os.waitpid(0, os.WNOHANG) + except os.error: + pid = None + + if pid: + self.children.remove(pid) + else: + break + + diff --git a/backoffice/thrift/server/__init__.py b/backoffice/thrift/server/__init__.py new file mode 100644 index 0000000..1bf6e25 --- /dev/null +++ b/backoffice/thrift/server/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TServer', 'TNonblockingServer'] diff --git a/backoffice/thrift/transport/THttpClient.py b/backoffice/thrift/transport/THttpClient.py new file mode 100644 index 0000000..5026978 --- /dev/null +++ b/backoffice/thrift/transport/THttpClient.py @@ -0,0 +1,126 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TTransport import * +from cStringIO import StringIO + +import urlparse +import httplib +import warnings +import socket + +class THttpClient(TTransportBase): + + """Http implementation of TTransport base.""" + + def __init__(self, uri_or_host, port=None, path=None): + """THttpClient supports two different types constructor parameters. + + THttpClient(host, port, path) - deprecated + THttpClient(uri) + + Only the second supports https.""" + + if port is not None: + warnings.warn("Please use the THttpClient('http://host:port/path') syntax", DeprecationWarning, stacklevel=2) + self.host = uri_or_host + self.port = port + assert path + self.path = path + self.scheme = 'http' + else: + parsed = urlparse.urlparse(uri_or_host) + self.scheme = parsed.scheme + assert self.scheme in ('http', 'https') + if self.scheme == 'http': + self.port = parsed.port or httplib.HTTP_PORT + elif self.scheme == 'https': + self.port = parsed.port or httplib.HTTPS_PORT + self.host = parsed.hostname + self.path = parsed.path + if parsed.query: + self.path += '?%s' % parsed.query + self.__wbuf = StringIO() + self.__http = None + self.__timeout = None + + def open(self): + if self.scheme == 'http': + self.__http = httplib.HTTP(self.host, self.port) + else: + self.__http = httplib.HTTPS(self.host, self.port) + + def close(self): + self.__http.close() + self.__http = None + + def isOpen(self): + return self.__http != None + + def setTimeout(self, ms): + if not hasattr(socket, 'getdefaulttimeout'): + raise NotImplementedError + + if ms is None: + self.__timeout = None + else: + self.__timeout = ms/1000.0 + + def read(self, sz): + return self.__http.file.read(sz) + + def write(self, buf): + self.__wbuf.write(buf) + + def __withTimeout(f): + def _f(*args, **kwargs): + orig_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(args[0].__timeout) + result = f(*args, **kwargs) + socket.setdefaulttimeout(orig_timeout) + return result + return _f + + def flush(self): + if self.isOpen(): + self.close() + self.open(); + + # Pull data out of buffer + data = self.__wbuf.getvalue() + self.__wbuf = StringIO() + + # HTTP request + self.__http.putrequest('POST', self.path) + + # Write headers + self.__http.putheader('Host', self.host) + self.__http.putheader('Content-Type', 'application/x-thrift') + self.__http.putheader('Content-Length', str(len(data))) + self.__http.endheaders() + + # Write payload + self.__http.send(data) + + # Get reply to flush the request + self.code, self.message, self.headers = self.__http.getreply() + + # Decorate if we know how to timeout + if hasattr(socket, 'getdefaulttimeout'): + flush = __withTimeout(flush) diff --git a/backoffice/thrift/transport/TSocket.py b/backoffice/thrift/transport/TSocket.py new file mode 100644 index 0000000..d77e358 --- /dev/null +++ b/backoffice/thrift/transport/TSocket.py @@ -0,0 +1,163 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TTransport import * +import os +import errno +import socket +import sys + +class TSocketBase(TTransportBase): + def _resolveAddr(self): + if self._unix_socket is not None: + return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, self._unix_socket)] + else: + return socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE | socket.AI_ADDRCONFIG) + + def close(self): + if self.handle: + self.handle.close() + self.handle = None + +class TSocket(TSocketBase): + """Socket implementation of TTransport base.""" + + def __init__(self, host='localhost', port=9090, unix_socket=None): + """Initialize a TSocket + + @param host(str) The host to connect to. + @param port(int) The (TCP) port to connect to. + @param unix_socket(str) The filename of a unix socket to connect to. + (host and port will be ignored.) + """ + + self.host = host + self.port = port + self.handle = None + self._unix_socket = unix_socket + self._timeout = None + + def setHandle(self, h): + self.handle = h + + def isOpen(self): + return self.handle != None + + def setTimeout(self, ms): + if ms is None: + self._timeout = None + else: + self._timeout = ms/1000.0 + + if (self.handle != None): + self.handle.settimeout(self._timeout) + + def open(self): + try: + res0 = self._resolveAddr() + for res in res0: + self.handle = socket.socket(res[0], res[1]) + self.handle.settimeout(self._timeout) + try: + self.handle.connect(res[4]) + except socket.error, e: + if res is not res0[-1]: + continue + else: + raise e + break + except socket.error, e: + if self._unix_socket: + message = 'Could not connect to socket %s' % self._unix_socket + else: + message = 'Could not connect to %s:%d' % (self.host, self.port) + raise TTransportException(type=TTransportException.NOT_OPEN, message=message) + + def read(self, sz): + try: + buff = self.handle.recv(sz) + except socket.error, e: + if (e.args[0] == errno.ECONNRESET and + (sys.platform == 'darwin' or sys.platform.startswith('freebsd'))): + # freebsd and Mach don't follow POSIX semantic of recv + # and fail with ECONNRESET if peer performed shutdown. + # See corresponding comment and code in TSocket::read() + # in lib/cpp/src/transport/TSocket.cpp. + self.close() + # Trigger the check to raise the END_OF_FILE exception below. + buff = '' + else: + raise + if len(buff) == 0: + raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket read 0 bytes') + return buff + + def write(self, buff): + if not self.handle: + raise TTransportException(type=TTransportException.NOT_OPEN, message='Transport not open') + sent = 0 + have = len(buff) + while sent < have: + plus = self.handle.send(buff) + if plus == 0: + raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket sent 0 bytes') + sent += plus + buff = buff[plus:] + + def flush(self): + pass + +class TServerSocket(TSocketBase, TServerTransportBase): + """Socket implementation of TServerTransport base.""" + + def __init__(self, port=9090, unix_socket=None): + self.host = None + self.port = port + self._unix_socket = unix_socket + self.handle = None + + def listen(self): + res0 = self._resolveAddr() + for res in res0: + if res[0] is socket.AF_INET6 or res is res0[-1]: + break + + # We need remove the old unix socket if the file exists and + # nobody is listening on it. + if self._unix_socket: + tmp = socket.socket(res[0], res[1]) + try: + tmp.connect(res[4]) + except socket.error, err: + eno, message = err.args + if eno == errno.ECONNREFUSED: + os.unlink(res[4]) + + self.handle = socket.socket(res[0], res[1]) + self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(self.handle, 'set_timeout'): + self.handle.set_timeout(None) + self.handle.bind(res[4]) + self.handle.listen(128) + + def accept(self): + client, addr = self.handle.accept() + result = TSocket() + result.setHandle(client) + return result diff --git a/backoffice/thrift/transport/TTransport.py b/backoffice/thrift/transport/TTransport.py new file mode 100644 index 0000000..12e51a9 --- /dev/null +++ b/backoffice/thrift/transport/TTransport.py @@ -0,0 +1,331 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from cStringIO import StringIO +from struct import pack,unpack +from thrift.Thrift import TException + +class TTransportException(TException): + + """Custom Transport Exception class""" + + UNKNOWN = 0 + NOT_OPEN = 1 + ALREADY_OPEN = 2 + TIMED_OUT = 3 + END_OF_FILE = 4 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + +class TTransportBase: + + """Base class for Thrift transport layer.""" + + def isOpen(self): + pass + + def open(self): + pass + + def close(self): + pass + + def read(self, sz): + pass + + def readAll(self, sz): + buff = '' + have = 0 + while (have < sz): + chunk = self.read(sz-have) + have += len(chunk) + buff += chunk + + if len(chunk) == 0: + raise EOFError() + + return buff + + def write(self, buf): + pass + + def flush(self): + pass + +# This class should be thought of as an interface. +class CReadableTransport: + """base class for transports that are readable from C""" + + # TODO(dreiss): Think about changing this interface to allow us to use + # a (Python, not c) StringIO instead, because it allows + # you to write after reading. + + # NOTE: This is a classic class, so properties will NOT work + # correctly for setting. + @property + def cstringio_buf(self): + """A cStringIO buffer that contains the current chunk we are reading.""" + pass + + def cstringio_refill(self, partialread, reqlen): + """Refills cstringio_buf. + + Returns the currently used buffer (which can but need not be the same as + the old cstringio_buf). partialread is what the C code has read from the + buffer, and should be inserted into the buffer before any more reads. The + return value must be a new, not borrowed reference. Something along the + lines of self._buf should be fine. + + If reqlen bytes can't be read, throw EOFError. + """ + pass + +class TServerTransportBase: + + """Base class for Thrift server transports.""" + + def listen(self): + pass + + def accept(self): + pass + + def close(self): + pass + +class TTransportFactoryBase: + + """Base class for a Transport Factory""" + + def getTransport(self, trans): + return trans + +class TBufferedTransportFactory: + + """Factory transport that builds buffered transports""" + + def getTransport(self, trans): + buffered = TBufferedTransport(trans) + return buffered + + +class TBufferedTransport(TTransportBase,CReadableTransport): + + """Class that wraps another transport and buffers its I/O. + + The implementation uses a (configurable) fixed-size read buffer + but buffers all writes until a flush is performed. + """ + + DEFAULT_BUFFER = 4096 + + def __init__(self, trans, rbuf_size = DEFAULT_BUFFER): + self.__trans = trans + self.__wbuf = StringIO() + self.__rbuf = StringIO("") + self.__rbuf_size = rbuf_size + + def isOpen(self): + return self.__trans.isOpen() + + def open(self): + return self.__trans.open() + + def close(self): + return self.__trans.close() + + def read(self, sz): + ret = self.__rbuf.read(sz) + if len(ret) != 0: + return ret + + self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size))) + return self.__rbuf.read(sz) + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + out = self.__wbuf.getvalue() + # reset wbuf before write/flush to preserve state on underlying failure + self.__wbuf = StringIO() + self.__trans.write(out) + self.__trans.flush() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self.__rbuf + + def cstringio_refill(self, partialread, reqlen): + retstring = partialread + if reqlen < self.__rbuf_size: + # try to make a read of as much as we can. + retstring += self.__trans.read(self.__rbuf_size) + + # but make sure we do read reqlen bytes. + if len(retstring) < reqlen: + retstring += self.__trans.readAll(reqlen - len(retstring)) + + self.__rbuf = StringIO(retstring) + return self.__rbuf + +class TMemoryBuffer(TTransportBase, CReadableTransport): + """Wraps a cStringIO object as a TTransport. + + NOTE: Unlike the C++ version of this class, you cannot write to it + then immediately read from it. If you want to read from a + TMemoryBuffer, you must either pass a string to the constructor. + TODO(dreiss): Make this work like the C++ version. + """ + + def __init__(self, value=None): + """value -- a value to read from for stringio + + If value is set, this will be a transport for reading, + otherwise, it is for writing""" + if value is not None: + self._buffer = StringIO(value) + else: + self._buffer = StringIO() + + def isOpen(self): + return not self._buffer.closed + + def open(self): + pass + + def close(self): + self._buffer.close() + + def read(self, sz): + return self._buffer.read(sz) + + def write(self, buf): + self._buffer.write(buf) + + def flush(self): + pass + + def getvalue(self): + return self._buffer.getvalue() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self._buffer + + def cstringio_refill(self, partialread, reqlen): + # only one shot at reading... + raise EOFError() + +class TFramedTransportFactory: + + """Factory transport that builds framed transports""" + + def getTransport(self, trans): + framed = TFramedTransport(trans) + return framed + + +class TFramedTransport(TTransportBase, CReadableTransport): + + """Class that wraps another transport and frames its I/O when writing.""" + + def __init__(self, trans,): + self.__trans = trans + self.__rbuf = StringIO() + self.__wbuf = StringIO() + + def isOpen(self): + return self.__trans.isOpen() + + def open(self): + return self.__trans.open() + + def close(self): + return self.__trans.close() + + def read(self, sz): + ret = self.__rbuf.read(sz) + if len(ret) != 0: + return ret + + self.readFrame() + return self.__rbuf.read(sz) + + def readFrame(self): + buff = self.__trans.readAll(4) + sz, = unpack('!i', buff) + self.__rbuf = StringIO(self.__trans.readAll(sz)) + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + wout = self.__wbuf.getvalue() + wsz = len(wout) + # reset wbuf before write/flush to preserve state on underlying failure + self.__wbuf = StringIO() + # N.B.: Doing this string concatenation is WAY cheaper than making + # two separate calls to the underlying socket object. Socket writes in + # Python turn out to be REALLY expensive, but it seems to do a pretty + # good job of managing string buffer operations without excessive copies + buf = pack("!i", wsz) + wout + self.__trans.write(buf) + self.__trans.flush() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self.__rbuf + + def cstringio_refill(self, prefix, reqlen): + # self.__rbuf will already be empty here because fastbinary doesn't + # ask for a refill until the previous buffer is empty. Therefore, + # we can start reading new frames immediately. + while len(prefix) < reqlen: + self.readFrame() + prefix += self.__rbuf.getvalue() + self.__rbuf = StringIO(prefix) + return self.__rbuf + + +class TFileObjectTransport(TTransportBase): + """Wraps a file-like object to make it work as a Thrift transport.""" + + def __init__(self, fileobj): + self.fileobj = fileobj + + def isOpen(self): + return True + + def close(self): + self.fileobj.close() + + def read(self, sz): + return self.fileobj.read(sz) + + def write(self, buf): + self.fileobj.write(buf) + + def flush(self): + self.fileobj.flush() diff --git a/backoffice/thrift/transport/TTwisted.py b/backoffice/thrift/transport/TTwisted.py new file mode 100644 index 0000000..b6dcb4e --- /dev/null +++ b/backoffice/thrift/transport/TTwisted.py @@ -0,0 +1,219 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from zope.interface import implements, Interface, Attribute +from twisted.internet.protocol import Protocol, ServerFactory, ClientFactory, \ + connectionDone +from twisted.internet import defer +from twisted.protocols import basic +from twisted.python import log +from twisted.web import server, resource, http + +from thrift.transport import TTransport +from cStringIO import StringIO + + +class TMessageSenderTransport(TTransport.TTransportBase): + + def __init__(self): + self.__wbuf = StringIO() + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + msg = self.__wbuf.getvalue() + self.__wbuf = StringIO() + self.sendMessage(msg) + + def sendMessage(self, message): + raise NotImplementedError + + +class TCallbackTransport(TMessageSenderTransport): + + def __init__(self, func): + TMessageSenderTransport.__init__(self) + self.func = func + + def sendMessage(self, message): + self.func(message) + + +class ThriftClientProtocol(basic.Int32StringReceiver): + + MAX_LENGTH = 2 ** 31 - 1 + + def __init__(self, client_class, iprot_factory, oprot_factory=None): + self._client_class = client_class + self._iprot_factory = iprot_factory + if oprot_factory is None: + self._oprot_factory = iprot_factory + else: + self._oprot_factory = oprot_factory + + self.recv_map = {} + self.started = defer.Deferred() + + def dispatch(self, msg): + self.sendString(msg) + + def connectionMade(self): + tmo = TCallbackTransport(self.dispatch) + self.client = self._client_class(tmo, self._oprot_factory) + self.started.callback(self.client) + + def connectionLost(self, reason=connectionDone): + for k,v in self.client._reqs.iteritems(): + tex = TTransport.TTransportException( + type=TTransport.TTransportException.END_OF_FILE, + message='Connection closed') + v.errback(tex) + + def stringReceived(self, frame): + tr = TTransport.TMemoryBuffer(frame) + iprot = self._iprot_factory.getProtocol(tr) + (fname, mtype, rseqid) = iprot.readMessageBegin() + + try: + method = self.recv_map[fname] + except KeyError: + method = getattr(self.client, 'recv_' + fname) + self.recv_map[fname] = method + + method(iprot, mtype, rseqid) + + +class ThriftServerProtocol(basic.Int32StringReceiver): + + MAX_LENGTH = 2 ** 31 - 1 + + def dispatch(self, msg): + self.sendString(msg) + + def processError(self, error): + self.transport.loseConnection() + + def processOk(self, _, tmo): + msg = tmo.getvalue() + + if len(msg) > 0: + self.dispatch(msg) + + def stringReceived(self, frame): + tmi = TTransport.TMemoryBuffer(frame) + tmo = TTransport.TMemoryBuffer() + + iprot = self.factory.iprot_factory.getProtocol(tmi) + oprot = self.factory.oprot_factory.getProtocol(tmo) + + d = self.factory.processor.process(iprot, oprot) + d.addCallbacks(self.processOk, self.processError, + callbackArgs=(tmo,)) + + +class IThriftServerFactory(Interface): + + processor = Attribute("Thrift processor") + + iprot_factory = Attribute("Input protocol factory") + + oprot_factory = Attribute("Output protocol factory") + + +class IThriftClientFactory(Interface): + + client_class = Attribute("Thrift client class") + + iprot_factory = Attribute("Input protocol factory") + + oprot_factory = Attribute("Output protocol factory") + + +class ThriftServerFactory(ServerFactory): + + implements(IThriftServerFactory) + + protocol = ThriftServerProtocol + + def __init__(self, processor, iprot_factory, oprot_factory=None): + self.processor = processor + self.iprot_factory = iprot_factory + if oprot_factory is None: + self.oprot_factory = iprot_factory + else: + self.oprot_factory = oprot_factory + + +class ThriftClientFactory(ClientFactory): + + implements(IThriftClientFactory) + + protocol = ThriftClientProtocol + + def __init__(self, client_class, iprot_factory, oprot_factory=None): + self.client_class = client_class + self.iprot_factory = iprot_factory + if oprot_factory is None: + self.oprot_factory = iprot_factory + else: + self.oprot_factory = oprot_factory + + def buildProtocol(self, addr): + p = self.protocol(self.client_class, self.iprot_factory, + self.oprot_factory) + p.factory = self + return p + + +class ThriftResource(resource.Resource): + + allowedMethods = ('POST',) + + def __init__(self, processor, inputProtocolFactory, + outputProtocolFactory=None): + resource.Resource.__init__(self) + self.inputProtocolFactory = inputProtocolFactory + if outputProtocolFactory is None: + self.outputProtocolFactory = inputProtocolFactory + else: + self.outputProtocolFactory = outputProtocolFactory + self.processor = processor + + def getChild(self, path, request): + return self + + def _cbProcess(self, _, request, tmo): + msg = tmo.getvalue() + request.setResponseCode(http.OK) + request.setHeader("content-type", "application/x-thrift") + request.write(msg) + request.finish() + + def render_POST(self, request): + request.content.seek(0, 0) + data = request.content.read() + tmi = TTransport.TMemoryBuffer(data) + tmo = TTransport.TMemoryBuffer() + + iprot = self.inputProtocolFactory.getProtocol(tmi) + oprot = self.outputProtocolFactory.getProtocol(tmo) + + d = self.processor.process(iprot, oprot) + d.addCallback(self._cbProcess, request, tmo) + return server.NOT_DONE_YET diff --git a/backoffice/thrift/transport/__init__.py b/backoffice/thrift/transport/__init__.py new file mode 100644 index 0000000..02c6048 --- /dev/null +++ b/backoffice/thrift/transport/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TTransport', 'TSocket', 'THttpClient'] diff --git a/backoffice/urls.py b/backoffice/urls.py new file mode 100644 index 0000000..c5142ce --- /dev/null +++ b/backoffice/urls.py @@ -0,0 +1,35 @@ +from django.conf.urls.defaults import * #@UnusedWildImport + +# Uncomment the next two lines to enable the admin: +from django.contrib import admin +admin.autodiscover() + +urlpatterns = patterns('', + url(r'^_static/(?P.*)$', 'django.views.static.serve', {'document_root': 'static'}, name='static'), + url(r'^$', 'backoffice.views.home', name='home'), + + url(r'^beta_invitation_export', 'backoffice.views.beta_invitation_export', name='beta_invitation_export'), + url(r'^log_info', 'backoffice.views.log_info', name='log_info'), + url(r'^control_panel$', 'backoffice.views.control_panel', name='control_panel'), + url(r'^biz_stats$', 'backoffice.views.biz_stats', name='biz_stats'), + url(r'^biz_stats.json$', 'backoffice.views.biz_stats_json', name='biz_stats_json'), + url(r'^biz_stats.json.daily$', 'backoffice.views.biz_stats_json_daily', name='biz_stats_json_daily'), + url(r'^accounts_info', 'backoffice.views.accounts_info', name='accounts_info'), + url(r'^resource_map', 'backoffice.views.resource_map', name='resource_map'), + url(r'^beta_requests$', 'backoffice.views.beta_requests_list', name='beta_requests'), + url(r'^beta_invitations$', 'backoffice.views.beta_invitations', name='beta_invitations'), + url(r'^beta_request_invite/(?P.*)$', 'backoffice.views.beta_request_invite', name='beta_request_invite'), + url(r'^worker_resource_map/(?P.*)$', 'backoffice.views.worker_resource_map', name='worker_resource_map'), + + + url(r'^manage_worker/(?P.*)$', 'backoffice.views.manage_worker', name='manage_worker'), + url(r'^mount_history/(?P[^/]*)/(?P.*)$', 'backoffice.views.mount_history', name='mount_history'), + url(r'^index_history/(?P[^/]*)/(?P.*)$', 'backoffice.views.index_history', name='index_history'), + url(r'^load_history/(?P[^/]*)', 'backoffice.views.load_history', name='load_history'), + + url(r'^operations', 'backoffice.views.operations', name='operations'), + + + url(r'^login$', 'backoffice.views.login', name='login'), + url(r'^logout$', 'backoffice.views.logout', name='logout'), +) diff --git a/backoffice/util.py b/backoffice/util.py new file mode 100644 index 0000000..c37f1cc --- /dev/null +++ b/backoffice/util.py @@ -0,0 +1,90 @@ +from django import shortcuts +from django.utils.translation import check_for_language, activate +from django.contrib.auth.decorators import login_required as dj_login_required +from django.http import HttpResponseRedirect, HttpResponseForbidden +from django.contrib import auth +from django.contrib.auth.models import User +from django.utils.http import urlquote +import socket +import base64 +import re +from django.template import Context as _Context +from django.conf import settings +#from ncrypt.cipher import CipherType, EncryptCipher, DecryptCipher +import binascii +import random +import hashlib +from lib import encoder +from django.core.urlresolvers import reverse + +extra_context = {} +def Context(context, request): + global extra_context + context['request'] = request + context['user'] = request.user + for k in extra_context: + if hasattr(extra_context[k], '__call__'): + context[k] = extra_context[k]() + else: + context[k] = extra_context[k] + + return _Context(context) + +MOBILE_PATTERN_1 = '/android|avantgo|blackberry|blazer|compal|elaine|fennec|hiptop|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile|o2|opera mini|palm( os)?|plucker|pocket|pre\/|psp|smartphone|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce; (iemobile|ppc)|xiino/i' +MOBILE_PATTERN_2 = '/1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|e\-|e\/|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(di|rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|xda(\-|2|g)|yas\-|your|zeto|zte\-/i' + +def domain(): + if settings.WEBAPP_PORT == 80: + return settings.COMMON_DOMAIN + else: + return "%s:%d" % (settings.COMMON_DOMAIN, settings.WEBAPP_PORT) + +def is_internal_navigation(request): + ref = request.META.get('HTTP_REFERER') + if ref: + pattern = r'http://[^/]*\.' + domain().replace('.', r'\.') + if re.match(pattern, ref): + return True + else: + return False + else: + return False + +def is_mobile(request): + user_agent = request.META['HTTP_USER_AGENT'].lower() + match1 = re.search(MOBILE_PATTERN_1, user_agent) + match2 = re.search(MOBILE_PATTERN_2, user_agent[:4]) + return match1 or match2 + +def render_to_response(template, context, *args, **kwargs): + original_template = template + if is_mobile(context['request']): + parts = template.split('/') + parts[-1] = 'mobile.' + parts[-1] + template = '/'.join(parts) + return shortcuts.render_to_response((template, original_template), context, *args, **kwargs) + +def login_required(view, *args, **kwargs): + dj_view = dj_login_required(view, *args, **kwargs) + def decorated(request, *args, **kwargs): +# if request.method == 'POST' and not request.user.is_authenticated(): +# context = {} +# context['POST'] = simplejson.dumps(request.POST) +# context['url'] = urlquote(request.get_full_path()) +# return render_to_response('post_login_redirect.html', Context(context, request)) + return dj_view(request, *args, **kwargs) + return decorated + +def staff_required(view): + def decorated(request, *args, **kwargs): + if not request.user or not request.user.is_staff: + return HttpResponseRedirect(reverse("login")) + return view(request, *args, **kwargs) + + return decorated + +def get_index_code(id): + return encoder.to_key(id) + +def get_index_id_for_code(code): + return encoder.from_key(code); \ No newline at end of file diff --git a/backoffice/views.py b/backoffice/views.py new file mode 100644 index 0000000..eb05132 --- /dev/null +++ b/backoffice/views.py @@ -0,0 +1,525 @@ +from django.conf import settings +from django.contrib import auth +from django.contrib.auth.models import User +from django.core.urlresolvers import reverse +from django.http import HttpResponseRedirect, Http404, HttpResponseNotFound, \ + HttpResponse, HttpResponseForbidden +from django.shortcuts import render_to_response +from models import PFUser, Worker, WorkerMountInfo, WorkerIndexInfo, \ + WorkerLoadInfo, Index, BetaTestRequest, BetaInvitation, generate_onetimepass, \ + Package, Account, Deploy, IndexConfiguration +from util import Context, login_required, staff_required +import datetime +import forms +import hashlib +import os +import re +import subprocess +import time +import urllib +import urllib2 +import rpc +import json + +class JsonResponse(HttpResponse): + def __init__(self, json_object, *args, **kwargs): + body = json.dumps(json_object) + if 'callback' in kwargs: + callback = kwargs.pop('callback') + if callback: + body = '%s(%s)' % (callback, body) + super(JsonResponse, self).__init__(body, *args, **kwargs) + +def login(request): + login_form = forms.LoginForm() + login_message = '' + if request.method == 'POST': + login_form = forms.LoginForm(data=request.POST) + if login_form.is_valid(): + try: + username = PFUser.objects.get(email=login_form.cleaned_data['email']).user.username + user = auth.authenticate(username=username, password=login_form.cleaned_data['password']) + if user is not None: + if user.is_active: + auth.login(request, user) + return HttpResponseRedirect(request.GET.get('next') or '/'); + else: + login_message = 'Account disabled' + else: + login_message = 'Wrong email or password' + except PFUser.DoesNotExist: #@UndefinedVariable + login_message = 'Wrong email or password' + + context = { + 'login_form': login_form, + 'login_message': login_message, + 'navigation_pos': 'home', + 'next': request.GET.get('next') or '/', + } + + return render_to_response('login.html', Context(context, request)) + +def logout(request): + auth.logout(request) + return HttpResponseRedirect('/') # request.GET['next']); + +@staff_required +@login_required +def home(request): + return HttpResponseRedirect(reverse("operations")) + +@login_required +def biz_stats(request): + context = {} + return render_to_response('biz_stats.html', Context(context, request)) + +@login_required +def biz_stats_json(request): + json_string = '' + with open('/data/logs/stats.json') as f: + json_string = f.read() + return HttpResponse(json_string) + +@login_required +def biz_stats_json_daily(request): + json_string_daily = '' + with open('/data/logs/stats.json.daily') as f: + json_string_daily = f.read() + return HttpResponse(json_string_daily) + +@staff_required +@login_required +def control_panel(request): + workers = Worker.objects.all() + + context = { + 'workers': workers, + 'navigation_pos': 'home', + } + + return render_to_response('control_panel.html', Context(context, request)) + +@staff_required +@login_required +def manage_worker(request, worker_id=None): + context = {} + + worker = Worker.objects.get(id=worker_id) + + mount_infos = WorkerMountInfo.objects.extra(where=['worker_id=%d and timestamp=(select max(timestamp) from %s where worker_id=%d)' % (int(worker_id), WorkerMountInfo._meta.db_table, int(worker_id))], order_by=['-used']) + indexes_infos = WorkerIndexInfo.objects.extra(where=['worker_id=%d and timestamp=(select max(timestamp) from %s where worker_id=%d)' % (int(worker_id), WorkerIndexInfo._meta.db_table, int(worker_id))], order_by=['-used_mem']) + load_info = WorkerLoadInfo.objects.extra(where=['worker_id=%d and timestamp=(select max(timestamp) from %s where worker_id=%d)' % (int(worker_id), WorkerLoadInfo._meta.db_table, int(worker_id))])[0] + + used_disk_total = 0 + used_mem_total = 0 + + mount_percentages = {} + + for info in mount_infos: + mount_percentages[info.mount] = (float)(info.used) / (info.used + info.available) * 100 + + for info in indexes_infos: + used_disk_total += info.used_disk + used_mem_total += info.used_mem + + context = { + 'mount_infos': mount_infos, + 'mount_percentages': mount_percentages, + 'indexes_infos': indexes_infos, + 'load_info': load_info, + 'used_disk_total': used_disk_total, + 'used_mem_total': used_mem_total, + 'worker': worker, + 'navigation_pos': 'home', + } + + return render_to_response('manage_worker.html', Context(context, request)) + +@staff_required +@login_required +def mount_history(request, worker_id=None, mount=None): + worker = Worker.objects.get(id=worker_id) + + if not mount.startswith('/'): + mount = '/' + mount + + mount_infos = WorkerMountInfo.objects.extra(where=['worker_id=%d and mount="%s"' % (int(worker_id), mount)], order_by=['timestamp']) + mount_percentages = {} + + for info in mount_infos: + mount_percentages[info.timestamp] = (float)(info.used) / (info.used + info.available) * 100 + + + context = { + 'worker': worker, + 'mount': mount, + 'mount_percentages': mount_percentages, + 'mount_infos': mount_infos, + } + + return render_to_response('mount_history.html', Context(context, request)) + +@staff_required +@login_required +def index_history(request, worker_id=None, index_id=None): + index = Index.objects.get(id=index_id) + worker = Worker.objects.get(id=worker_id) + + index_infos = WorkerIndexInfo.objects.filter(worker__id=worker_id, deploy__index__id=index_id) + + context = { + 'index': index, + 'worker': worker, + 'index_infos': index_infos, + } + + return render_to_response('index_history.html', Context(context, request)) + + +@staff_required +@login_required +def beta_requests_list(request): + requests = BetaTestRequest.objects.all().order_by('-request_date') + + context = { + 'requests': requests, + } + + return render_to_response('beta_requests.html', Context(context, request)) + +@staff_required +@login_required +def beta_invitation_export(request): + requests = BetaTestRequest.objects.all().order_by('-request_date') + invitations = BetaInvitation.objects.filter(beta_requester__isnull=True).order_by('-invitation_date') + + context = { + 'requests': requests, + 'invitations': invitations, + } + + return render_to_response('beta_invitation_export.html', Context(context, request)) + +FORCED_PACKAGE_CODE = 'BETA' + +@staff_required +@login_required +def beta_request_invite(request, request_id=None): + beta_request = BetaTestRequest.objects.get(id=request_id) + new_invitation = BetaInvitation() + new_invitation.beta_requester = beta_request + new_invitation.forced_package = Package.objects.get(code=FORCED_PACKAGE_CODE) + + new_invitation.save() + new_invitation.password = generate_onetimepass(new_invitation.id) + new_invitation.save() + + return HttpResponseRedirect(reverse('beta_requests')); + +@staff_required +@login_required +def beta_invitations(request): + if request.method == 'POST': + form = forms.InvitationForm(data=request.POST) + if form.is_valid(): + new_invitation = BetaInvitation() + new_invitation.assigned_customer = form.cleaned_data['requesting_customer'] + new_invitation.forced_package = Package.objects.get(code=FORCED_PACKAGE_CODE) + + new_invitation.save() + new_invitation.password = generate_onetimepass(new_invitation.id) + new_invitation.save() + return HttpResponseRedirect(reverse('beta_invitations')); + else: + form = forms.InvitationForm() + + invitations = BetaInvitation.objects.all().order_by('-invitation_date') + + context = { + 'invitations': invitations, + 'form': form, + } + + return render_to_response('beta_invitations.html', Context(context, request)) + +@staff_required +@login_required +def load_history(request, worker_id=None): + worker = Worker.objects.get(id=worker_id) + + load_infos = WorkerLoadInfo.objects.extra(where=['worker_id=%d' % (int(worker_id))], order_by=['timestamp']) + + context = { + 'worker': worker, + 'load_infos': load_infos, + } + return render_to_response('load_history.html', Context(context, request)) + + +@staff_required +@login_required +def log_info(request): + class ddict(dict): + def __init__(self, default): + self.default = default + def __getitem__(self, key): + return self.setdefault(key, self.default()) + ndict = lambda: ddict(lambda: 0) + + ps = subprocess.Popen('tail -1000000 /data/logs/api.log | grep "Search:INFO"', shell=True, stdout=subprocess.PIPE) + + sums = ddict(ndict) + sucs = ddict(ndict) + cnts = ddict(ndict) + + for l in ps.stdout: + m = re.match(r'.*(\d{2}/\d{2}-\d{2}.\d)\d.*\[(\d{3})] in (\d+\.\d{3})s for \[(.*) /v1/indexes/([^\]]*)/([^\]]*)\]', l) + if m: + dt = m.group(1) + code = m.group(2) + time = float(m.group(3)) + act = m.group(4) + cod = m.group(5) + met = m.group(6) + req = act +' ' + met + sums[req][dt] += time + cnts[req][dt] += 1 + if code == '200': + sucs[req][dt] += 1 + count = 1 + rows_by_req = {} + for req,v in sums.iteritems(): + rows = [] + rows_by_req[req] = rows + count += 1 + for dt,time in v.iteritems(): + y = 2010 + m = int(dt[3:5]) - 1 + d = int(dt[0:2]) + h = int(dt[6:8]) + i = int(dt[9:10])*10 + avg = time / cnts[req][dt] + rows.append('[new Date(%d, %d, %d, %d, %d), %f]' % (y,m,d,h,i,avg)) + context = { 'rows_by_req': rows_by_req } + return render_to_response('log_info.html', Context(context, request)) + +@staff_required +@login_required +def accounts_info(request): + context = { 'accounts': Account.objects.all().order_by('package', 'creation_time') } + return render_to_response('accounts_info.html', Context(context, request)) + +def _size(deploy): + try: + return deploy.index.current_docs_number + except Index.DoesNotExist: + return 0 + +@staff_required +@login_required +def resource_map(request): + if request.method == 'POST': + if request.POST['task'] == 'redeploy': + id = request.POST['index_id'] + rpc.get_deploy_manager().redeploy_index(Index.objects.get(pk=id).code) + return HttpResponseRedirect('/resource_map') + workers = [w for w in Worker.objects.select_related(depth=5).order_by('id').all()] + for w in workers: + w.sorted_deploys = sorted(w.deploys.all(), key=_size, reverse=True) + w.used = w.get_used_ram() + + context = { + 'workers': workers, + 'packages': Package.objects.all() + } + + return render_to_response('resource_map.html', Context(context, request)) + +def deploy_dict(d): + return dict( + id=d.id, + index=d.index_id, + worker=d.worker_id, + base_port=d.base_port, + status=d.status, + timestamp=time.mktime(d.timestamp.timetuple())*1000 if d.timestamp else None, + parent=d.parent_id, + effective_xmx=d.effective_xmx, + effective_bdb=d.effective_bdb + ) + +def index_dict(i): + return dict( + id=i.id, + account=i.account_id, + code=i.code, + name=i.name, + creation_time=time.mktime(i.creation_time.timetuple())*1000 if i.creation_time else None, + analyzer_config=i.analyzer_config, + configuration=i.configuration_id, + public_api=i.public_api, + docs=i.current_docs_number, + status=i.status + ) + +def account_dict(a): + return dict( + id=a.id, + package=a.package_id, + code=a.get_public_apikey(), + private_url=a.get_private_apiurl(), + public_url=a.get_public_apiurl(), + creation_time=time.mktime(a.creation_time.timetuple())*1000 if a.creation_time else None, + #default_analyzer_config=a.default_analyzer, + configuration=a.configuration_id, + status=a.status, + email=a.user.email if a.user else None, + ) + +def configuration_dict(c): + return dict( + id=c.id, + description=c.description, + creation_date=time.mktime(c.creation_date.timetuple())*1000 if c.creation_date else None, + data=c.get_data() + ) + +def package_dict(p): + return dict( + id=p.id, + name=p.name, + code=p.code, + price=p.base_price, + docs=p.index_max_size, + indexes=p.max_indexes, + configuration=p.configuration.id + ) + +def worker_dict(w): + return dict( + id=w.id, + wan_dns=w.wan_dns, + lan_dns=w.lan_dns, + name=w.instance_name, + status=w.status, + ram=w.ram + ) + +@staff_required +@login_required +def operations(request): + #if request.method == 'POST': + # if request.POST['task'] == 'redeploy': + # id = request.POST['index_id'] + # rpc.get_deploy_manager().redeploy_index(Index.objects.get(pk=id).code) + # return HttpResponseRedirect('/resource_map') + + level = request.GET.get('level', 'top') + if level == 'top': + return render_to_response('operations/index.html', Context({}, request)) + elif level == 'refresh': + data = { + 'Config': map(configuration_dict, IndexConfiguration.objects.all()), + 'Account': map(account_dict, Account.objects.select_related('user').all()), + 'Deploy': map(deploy_dict, Deploy.objects.all()), + 'Index': map(index_dict, Index.objects.all()), + 'Package': map(package_dict, Package.objects.all()), + 'Worker': map(worker_dict, Worker.objects.all()), + } + return JsonResponse(data) + elif level == 'index': + id = request.GET.get('id') + index = Index.objects.get(pk=id); + data = { + 'Index': index_dict(index), + 'Deploy': map(deploy_dict, index.deploys.all()), + } + return JsonResponse(data) + elif level == 'stats': + id = request.GET.get('id') + d = Deploy.objects.get(pk=id) + client = rpc.getThriftIndexerClient(d.worker.lan_dns, int(d.base_port), 3000) + return JsonResponse(client.get_stats()) + elif level == 'log': + id = request.GET.get('id') + file = request.GET.get('file') + d = Deploy.objects.get(pk=id) + client = rpc.get_worker_controller(d.worker, 4000) + lines = client.tail(file, 300, d.index.code, d.base_port) + return JsonResponse(lines) + elif level == 'redeploy': + id = request.GET.get('id') + rpc.get_deploy_manager().redeploy_index(Index.objects.get(pk=id).code) + return HttpResponse() + elif level == 'decommission': + id = request.GET.get('id') + Worker.objects.filter(id=id).update(status=Worker.States.decommissioning) + return JsonResponse(worker_dict(Worker.objects.get(id=id))) + elif level == 'delete_worker': + id = request.GET.get('id') + w = Worker.objects.get(id=id) + if w.status != Worker.States.decommissioning: + return HttpResponse('worker not decommissioning', status=409) + if w.deploys.count(): + return HttpResponse('worker not empty', status=409) + w.delete() + return HttpResponse() + elif level == 'delete_account': + id = request.GET.get('id') + a = Account.objects.get(id=id) + user = a.user.user + if a.indexes.count(): + return HttpResponse('account has index', status=409) + if a.payment_informations.count(): + return HttpResponse('account has payment information', status=409) + user = a.user.user + a.delete() + user.delete() + return HttpResponse() + elif level == 'account_set_pkg': + id = request.GET.get('id') + pid = request.GET.get('pkg') + p = Package.objects.get(id=pid) + updated = Account.objects.filter(id=id).update(package=p) + if updated: + return JsonResponse(account_dict(Account.objects.get(id=id))) + else: + return HttpResponse('account not found', status=409) + elif level == 'account_set_cfg': + id = request.GET.get('id') + cid = request.GET.get('cfg') + c = IndexConfiguration.objects.get(id=cid) + updated = Account.objects.filter(id=id).update(configuration=c) + if updated: + return JsonResponse(account_dict(Account.objects.get(id=id))) + else: + return HttpResponse('account not found', status=409) + elif level == 'index_set_cfg': + id = request.GET.get('id') + cid = request.GET.get('cfg') + c = IndexConfiguration.objects.get(id=cid) + updated = Index.objects.filter(id=id).update(configuration=c) + if updated: + return JsonResponse(index_dict(Index.objects.get(id=id))) + else: + return HttpResponse('index not found', status=409) + return HttpResponseNotFound() + +@staff_required +@login_required +def worker_resource_map(request, worker_id): + w = Worker.objects.get(id=int(worker_id)) + w.sorted_deploys = sorted(w.deploys.all(), key=_size, reverse=True) + xmx = 0 + for d in w.sorted_deploys: + xmx += d.effective_xmx + w.xmx = xmx + w.used = w.get_used_ram() + + + context = { + 'worker' : w + } + + return render_to_response('worker_resource_map.html', Context(context,request)) + diff --git a/demo_index/freebase_dataset_generator.py b/demo_index/freebase_dataset_generator.py new file mode 100644 index 0000000..3a96cff --- /dev/null +++ b/demo_index/freebase_dataset_generator.py @@ -0,0 +1,53 @@ +#!/usr/bin/python +import freebase +import pickle + +query = [{ "/music/instrument/family": [{"name": None}], "id": None, "name": None, "a:type": "/music/instrument", "b:type": "/common/topic", "/common/topic/article": {"id": None}, "/common/topic/image": [{"id": None}], "/music/instrument/instrumentalists": {"return": "count", "optional": True}}] + +data={} +counter = 0 +for row in freebase.mqlreaditer(query): + datum = {} + datum['name'] = row['name'] + datum['url'] = 'http://freebase.com/view%s' % (row['id']) + datum['image'] = 'http://img.freebase.com/api/trans/raw%s' % (row['/common/topic/image'][0]['id']) + datum['thumbnail'] = 'http://indextank.com/_static/common/demo/%s.jpg' % (row['/common/topic/image'][0]['id'].split('/')[2]) + datum['text'] = freebase.blurb(row['/common/topic/article']['id'], maxlength=500) + datum['variables'] = {} + datum['variables'][0] = row['/music/instrument/instrumentalists'] + families = [] + for f in row['/music/instrument/family']: + families.append(f['name']) + datum['families'] = families + + data[datum['name']] = datum + print('done %i' % counter) + counter+=1 + + +family = {} +for datum in data.values(): + for f in datum['families']: + if family.has_key(f): + family[f] += 1 + else: + family[f] = 1 + + +for datum in data.values(): + datum['categories'] = {} + t = datum['families'] + if t: + t.sort(key=family.get, reverse=True) + datum['categories']['family'] = t[0] + del(datum['families']) + +results = [] +for datum in data.values(): + t = {} + t['docid'] = datum['name'] + t['fields'] = {'name': datum['name'], 'url': datum['url'], 'text': datum['text'], 'image':datum['image'], 'thumbnail': datum['thumbnail']} + t['categories'] = datum['categories'] + t['variables'] = datum['variables'] + results.append(t) +pickle.dump(results, file('instruments.dat', 'w')) diff --git a/demo_index/instruments.dat b/demo_index/instruments.dat new file mode 100644 index 0000000..0966716 --- /dev/null +++ b/demo_index/instruments.dat @@ -0,0 +1,12772 @@ +(lp0 +(dp1 +S'fields' +p2 +(dp3 +S'url' +p4 +Vhttp://freebase.com/view/en/e-flat_clarinet +p5 +sS'text' +p6 +S'The E-flat clarinet is a member of the clarinet family. It is usually classed as a soprano clarinet, although some authors describe it as a "sopranino" or even "piccolo" clarinet. Smaller in size and higher in pitch than the more common B\xe2\x99\xad clarinet, it is a transposing instrument in E\xe2\x99\xad, sounding a minor third higher than written. In Italian it sometimes referred to as a quartino, generally appearing in scores as quartino in Mi\xe2\x99\xad.\nThe E\xe2\x99\xad clarinet is used in orchestras, concert bands, marching...' +p7 +sS'image' +p8 +Vhttp://img.freebase.com/api/trans/raw/m/02cy_r6 +p9 +sS'name' +p10 +VE-flat clarinet +p11 +sS'thumbnail' +p12 +Vhttp://indextank.com/_static/common/demo/02cy_r6.jpg +p13 +ssS'variables' +p14 +(dp15 +I0 +I0 +ssS'docid' +p16 +g11 +sS'categories' +p17 +(dp18 +S'family' +p19 +VClarinet +p20 +ssa(dp21 +g2 +(dp22 +g4 +Vhttp://freebase.com/view/en/swedish_bagpipes +p23 +sg6 +S'Swedish bagpipes (Swedish: Svensk s\xc3\xa4ckpipa) are a variety of bagpipes from Sweden. The term itself generically translates to "bagpipes" in Swedish, but is used in English to describe the specifically Swedish bagpipe from the Dalarna region.\nMedieval paintings in churches suggest that the instrument was spread all over Sweden. The instrument was practically extinct by the middle of the 20th century; the instrument that today is referred to as Swedish bagpipes is a construction based on...' +p24 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029mx2d +p25 +sg10 +VSwedish bagpipes +p26 +sg12 +Vhttp://indextank.com/_static/common/demo/029mx2d.jpg +p27 +ssg14 +(dp28 +I0 +I0 +ssg16 +g26 +sg17 +(dp29 +g19 +VBagpipes +p30 +ssa(dp31 +g2 +(dp32 +g4 +Vhttp://freebase.com/view/en/portuguese_guitar +p33 +sg6 +S'The Portuguese guitar or Portuguese guitarra (Portuguese: guitarra portuguesa) is a plucked string instrument with twelve steel strings, strung in six courses comprising two strings each. It has a distinctive tuning mechanism. It is most notably associated with fado.\nThe origin of the Portuguese guitar is a subject of debate. Throughout the 19th century the Portuguese guitar was being made in several sizes and shapes and subject to several regional aesthetic trends. A sizeable guitar making...' +p34 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02b57zg +p35 +sg10 +VPortuguese guitar +p36 +sg12 +Vhttp://indextank.com/_static/common/demo/02b57zg.jpg +p37 +ssg14 +(dp38 +I0 +I1 +ssg16 +g36 +sg17 +(dp39 +g19 +VPlucked string instrument +p40 +ssa(dp41 +g2 +(dp42 +g4 +Vhttp://freebase.com/view/en/accordion +p43 +sg6 +S'The accordion is a box-shaped musical instrument of the bellows-driven free-reed aerophone family, sometimes referred to as a squeezebox. A person who plays the accordion is called an accordionist.\nIt is played by compressing or expanding a bellows whilst pressing buttons or keys, causing valves, called pallets, to open, which allow air to flow across strips of brass or steel, called reeds, that vibrate to produce sound inside the body.\nThe instrument is sometimes considered a one-man-band...' +p44 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03r6gyw +p45 +sg10 +VAccordion +p46 +sg12 +Vhttp://indextank.com/_static/common/demo/03r6gyw.jpg +p47 +ssg14 +(dp48 +I0 +I233 +ssg16 +g46 +sg17 +(dp49 +g19 +VKeyboard instrument +p50 +ssa(dp51 +g2 +(dp52 +g4 +Vhttp://freebase.com/view/en/tuba +p53 +sg6 +S'The tuba is the largest and lowest pitched brass instrument. Sound is produced by vibrating or "buzzing" the lips into a large cupped mouthpiece. It is one of the most recent additions to the modern symphony orchestra, first appearing in the mid-19th century, when it largely replaced the ophicleide. Tuba is Latin for trumpet or horn. The horn referred to would most likely resemble what is known as a baroque trumpet.\nPrussian Patent No. 19 was granted to Wilhelm Friedrich Wieprecht and Carl...' +p54 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bjtvj +p55 +sg10 +VTuba +p56 +sg12 +Vhttp://indextank.com/_static/common/demo/02bjtvj.jpg +p57 +ssg14 +(dp58 +I0 +I28 +ssg16 +g56 +sg17 +(dp59 +g19 +VBrass instrument +p60 +ssa(dp61 +g2 +(dp62 +g4 +Vhttp://freebase.com/view/en/cuatro +p63 +sg6 +S'The cuatro is any of several Latin American instruments of the guitar or lute family. The cuatro is smaller than a guitar. Cuatro means four in Spanish, although current instruments may have more than four strings.\nAn instrument of the guitar family, found in South America, Trinidad & Tobago and other territories of the West Indies. Its 15th century predecessor was the Portuguese Cavaquinho, which, like the cuatro had four strings. The cuatro is widely used in ensembles in Colombia, Jamaica,...' +p64 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02fpz2k +p65 +sg10 +VCuatro +p66 +sg12 +Vhttp://indextank.com/_static/common/demo/02fpz2k.jpg +p67 +ssg14 +(dp68 +I0 +I9 +ssg16 +g66 +sg17 +(dp69 +g19 +VPlucked string instrument +p70 +ssa(dp71 +g2 +(dp72 +g4 +Vhttp://freebase.com/view/en/gudastviri +p73 +sg6 +S'The gudastviri (Georgian: \xe1\x83\x92\xe1\x83\xa3\xe1\x83\x93\xe1\x83\x90\xe1\x83\xa1\xe1\x83\xa2\xe1\x83\x95\xe1\x83\x98\xe1\x83\xa0\xe1\x83\x98) is a droneless, double-chantered, horn-belled bagpipe played in Georgia. The term comes from the words guda (bag) and stviri (whistling). In some regions, the instrument is called the chiboni, stviri, or tulumi.\nThis type of bagpipe is found in many regions of Georgia, and is known by different names in various areas.\nThese variants differ from one another in timbre, capacity/size of the bag, and number of holes on the two pipes.\nThe gudastviri is made...' +p74 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04pzpv_ +p75 +sg10 +VGudastviri +p76 +sg12 +Vhttp://indextank.com/_static/common/demo/04pzpv_.jpg +p77 +ssg14 +(dp78 +I0 +I0 +ssg16 +g76 +sg17 +(dp79 +g19 +VBagpipes +p80 +ssa(dp81 +g2 +(dp82 +g4 +Vhttp://freebase.com/view/en/drum_kit +p83 +sg6 +S'A drum kit (also drum set, or trap set) is a collection of drums, cymbals and often other percussion instruments, such as cowbells, wood blocks, triangles, chimes, or tambourines, arranged for convenient playing by a single person (drummer).\nThe individual instruments of a drum-set are hit by a variety of implements held in the hand, including sticks, brushes, and mallets. Two notable exceptions include the bass drum, played by a foot-operated pedal, and the hi-hat cymbals, which may be...' +p84 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bdbkf +p85 +sg10 +VDrum kit +p86 +sg12 +Vhttp://indextank.com/_static/common/demo/02bdbkf.jpg +p87 +ssg14 +(dp88 +I0 +I1709 +ssg16 +g86 +sg17 +(dp89 +g19 +VPercussion +p90 +ssa(dp91 +g2 +(dp92 +g4 +Vhttp://freebase.com/view/en/viol +p93 +sg6 +S'The viol (also known as the Viola da gamba) is any one of a family of bowed, fretted and stringed musical instruments developed in the mid-late 15th century and used primarily in the Renaissance and Baroque periods. The family is related to and descends primarily from the Renaissance vihuela, a plucked instrument that preceded the guitar. An influence in the playing posture has been credited to the example of Moorish rabab players.\nViols are different in several respects from instruments of...' +p94 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03slmhd +p95 +sg10 +VViol +p96 +sg12 +Vhttp://indextank.com/_static/common/demo/03slmhd.jpg +p97 +ssg14 +(dp98 +I0 +I8 +ssg16 +g96 +sg17 +(dp99 +g19 +VBowed string instruments +p100 +ssa(dp101 +g2 +(dp102 +g4 +Vhttp://freebase.com/view/en/tiple +p103 +sg6 +S'Tiple (pronounced as\xc2\xa0:tee-pleh) is the Spanish word for treble or soprano, is often applied to specific instruments, generally to refer to a small chordophone of the guitar family.\nThe tiple is the smallest of the three string instruments of Puerto Rico that make up the orquesta jibara (i.e., the Cuatro, the Tiple and the Bordonua). According to investigations made by Jose Reyes Zamora, the tiple in Puerto Rico dates back to the 18th century. It is believed to have evolved from the Spanish...' +p104 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/042jl6k +p105 +sg10 +VTiple +p106 +sg12 +Vhttp://indextank.com/_static/common/demo/042jl6k.jpg +p107 +ssg14 +(dp108 +I0 +I0 +ssg16 +g106 +sg17 +(dp109 +g19 +VPlucked string instrument +p110 +ssa(dp111 +g2 +(dp112 +g4 +Vhttp://freebase.com/view/en/askomandoura +p113 +sg6 +S'The askomandoura (Greek: \xce\xb1\xcf\x83\xce\xba\xce\xbf\xce\xbc\xce\xb1\xce\xbd\xcf\x84\xce\xbf\xcf\x8d\xcf\x81\xce\xb1) is a type of bagpipe played as a traditional instrument on the Greek island of Crete, similar to the tsampouna.\nIts use in Crete is attested in illustrations from the mid-15th Century.' +p114 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0ccpq4r +p115 +sg10 +VAskomandoura +p116 +sg12 +Vhttp://indextank.com/_static/common/demo/0ccpq4r.jpg +p117 +ssg14 +(dp118 +I0 +I0 +ssg16 +g116 +sg17 +(dp119 +g19 +VBagpipes +p120 +ssa(dp121 +g2 +(dp122 +g4 +Vhttp://freebase.com/view/en/guiro +p123 +sg6 +S'The g\xc3\xbciro (Spanish pronunciation:\xc2\xa0[\xcb\x88\xc9\xa1wi\xc9\xbeo]) is a Dominican [percussion instrument]] consisting of an open-ended, hollow gourd with parallel notches cut in one side. It is played by rubbing a wooden stick ("pua") along the notches to produce a ratchet-like sound. The g\xc3\xbciro is commonly used in Latin-American music, and plays a key role in the typical cumbia rhythm section. The g\xc3\xbciro is also known as calabazo, guayo, ralladera, or rascador. In Brazil it is commonly known as "reco-reco".\nThe...' +p124 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03rh0wz +p125 +sg10 +VGüiro +p126 +sg12 +Vhttp://indextank.com/_static/common/demo/03rh0wz.jpg +p127 +ssg14 +(dp128 +I0 +I1 +ssg16 +g126 +sg17 +(dp129 +g19 +VPercussion +p130 +ssa(dp131 +g2 +(dp132 +g4 +Vhttp://freebase.com/view/en/piccolo_oboe +p133 +sg6 +S'The piccolo oboe, also known as the piccoloboe, is the smallest and highest pitched member of the oboe family, historically known as the oboe musette. (It should not be confused with the similarly named musette, which is bellows-blown and characterized by a drone.) Pitched in E-flat or F above the regular oboe (which is a C instrument), the piccolo oboe is a sopranino version of the oboe, comparable to the E-flat clarinet.\nPiccolo oboes are produced by the French makers F. Lor\xc3\xa9e and Marigaux...' +p134 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02clxsj +p135 +sg10 +VPiccolo oboe +p136 +sg12 +Vhttp://indextank.com/_static/common/demo/02clxsj.jpg +p137 +ssg14 +(dp138 +I0 +I1 +ssg16 +g136 +sg17 +(dp139 +g19 +VOboe +p140 +ssa(dp141 +g2 +(dp142 +g4 +Vhttp://freebase.com/view/en/heckelphone +p143 +sg6 +S'The Heckelphone (German: Heckelphon) is a musical instrument invented by Wilhelm Heckel and his sons. Introduced in 1904, it is similar to the cor anglais (English horn).\nThe Heckelphone is a double reed instrument of the oboe family, but with a wider bore and hence a heavier and more penetrating tone. It is pitched an octave below the oboe and furnished with an additional semitone taking its range down to A. It was intended to provide a broad oboe-like sound in the middle register of the...' +p144 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044xfzb +p145 +sg10 +VHeckelphone +p146 +sg12 +Vhttp://indextank.com/_static/common/demo/044xfzb.jpg +p147 +ssg14 +(dp148 +I0 +I0 +ssg16 +g146 +sg17 +(dp149 +g19 +VOboe +p150 +ssa(dp151 +g2 +(dp152 +g4 +Vhttp://freebase.com/view/en/harp_guitar +p153 +sg6 +S'The harp guitar (or "harp-guitar") is a stringed instrument with a history of well over two centuries. While there are several unrelated historical stringed instruments that have appropriated the name \xe2\x80\x9charp-guitar\xe2\x80\x9d over the centuries, the term today is understood as the accepted vernacular to refer to a particular family of instruments defined as "A guitar, in any of its accepted forms, with any number of additional unstopped strings that can accommodate individual plucking." Additionally,...' +p154 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d5pp1 +p155 +sg10 +VHarp guitar +p156 +sg12 +Vhttp://indextank.com/_static/common/demo/02d5pp1.jpg +p157 +ssg14 +(dp158 +I0 +I2 +ssg16 +g156 +sg17 +(dp159 +g19 +VPlucked string instrument +p160 +ssa(dp161 +g2 +(dp162 +g4 +Vhttp://freebase.com/view/en/bandoneon +p163 +sg6 +S'The bandone\xc3\xb3n is a type of concertina particularly popular in Argentina and Uruguay. It plays an essential role in the orquesta t\xc3\xadpica, the tango orchestra. The bandone\xc3\xb3n, called bandonion by a German instrument dealer, Heinrich Band (1821\xe2\x80\x931860), was originally intended as an instrument for religious music and the popular music of the day, in contrast to its predecessor, the German concertina (or Konzertina), considered to be a folk instrument by some modern authors. German sailors and...' +p164 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04pggqy +p165 +sg10 +VBandoneón +p166 +sg12 +Vhttp://indextank.com/_static/common/demo/04pggqy.jpg +p167 +ssg14 +(dp168 +I0 +I3 +ssg16 +g166 +sg17 +(dp169 +g19 +VAccordion +p170 +ssa(dp171 +g2 +(dp172 +g4 +Vhttp://freebase.com/view/en/an_b_u +p173 +sg6 +S'The \xc4\x91\xc3\xa0n b\xe1\xba\xa7u (\xc4\x91\xc3\xa0n \xc4\x91\xe1\xbb\x99c huy\xe1\xbb\x81n or \xc4\x91\xe1\xbb\x99c huy\xe1\xbb\x81n c\xe1\xba\xa7m, \xe7\x8d\xa8\xe7\xb5\x83\xe7\x90\xb4) is a Vietnamese monochord. While the earliest written records of the Dan Bau date its origin to 1770, many scholars estimate its age to be up to one thousand years older than that. A popular legend of its beginning tells of a blind woman playing it in the market to earn a living for her family while her husband was at war. Whether this tale is based in fact or not, it remains true that the Dan Bau has historically been played by blind...' +p174 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0c5bjvy +p175 +sg10 +V\u0110àn b\u1ea7u +p176 +sg12 +Vhttp://indextank.com/_static/common/demo/0c5bjvy.jpg +p177 +ssg14 +(dp178 +I0 +I0 +ssg16 +g176 +sg17 +(dp179 +g19 +VPlucked string instrument +p180 +ssa(dp181 +g2 +(dp182 +g4 +Vhttp://freebase.com/view/en/positive_organ +p183 +sg6 +S'A positive organ (pronounced "positeev"; also positiv organ, positif organ, portable organ, chair organ, or simply positive, positiv, positif, or chair) (from the Latin verb ponere, "to place") is a small, usually one-manual, pipe organ that is built to be more or less mobile. It was common in sacred and secular music between the 10th and the 18th centuries, in chapels and small churches, as a chamber organ and for the basso continuo in ensemble works. The smallest common kind of positive,...' +p184 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044y95b +p185 +sg10 +VPositive organ +p186 +sg12 +Vhttp://indextank.com/_static/common/demo/044y95b.jpg +p187 +ssg14 +(dp188 +I0 +I0 +ssg16 +g186 +sg17 +(dp189 +g19 +VOrgan +p190 +ssa(dp191 +g2 +(dp192 +g4 +Vhttp://freebase.com/view/en/basset-horn +p193 +sg6 +S'The basset horn (sometimes written basset-horn) is a musical instrument, a member of the clarinet family.\nLike the clarinet, the instrument is a wind instrument with a single reed and a cylindrical bore. However, the basset horn is larger and has a bend near the mouthpiece rather than an entirely straight body (older instruments are typically curved or bent in the middle), and while the clarinet is typically a transposing instrument in B\xe2\x99\xad or A (meaning a written C sounds as a B\xe2\x99\xad or A), the...' +p194 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cpny7 +p195 +sg10 +VBasset-horn +p196 +sg12 +Vhttp://indextank.com/_static/common/demo/02cpny7.jpg +p197 +ssg14 +(dp198 +I0 +I1 +ssg16 +g196 +sg17 +(dp199 +g19 +VClarinet +p200 +ssa(dp201 +g2 +(dp202 +g4 +Vhttp://freebase.com/view/en/russian_guitar +p203 +sg6 +S'The Russian guitar (sometimes referred to as a "Gypsy guitar") is a seven-string acoustic guitar that arrived in Russia toward the end of the 18th century and the beginning of the 19th century, most probably as an evolution of the cittern, kobza, and torban. It is known in Russian as the semistrunnaya gitara (\xd1\x81\xd0\xb5\xd0\xbc\xd0\xb8\xd1\x81\xd1\x82\xd1\x80\xd1\x83\xd0\xbd\xd0\xbd\xd0\xb0\xd1\x8f \xd0\xb3\xd0\xb8\xd1\x82\xd0\xb0\xd1\x80\xd0\xb0), or affectionately as the semistrunka (\xd1\x81\xd0\xb5\xd0\xbc\xd0\xb8\xd1\x81\xd1\x82\xd1\x80\xd1\x83\xd0\xbd\xd0\xba\xd0\xb0), which translates to "seven-string". These guitars are typically tuned to an Open G chord as follows: DGBdgbd\'....' +p204 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s3wln +p205 +sg10 +VRussian guitar +p206 +sg12 +Vhttp://indextank.com/_static/common/demo/03s3wln.jpg +p207 +ssg14 +(dp208 +I0 +I0 +ssg16 +g206 +sg17 +(dp209 +g19 +VPlucked string instrument +p210 +ssa(dp211 +g2 +(dp212 +g4 +Vhttp://freebase.com/view/en/organ +p213 +sg6 +S'The organ (from Greek \xcf\x8c\xcf\x81\xce\xb3\xce\xb1\xce\xbd\xce\xbf\xce\xbd organon, "organ, instrument, tool"), is a keyboard instrument of one or more divisions, each played with its own keyboard operated either with the hands or with the feet. The organ is a relatively old musical instrument in the Western musical tradition, dating from the time of Ctesibius of Alexandria who is credited with the invention of the hydraulis. By around the 8th century it had overcome early associations with gladiatorial combat and gradually assumed a...' +p214 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029ffyk +p215 +sg10 +VOrgan +p216 +sg12 +Vhttp://indextank.com/_static/common/demo/029ffyk.jpg +p217 +ssg14 +(dp218 +I0 +I351 +ssg16 +g216 +sg17 +(dp219 +g19 +VKeyboard instrument +p220 +ssa(dp221 +g2 +(dp222 +g4 +Vhttp://freebase.com/view/en/hammond_organ +p223 +sg6 +S'The Hammond organ is an electric organ invented by Laurens Hammond in 1934 and manufactured by the Hammond Organ Company. While the Hammond organ was originally sold to churches as a lower-cost alternative to the wind-driven pipe organ, in the 1960s and 1970s it became a standard keyboard instrument for jazz, blues, rock music, church and gospel music.\nThe original Hammond organ used additive synthesis of waveforms from harmonic series made by mechanical tonewheels that rotate in front of...' +p224 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04rrh5l +p225 +sg10 +VHammond organ +p226 +sg12 +Vhttp://indextank.com/_static/common/demo/04rrh5l.jpg +p227 +ssg14 +(dp228 +I0 +I200 +ssg16 +g226 +sg17 +(dp229 +g19 +VElectronic organ +p230 +ssa(dp231 +g2 +(dp232 +g4 +Vhttp://freebase.com/view/en/morin_khuur +p233 +sg6 +S'The morin khuur (Mongolian: \xd0\xbc\xd0\xbe\xd1\x80\xd0\xb8\xd0\xbd \xd1\x85\xd1\x83\xd1\x83\xd1\x80) is a traditional Mongolian bowed stringed instrument. It is one of the most important musical instruments of the Mongolian people, and is considered a symbol of the Mongolian nation. The morin khuur is one of the Masterpieces of the Oral and Intangible Heritage of Humanity identified by UNESCO. It produces a sound which is poetically described as expansive and unrestrained, like a wild horse neighing, or like a breeze in the grasslands.\nThe full...' +p234 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/042d2j2 +p235 +sg10 +VMorin khuur +p236 +sg12 +Vhttp://indextank.com/_static/common/demo/042d2j2.jpg +p237 +ssg14 +(dp238 +I0 +I0 +ssg16 +g236 +sg17 +(dp239 +g19 +VBowed string instruments +p240 +ssa(dp241 +g2 +(dp242 +g4 +Vhttp://freebase.com/view/en/fender_precision_bass +p243 +sg6 +S'The Fender Precision Bass (often shortened to "P Bass") is an Electric bass.\nDesigned by Leo Fender as a prototype in 1950 and brought to market in 1951, the Precision was the first bass to earn widespread attention and use. A revolutionary instrument for the time, the Precision Bass has made an immeasurable impact on the sound of popular music ever since.\nAlthough the Precision was the first mass-produced and widely-used bass, it was not the first model of the instrument, as is sometimes...' +p244 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bc45s +p245 +sg10 +VFender Precision Bass +p246 +sg12 +Vhttp://indextank.com/_static/common/demo/02bc45s.jpg +p247 +ssg14 +(dp248 +I0 +I122 +ssg16 +g246 +sg17 +(dp249 +g19 +VBass guitar +p250 +ssa(dp251 +g2 +(dp252 +g4 +Vhttp://freebase.com/view/m/07fdrb +p253 +sg6 +S"Pipe describes a number of musical instruments, historically referring to perforated wind instruments. The word is an onomatopoeia, and comes from the tone which can resemble that of a bird chirping.\nFipple flutes are found in many cultures around the world. Often with six holes, the shepherd's pipe is a common pastoral image. Shepherds often piped both to soothe the sheep and to amuse themselves. Modern manufactured six-hole folk pipes are referred to as pennywhistle or tin whistle. The..." +p254 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02g57g5 +p255 +sg10 +VPipe +p256 +sg12 +Vhttp://indextank.com/_static/common/demo/02g57g5.jpg +p257 +ssg14 +(dp258 +I0 +I0 +ssg16 +g256 +sg17 +(dp259 +g19 +VWoodwind instrument +p260 +ssa(dp261 +g2 +(dp262 +g4 +Vhttp://freebase.com/view/en/western_concert_flute +p263 +sg6 +S'The Western concert flute is a transverse (side-blown) woodwind instrument made of metal or wood. It is the most common variant of the flute. A musician who plays the flute is called a flautist, flutist, or flute player.\nThis type of flute is used in many ensembles including concert bands, orchestras, flute ensembles, and occasionally jazz bands and big bands. Other flutes in this family include the piccolo, alto flute, bass flute, contrabass flute and double contrabass flute. Millions of...' +p264 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bspr7 +p265 +sg10 +VWestern concert flute +p266 +sg12 +Vhttp://indextank.com/_static/common/demo/02bspr7.jpg +p267 +ssg14 +(dp268 +I0 +I3 +ssg16 +g266 +sg17 +(dp269 +g19 +VFlute (transverse) +p270 +ssa(dp271 +g2 +(dp272 +g4 +Vhttp://freebase.com/view/en/pipa +p273 +sg6 +S'The pipa (Chinese: \xe7\x90\xb5\xe7\x90\xb6; pinyin: p\xc3\xadp\xc3\xa1) is a four-stringed Chinese musical instrument, belonging to the plucked category of instruments (\xe5\xbc\xb9\xe6\x8b\xa8\xe4\xb9\x90\xe5\x99\xa8/\xe5\xbd\x88\xe6\x92\xa5\xe6\xa8\x82\xe5\x99\xa8). Sometimes called the Chinese lute, the instrument has a pear-shaped wooden body with a varying number of frets ranging from 12\xe2\x80\x9326. Another Chinese 4 string plucked lute is the liuqin, which looks like a smaller version of the pipa.\nThe pipa appeared in the Qin Dynasty (221 - 206 BCE) and was developed during the Han Dynasty. It is one of the most...' +p274 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f1b99 +p275 +sg10 +VPipa +p276 +sg12 +Vhttp://indextank.com/_static/common/demo/02f1b99.jpg +p277 +ssg14 +(dp278 +I0 +I2 +ssg16 +g276 +sg17 +(dp279 +g19 +VPlucked string instrument +p280 +ssa(dp281 +g2 +(dp282 +g4 +Vhttp://freebase.com/view/en/oboe_da_caccia +p283 +sg6 +S'The oboe da caccia (literally "hunting oboe" in Italian) is a double reed woodwind instrument in the oboe family, pitched a fifth below the oboe and used primarily in the Baroque period of European classical music. It has a curved tube and a brass bell, unusual for an oboe.\nIts range is close to that of the English horn\xe2\x80\x94that is, from the F below middle C (notated C4 but sounding F3) to the G above the treble staff (notated D6 but sounding G5). The oboe da caccia is thus a transposing...' +p284 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bykkl +p285 +sg10 +VOboe da caccia +p286 +sg12 +Vhttp://indextank.com/_static/common/demo/02bykkl.jpg +p287 +ssg14 +(dp288 +I0 +I0 +ssg16 +g286 +sg17 +(dp289 +g19 +VOboe +p290 +ssa(dp291 +g2 +(dp292 +g4 +Vhttp://freebase.com/view/en/gittern +p293 +sg6 +S'The gittern was a relatively small, quill-plucked, gut strung instrument that originated around the 13th century and came to Europe via Moorish Spain. It was also called the quinterne in Germany, the guitarra in Spain, and the chitarra in Italy. A popular instrument with the minstrels and amateur musicians of the 14th century, the gittern eventually out-competed its rival, the citole. Soon after, its popularity began to fade, giving rise to the larger and more evocative lute and guitar.\nUp...' +p294 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04rqgmm +p295 +sg10 +VGittern +p296 +sg12 +Vhttp://indextank.com/_static/common/demo/04rqgmm.jpg +p297 +ssg14 +(dp298 +I0 +I0 +ssg16 +g296 +sg17 +(dp299 +g19 +VPlucked string instrument +p300 +ssa(dp301 +g2 +(dp302 +g4 +Vhttp://freebase.com/view/m/03kljf +p303 +sg6 +S'A serpent is a bass wind instrument, descended from the cornett, and a distant ancestor of the tuba, with a mouthpiece like a brass instrument but side holes like a woodwind. It is usually a long cone bent into a snakelike shape, hence the name. The serpent is closely related to the cornett, although it is not part of the cornett family, due to the absence of a thumb hole. It is generally made out of wood, with walnut being a particularly popular choice. The outside is covered with dark...' +p304 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02b7nx9 +p305 +sg10 +VSerpent +p306 +sg12 +Vhttp://indextank.com/_static/common/demo/02b7nx9.jpg +p307 +ssg14 +(dp308 +I0 +I0 +ssg16 +g306 +sg17 +(dp309 +g19 +VWind instrument +p310 +ssa(dp311 +g2 +(dp312 +g4 +Vhttp://freebase.com/view/en/tar +p313 +sg6 +S'The t\xc4\x81r (Persian: \xd8\xaa\xd8\xa7\xd8\xb1) is a long-necked, waisted Iranian instrument. It has been adopted by other cultures and countries like Azerbaijan, Armenia, Georgia, and other areas near the Caucasus region. The word tar ( \xd8\xaa\xd8\xa7\xd8\xb1\') itself means "string" in Persian, though it might have the same meaning in languages influenced by Persian or any other branches of Iranian languages like Kurdish. Therefore, Tar is common amongst all the Iranian people as well as the territories that are named as Iranian...' +p314 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02ff9bp +p315 +sg10 +VTar +p316 +sg12 +Vhttp://indextank.com/_static/common/demo/02ff9bp.jpg +p317 +ssg14 +(dp318 +I0 +I5 +ssg16 +g316 +sg17 +(dp319 +g19 +VPlucked string instrument +p320 +ssa(dp321 +g2 +(dp322 +g4 +Vhttp://freebase.com/view/en/fipple +p323 +sg6 +S'A fipple is a constricted mouthpiece common to many end-blown woodwind instruments, such as the tin whistle and the recorder. These instruments are known variously as fipple flutes, duct flutes, or tubular-ducted flutes.\nIn the accompanying illustration of the head of a recorder, the wooden fipple plug (A), with a "ducted flue" windway above it in the mouthpiece of the instrument, compresses the player\'s breath, so that it travels along the duct (B), called the "windway". Exiting from the...' +p324 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cy2yq +p325 +sg10 +VFipple +p326 +sg12 +Vhttp://indextank.com/_static/common/demo/02cy2yq.jpg +p327 +ssg14 +(dp328 +I0 +I0 +ssg16 +g326 +sg17 +(dp329 +g19 +VWoodwind instrument +p330 +ssa(dp331 +g2 +(dp332 +g4 +Vhttp://freebase.com/view/en/electric_guitar +p333 +sg6 +S'An electric guitar is a guitar that uses the principle of electromagnetic induction to convert vibrations of its metal strings into electric signals. Since the generated signal is too weak to drive a loudspeaker, it is amplified before sending it to a loudspeaker. Since the output of an electric guitar is an electric signal, the signal may easily be altered using electronic circuits to add color to the sound. Often the signal is modified using effects such as reverb and distortion. Conceived...' +p334 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03qw7_m +p335 +sg10 +VElectric guitar +p336 +sg12 +Vhttp://indextank.com/_static/common/demo/03qw7_m.jpg +p337 +ssg14 +(dp338 +I0 +I344 +ssg16 +g336 +sg17 +(dp339 +g19 +VGuitar +p340 +ssa(dp341 +g2 +(dp342 +g4 +Vhttp://freebase.com/view/en/soprillo +p343 +sg6 +S'The sopranissimo or soprillo saxophone is the smallest member of the saxophone family. It is pitched in B\xe2\x99\xad, one octave above the soprano saxophone. Because of the difficulties in building such a small instrument\xe2\x80\x94the soprillo is 12\xc2\xa0inches long, 13\xc2\xa0inches with the mouthpiece\xe2\x80\x94it is only recently that a true sopranissimo saxophone been produced. The keywork only extends to a written high E\xe2\x99\xad (rather than F like most saxophones) and the upper octave key has to be placed in the mouthpiece.\nThe...' +p344 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/042vsmy +p345 +sg10 +VSoprillo +p346 +sg12 +Vhttp://indextank.com/_static/common/demo/042vsmy.jpg +p347 +ssg14 +(dp348 +I0 +I0 +ssg16 +g346 +sg17 +(dp349 +g19 +VSaxophone +p350 +ssa(dp351 +g2 +(dp352 +g4 +Vhttp://freebase.com/view/en/esraj +p353 +sg6 +S'The esraj (Bengali: \xe0\xa6\x8f\xe0\xa6\xb8\xe0\xa7\x8d\xe0\xa6\xb0\xe0\xa6\xbe\xe0\xa6\x9c; Hindi: \xe0\xa4\x87\xe0\xa4\xb8\xe0\xa4\xb0\xe0\xa4\xbe\xe0\xa4\x9c; also called israj) is a string instrument found in two forms throughout the north, central, and east regions of India. It is a young instrument by Indian terms, being only about 200 years old. The dilruba is found in the north, where it is used in religious music and light classical songs in the urban areas. Its name is translated as "robber of the heart." The esraj is found in the east and central areas, particularly Bengal (Bangladesh and Indian...' +p354 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/078kmxh +p355 +sg10 +VEsraj +p356 +sg12 +Vhttp://indextank.com/_static/common/demo/078kmxh.jpg +p357 +ssg14 +(dp358 +I0 +I0 +ssg16 +g356 +sg17 +(dp359 +g19 +VBowed string instruments +p360 +ssa(dp361 +g2 +(dp362 +g4 +Vhttp://freebase.com/view/en/harmonium +p363 +sg6 +S'A harmonium is a free-standing keyboard instrument similar to a reed organ. Sound is produced by air being blown through sets of free reeds, resulting in a sound similar to that of an accordion. The air is usually supplied by bellows operated by foot, hand, or knees.\nIn North America, the most common pedal-pumped free-reed keyboard instrument is known as the American Reed Organ, (or parlor organ, pump organ, cabinet organ, cottage organ, etc.) and along with the earlier melodeon, is operated...' +p364 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f2d3m +p365 +sg10 +VHarmonium +p366 +sg12 +Vhttp://indextank.com/_static/common/demo/02f2d3m.jpg +p367 +ssg14 +(dp368 +I0 +I39 +ssg16 +g366 +sg17 +(dp369 +g19 +VKeyboard instrument +p370 +ssa(dp371 +g2 +(dp372 +g4 +Vhttp://freebase.com/view/en/pandeiro +p373 +sg6 +S'The pandeiro (Portuguese pronunciation:\xc2\xa0[p\xc9\x90\xcc\x83\xcb\x88dej\xc9\xbeu]) is a type of hand frame drum.\nThere are two important distinctions between a pandeiro and the common tambourine. The tension of the head on the pandeiro can be tuned, allowing the player a choice of high and low notes. Also, the metal jingles (called platinelas in Portuguese) are cupped, creating a crisper, drier and less sustained tone on the pandeiro than on the tambourine. This provides clarity when swift, complex rhythms are played.\nIt...' +p374 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dl9k2 +p375 +sg10 +VPandeiro +p376 +sg12 +Vhttp://indextank.com/_static/common/demo/02dl9k2.jpg +p377 +ssg14 +(dp378 +I0 +I2 +ssg16 +g376 +sg17 +(dp379 +g19 +VPercussion +p380 +ssa(dp381 +g2 +(dp382 +g4 +Vhttp://freebase.com/view/en/appalachian_dulcimer +p383 +sg6 +S'The Appalachian dulcimer (or mountain dulcimer) is a fretted string instrument of the zither family, typically with three or four strings. It is native to the Appalachian region of the United States. The body extends the length of the fingerboard, and its fretting is generally diatonic.\nAlthough the Appalachian dulcimer appeared in regions dominated by Irish and Scottish settlement, the instrument has no known precedent in Ireland or Scotland. However, several diatonic fretted zithers exist...' +p384 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cwrh0 +p385 +sg10 +VAppalachian dulcimer +p386 +sg12 +Vhttp://indextank.com/_static/common/demo/02cwrh0.jpg +p387 +ssg14 +(dp388 +I0 +I24 +ssg16 +g386 +sg17 +(dp389 +g19 +VZither +p390 +ssa(dp391 +g2 +(dp392 +g4 +Vhttp://freebase.com/view/en/bass_flute +p393 +sg6 +S'The bass flute is the bass member of the flute family. It is in the key of C, pitched one octave below the concert flute. Because of the length of its tube (approximately 146\xc2\xa0cm), it is usually made with a "J" shaped head joint, which brings the embouchure hole within reach of the player. It is usually only used in flute choirs, as it is easily drowned out by other instruments of comparable register, such as the clarinet.\nPrior to the mid-20th century, the term "bass flute" was sometimes...' +p394 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bkpyl +p395 +sg10 +VBass flute +p396 +sg12 +Vhttp://indextank.com/_static/common/demo/02bkpyl.jpg +p397 +ssg14 +(dp398 +I0 +I2 +ssg16 +g396 +sg17 +(dp399 +g19 +VFlute (transverse) +p400 +ssa(dp401 +g2 +(dp402 +g4 +Vhttp://freebase.com/view/en/saxotromba +p403 +sg6 +S'The saxotromba is a valved brasswind instrument invented by the Belgian instrument-maker Adolphe Sax around 1844. It was designed for the mounted bands of the French military, probably as a substitute for the French horn. The saxotrombas comprised a family of half-tube instruments of different pitches. By about 1867 the saxotromba was no longer being used by the French military, but specimens of various sizes continued to be manufactured until the early decades of the twentieth century,...' +p404 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05k84bg +p405 +sg10 +VSaxotromba +p406 +sg12 +Vhttp://indextank.com/_static/common/demo/05k84bg.jpg +p407 +ssg14 +(dp408 +I0 +I0 +ssg16 +g406 +sg17 +(dp409 +g19 +VBrass instrument +p410 +ssa(dp411 +g2 +(dp412 +g4 +Vhttp://freebase.com/view/en/double_contrabass_flute +p413 +sg6 +S"The double contrabass flute (sometimes also called the octobass flute or subcontrabass flute) is the largest and lowest pitched metal flute in the world (the hyperbass flute has an even lower range, though it is made out of PVC pipes and wood). It is pitched in the key of C, three octaves below the concert flute (two octaves below the bass flute and one octave below the contrabass flute). Its lowest note is C1, one octave below the cello's lowest C. This note is relatively easy to play in..." +p414 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d5lrf +p415 +sg10 +VDouble contrabass flute +p416 +sg12 +Vhttp://indextank.com/_static/common/demo/02d5lrf.jpg +p417 +ssg14 +(dp418 +I0 +I0 +ssg16 +g416 +sg17 +(dp419 +g19 +VFlute (transverse) +p420 +ssa(dp421 +g2 +(dp422 +g4 +Vhttp://freebase.com/view/en/oud +p423 +sg6 +S'The oud (Arabic: \xd8\xb9\xd9\x88\xd8\xaf\xe2\x80\x8e \xca\xbf\xc5\xabd, plural:\xd8\xa3\xd8\xb9\xd9\x88\xd8\xa7\xd8\xaf, a\xe2\x80\x98w\xc4\x81d; Assyrian:\xdc\xa5\xdc\x98\xdc\x95 \xc5\xabd, Persian: \xd8\xa8\xd8\xb1\xd8\xa8\xd8\xb7 barbat; Turkish: ud or ut; Greek: \xce\xbf\xcf\x8d\xcf\x84\xce\xb9; Armenian: \xd5\xb8\xd6\x82\xd5\xa4, Azeri: ud; Hebrew: \xd7\xa2\xd7\x95\xd7\x93 ud\xe2\x80\x8e; Somali: cuud or kaban) is a pear-shaped stringed instrument commonly used in North Africa (Chaabi, Egyptian music, Andalusian, ...) and Middle Eastern music. The modern oud and the European lute both descend from a common ancestor via diverging evolutionary paths. The oud is readily distinguished by its lack of frets and smaller...' +p424 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02btxfq +p425 +sg10 +VOud +p426 +sg12 +Vhttp://indextank.com/_static/common/demo/02btxfq.jpg +p427 +ssg14 +(dp428 +I0 +I38 +ssg16 +g426 +sg17 +(dp429 +g19 +VPlucked string instrument +p430 +ssa(dp431 +g2 +(dp432 +g4 +Vhttp://freebase.com/view/en/saxtuba +p433 +sg6 +S"The saxtuba is an obsolete valved brasswind instrument conceived by the Belgian instrument-maker Adolphe Sax around 1845. The design of the instrument was inspired by the ancient Roman cornu and tuba. The saxtubas, which comprised a family of half-tube and whole-tube instruments of varying pitches, were first employed in Fromental Hal\xc3\xa9vy's opera Le Juif errant (The Wandering Jew) in 1852. Their only other public appearance of note was at a military ceremony on the Champ de Mars in Paris in..." +p434 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05l4224 +p435 +sg10 +VSaxtuba +p436 +sg12 +Vhttp://indextank.com/_static/common/demo/05l4224.jpg +p437 +ssg14 +(dp438 +I0 +I0 +ssg16 +g436 +sg17 +(dp439 +g19 +VBrass instrument +p440 +ssa(dp441 +g2 +(dp442 +g4 +Vhttp://freebase.com/view/en/sitar +p443 +sg6 +S'The sitar (pronounced /s\xc9\xaa\xcb\x88t\xc9\x91\xcb\x90(r)/; Hindi: \xe0\xa4\xb8\xe0\xa4\xbf\xe0\xa4\xa4\xe0\xa4\xbe\xe0\xa4\xb0, Bengali: \xe0\xa6\xb8\xe0\xa7\x87\xe0\xa6\xa4\xe0\xa6\xbe\xe0\xa6\xb0, Urdu: \xd8\xb3\xd8\xaa\xd8\xa7\xd8\xb1, Persian: \xd8\xb3\xdb\x8c\xe2\x80\x8c\xd8\xaa\xd8\xa7\xd8\xb1\xc2\xa0; Hindustani pronunciation:\xc2\xa0[\xcb\x88s\xc9\xaa.t\xcc\xaaa\xcb\x90r]) is a plucked stringed instrument predominantly used in Hindustani classical music, where it has been ubiquitous since the Middle Ages. It derives its resonance from sympathetic strings, a long hollow neck and a gourd resonating chamber.\nUsed throughout the Indian subcontinent, particularly in Northern India, Pakistan, and Bangladesh, the sitar became known in...' +p444 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03tdmc0 +p445 +sg10 +VSitar +p446 +sg12 +Vhttp://indextank.com/_static/common/demo/03tdmc0.jpg +p447 +ssg14 +(dp448 +I0 +I71 +ssg16 +g446 +sg17 +(dp449 +g19 +VPlucked string instrument +p450 +ssa(dp451 +g2 +(dp452 +g4 +Vhttp://freebase.com/view/en/pipe_organ +p453 +sg6 +S'The pipe organ is a musical instrument that produces sound by driving pressurized air (called wind) through pipes selected via a keyboard. Because each organ pipe produces a single pitch, the pipes are provided in sets called ranks, each of which has a common timbre and volume throughout the keyboard compass. Most organs have multiple ranks of pipes of differing timbre, pitch and loudness that the player can employ singly or in combination through the use of controls called stops.\nA pipe...' +p454 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029ffyk +p455 +sg10 +VPipe organ +p456 +sg12 +Vhttp://indextank.com/_static/common/demo/029ffyk.jpg +p457 +ssg14 +(dp458 +I0 +I5 +ssg16 +g456 +sg17 +(dp459 +g19 +VOrgan +p460 +ssa(dp461 +g2 +(dp462 +g4 +Vhttp://freebase.com/view/en/washtub_bass +p463 +sg6 +S'The washtub bass, or "gutbucket", is a stringed instrument used in American folk music that uses a metal washtub as a resonator. Although it is possible for a washtub bass to have four or more strings and tuning pegs, traditional washtub basses have a single string whose pitch is adjusted by pushing or pulling on a staff or stick to change the tension.\nThe washtub bass was used in jug bands that were popular in some African Americans communities in the early 1900s. In the 1950s, British...' +p464 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bkd72 +p465 +sg10 +VWashtub bass +p466 +sg12 +Vhttp://indextank.com/_static/common/demo/02bkd72.jpg +p467 +ssg14 +(dp468 +I0 +I0 +ssg16 +g466 +sg17 +(dp469 +g19 +VOther string instruments +p470 +ssa(dp471 +g2 +(dp472 +g4 +Vhttp://freebase.com/view/en/requinto +p473 +sg6 +S'The term requinto is used in both Spanish and Portuguese to mean a smaller, higher-pitched version of another instrument. Thus, there are requinto guitars, drums, and several wind instruments.\nRequinto was 19th century Spanish for "little clarinet". Today, the word requinto, when used in relation to a clarinet, refers to the E-flat clarinet, also known as requint in Valencian language.\nRequinto can also mean a high-pitched flute (akin to a piccolo), or the person who plays it. In Galicia,...' +p474 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05lw32q +p475 +sg10 +VRequinto +p476 +sg12 +Vhttp://indextank.com/_static/common/demo/05lw32q.jpg +p477 +ssg14 +(dp478 +I0 +I1 +ssg16 +g476 +sg17 +(dp479 +g19 +VPlucked string instrument +p480 +ssa(dp481 +g2 +(dp482 +g4 +Vhttp://freebase.com/view/en/eight_string_guitar +p483 +sg6 +S'An eight-string guitar is a guitar with eight strings instead of the commonly used six strings. Such guitars are not as common as the six string variety, but are used by classical, jazz, and metal guitarists to expand the range of their instrument by adding two strings.\nThere are several variants of this instrument, one probably originating from Russia along with the seven string guitar variant in the 19th century. The eight string guitar has recently begun to gain popularity, notably among...' +p484 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dg4vf +p485 +sg10 +VEight string guitar +p486 +sg12 +Vhttp://indextank.com/_static/common/demo/02dg4vf.jpg +p487 +ssg14 +(dp488 +I0 +I0 +ssg16 +g486 +sg17 +(dp489 +g19 +VPlucked string instrument +p490 +ssa(dp491 +g2 +(dp492 +g4 +Vhttp://freebase.com/view/en/rebab +p493 +sg6 +S'The rebab (Arabic \xd8\xa7\xd9\x84\xd8\xb1\xd8\xa8\xd8\xa7\xd8\xa8\xd8\xa9 or \xd8\xb1\xd8\xa8\xd8\xa7\xd8\xa8\xd8\xa9 - "a bowed (instrument)"), also rebap, rabab, rebeb, rababah, or al-rababa) is a type of string instrument so named no later than the 8th century and spread via Islamic trading routes over much of North Africa, the Middle East, parts of Europe, and the Far East. The bowed variety often has a spike at the bottom to rest on the ground, and is thus called a spike fiddle in certain areas, but there exist plucked versions like the kabuli rebab (sometimes...' +p494 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041hf7c +p495 +sg10 +VRebab +p496 +sg12 +Vhttp://indextank.com/_static/common/demo/041hf7c.jpg +p497 +ssg14 +(dp498 +I0 +I0 +ssg16 +g496 +sg17 +(dp499 +g19 +VBowed string instruments +p500 +ssa(dp501 +g2 +(dp502 +g4 +Vhttp://freebase.com/view/en/an_nguy_t +p503 +sg6 +S"The \xc4\x91\xc3\xa0n nguy\xe1\xbb\x87t (also called nguy\xe1\xbb\x87t c\xe1\xba\xa7m, \xc4\x91\xc3\xa0n k\xc3\xacm, moon lute, or moon guitar) is a two-stringed Vietnamese traditional musical instrument. It is used in both folk and classical music, and remains popular throughout Vietnam (although during the 20th century many Vietnamese musicians increasingly gravitated toward the acoustic and electric guitar).\nThe \xc4\x91\xc3\xa0n nguy\xe1\xbb\x87t's strings, formerly made of twisted silk, are today generally made of nylon or fishing line. They are kept at a fairly low tension in..." +p504 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cwm30 +p505 +sg10 +V\u0110àn nguy\u1ec7t +p506 +sg12 +Vhttp://indextank.com/_static/common/demo/02cwm30.jpg +p507 +ssg14 +(dp508 +I0 +I0 +ssg16 +g506 +sg17 +(dp509 +g19 +VPlucked string instrument +p510 +ssa(dp511 +g2 +(dp512 +g4 +Vhttp://freebase.com/view/en/pump_organ +p513 +sg6 +S'The pump organ is a version of the reed organ where the player maintains the air pressure needed for creating the sound in the free reeds by pumping pedals with their feet.\nThe portative organ is a miniature version of the pipe organ where air pressure is also maintained by pumping, but in this case by hand.' +p514 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04rcd28 +p515 +sg10 +VPump organ +p516 +sg12 +Vhttp://indextank.com/_static/common/demo/04rcd28.jpg +p517 +ssg14 +(dp518 +I0 +I2 +ssg16 +g516 +sg17 +(dp519 +g19 +VOrgan +p520 +ssa(dp521 +g2 +(dp522 +g4 +Vhttp://freebase.com/view/en/theorbo +p523 +sg6 +S'A theorbo (Italian: tiorba, also tuorbe; French: th\xc3\xa9orbe, German: Theorbe) is a plucked string instrument. As a name, theorbo signifies a number of long-necked lutes with second pegboxes, such as the liuto attiorbato, the French th\xc3\xa9orbe des pi\xc3\xa8ces, the English theorbo, the archlute, the German baroque lute, the ang\xc3\xa9lique or angelica. The etymology of the name tiorba has not yet been explained. It is hypothesized that its origin might have been in the Slavic or Turkish "torba", meaning "bag"...' +p524 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029kp2s +p525 +sg10 +VTheorbo +p526 +sg12 +Vhttp://indextank.com/_static/common/demo/029kp2s.jpg +p527 +ssg14 +(dp528 +I0 +I2 +ssg16 +g526 +sg17 +(dp529 +g19 +VPlucked string instrument +p530 +ssa(dp531 +g2 +(dp532 +g4 +Vhttp://freebase.com/view/en/huqin +p533 +sg6 +S'Huqin (\xe8\x83\xa1\xe7\x90\xb4; pinyin: h\xc3\xbaq\xc3\xadn) is a family of bowed string instruments, more specifically, a spike fiddle popularly used in Chinese music. The instruments consist of a round, hexagonal, or octagonal sound box at the bottom with a neck attached that protrudes upwards. They also have two strings (except the sihu, which has four strings tuned in pairs) and their soundboxes are typically covered with either snakeskin (most often python) or thin wood. Huqin instruments have either two (or, more...' +p534 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cpkr7 +p535 +sg10 +VHuqin +p536 +sg12 +Vhttp://indextank.com/_static/common/demo/02cpkr7.jpg +p537 +ssg14 +(dp538 +I0 +I0 +ssg16 +g536 +sg17 +(dp539 +g19 +VBowed string instruments +p540 +ssa(dp541 +g2 +(dp542 +g4 +Vhttp://freebase.com/view/en/northumbrian_smallpipes +p543 +sg6 +S'The Northumbrian smallpipes (also known as the Northumbrian pipes) are bellows-blown bagpipes from the North East of England. In , a survey of the bagpipes in the Pitt Rivers Museum, Oxford University, the organologist Anthony Baines wrote: It is perhaps the most civilized of the bagpipes, making no attempt to go farther than the traditional bagpipe music of melody over drone, but refining this music to the last degree.\nThe instrument consists of one chanter (generally with keys) and usually...' +p544 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cyw01 +p545 +sg10 +VNorthumbrian smallpipes +p546 +sg12 +Vhttp://indextank.com/_static/common/demo/02cyw01.jpg +p547 +ssg14 +(dp548 +I0 +I1 +ssg16 +g546 +sg17 +(dp549 +g19 +VBagpipes +p550 +ssa(dp551 +g2 +(dp552 +g4 +Vhttp://freebase.com/view/en/gravikord +p553 +sg6 +S'The gravikord is an electric double bridge-harp invented by Robert Grawi in 1986.\nThe gravikord is a new instrument developed on the basis of the West African kora. It is made of welded stainless steel tubing, with 24 nylon strings but no resonating gourd or skin. The bridge is made from a machined synthetic material with an integral piezo-electric sensor. Two handles located in elevation near the middle of the bridge allow holding the instrument. The bridge is curved to follow the arc of a...' +p554 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0dh91c5 +p555 +sg10 +VGravikord +p556 +sg12 +Vhttp://indextank.com/_static/common/demo/0dh91c5.jpg +p557 +ssg14 +(dp558 +I0 +I0 +ssg16 +g556 +sg17 +(dp559 +g19 +VPlucked string instrument +p560 +ssa(dp561 +g2 +(dp562 +g4 +Vhttp://freebase.com/view/en/auditorium_organ +p563 +sg6 +S'The Boardwalk Hall Auditorium Organ is the pipe organ in the Main Auditorium of the Boardwalk Hall (formerly known as the Atlantic City Convention Hall) in Atlantic City, New Jersey, built by the Midmer-Losh Organ Company. It is the largest organ in the world, as measured by the number of pipes. The Wanamaker Grand Court Organ has more ranks.\nThe Boardwalk Hall is a very large building, with a total floor area of 41,000\xc2\xa0sq\xc2\xa0ft (3,800\xc2\xa0m). Consequently, the organ runs on much higher wind...' +p564 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dyvgw +p565 +sg10 +VAuditorium Organ +p566 +sg12 +Vhttp://indextank.com/_static/common/demo/02dyvgw.jpg +p567 +ssg14 +(dp568 +I0 +I0 +ssg16 +g566 +sg17 +(dp569 +g19 +VPipe organ +p570 +ssa(dp571 +g2 +(dp572 +g4 +Vhttp://freebase.com/view/en/zampogna +p573 +sg6 +S'Zampogna is a generic term for a number of Italian double chantered pipes that can be found as far north as the southern part of the Marche, throughout areas in Abruzzo, Latium, Molise, Basilicata, Campania, Calabria, and Sicily. The tradition is now mostly associated with Christmas, and the most famous Italian carol, "Tu scendi dalle stelle" (You Come Down From the Stars) is derived from traditional zampogna music. However, there is an ongoing resurgence of the instrument in secular use...' +p574 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041dfcz +p575 +sg10 +VZampogna +p576 +sg12 +Vhttp://indextank.com/_static/common/demo/041dfcz.jpg +p577 +ssg14 +(dp578 +I0 +I0 +ssg16 +g576 +sg17 +(dp579 +g19 +VBagpipes +p580 +ssa(dp581 +g2 +(dp582 +g4 +Vhttp://freebase.com/view/en/hurdy_gurdy +p583 +sg6 +S'The hurdy gurdy or hurdy-gurdy (also known as a wheel fiddle) is a stringed musical instrument that produces sound by a crank-turned rosined wheel rubbing against the strings. The wheel functions much like a violin bow, and single notes played on the instrument sound similar to a violin. Melodies are played on a keyboard that presses tangents (small wedges, usually made of wood) against one or more of the strings to change their pitch. Like most other acoustic string instruments, it has a...' +p584 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029fyb2 +p585 +sg10 +VHurdy gurdy +p586 +sg12 +Vhttp://indextank.com/_static/common/demo/029fyb2.jpg +p587 +ssg14 +(dp588 +I0 +I5 +ssg16 +g586 +sg17 +(dp589 +g19 +VString instrument +p590 +ssa(dp591 +g2 +(dp592 +g4 +Vhttp://freebase.com/view/en/djembe +p593 +sg6 +S'A djembe ( /\xcb\x88d\xca\x92\xc9\x9bmbe\xc9\xaa/ JEM-bay) also known as jembe, jenbe, djimbe, jymbe, yembe, or jimbay, or sanbanyi in Susu; is a skin-covered drum meant to be played with bare hands. According to the Bamana people in Mali, the name of the djembe comes directly from the saying "Anke dj\xc3\xa9, anke b\xc3\xa9" which translates to "everyone gather together in peace" and defines the drum\'s purpose. In the Bambara language, "dj\xc3\xa9" is the verb for "gather" and "b\xc3\xa9" translates as "peace".\nIt is a member of the...' +p594 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02chnm5 +p595 +sg10 +VDjembe +p596 +sg12 +Vhttp://indextank.com/_static/common/demo/02chnm5.jpg +p597 +ssg14 +(dp598 +I0 +I9 +ssg16 +g596 +sg17 +(dp599 +g19 +VPercussion +p600 +ssa(dp601 +g2 +(dp602 +g4 +Vhttp://freebase.com/view/en/picco_pipe +p603 +sg6 +S'The picco pipe is the smallest form of ducted flue tabor pipe or flute-a-bec.\nIt is 3\xc2\xbd" long, with the windway taking up 1\xc2\xbd". It has only three holes: two in front and a dorsal thumb hole. It has the same mouthpiece as a recorder. The bore end hole of the picco Pipe has a small flare, and the lowest notes were played with a finger blocking this end.\nThe range is from b to c3, using the slight frequency shift between registers to sound a full chromatic scale, like the tabor pipe.\nIt was...' +p604 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03rk1x4 +p605 +sg10 +VPicco pipe +p606 +sg12 +Vhttp://indextank.com/_static/common/demo/03rk1x4.jpg +p607 +ssg14 +(dp608 +I0 +I0 +ssg16 +g606 +sg17 +(dp609 +g19 +VPipe +p610 +ssa(dp611 +g2 +(dp612 +g4 +Vhttp://freebase.com/view/en/yangqin +p613 +sg6 +S'The trapezoidal yangqin (simplified Chinese: \xe6\x89\xac\xe7\x90\xb4; traditional Chinese: \xe6\x8f\x9a\xe7\x90\xb4; pinyin: y\xc3\xa1ngq\xc3\xadn) is a Chinese hammered dulcimer, originally from Central Asia (Persia (modern-day Iran)). It used to be written with the characters \xe6\xb4\x8b\xe7\x90\xb4 (lit. "foreign zither"), but over time the first character changed to \xe6\x8f\x9a (also pronounced "y\xc3\xa1ng"), which means "acclaimed". It is also spelled yang quin or yang ch\'in. Hammered dulcimers of various types are now very popular not only in China, but also Eastern Europe, the...' +p614 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02b47fj +p615 +sg10 +VYangqin +p616 +sg12 +Vhttp://indextank.com/_static/common/demo/02b47fj.jpg +p617 +ssg14 +(dp618 +I0 +I1 +ssg16 +g616 +sg17 +(dp619 +g19 +VStruck string instruments +p620 +ssa(dp621 +g2 +(dp622 +g4 +Vhttp://freebase.com/view/en/street_organ +p623 +sg6 +S'A street organ is a mechanical organ designed to play in the street. The operator of a street organ is called an organ grinder. The two main types are the smaller German street organ and the larger Dutch street organ.\nIn the United Kingdom, street organ is often used to refer to a mechanically played piano like instrument. This is incorrect, and such instruments are called a Barrel piano.\nDutch street organs (unlike the simple street organ) are large organs that play book music. They are...' +p624 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02gf070 +p625 +sg10 +VStreet organ +p626 +sg12 +Vhttp://indextank.com/_static/common/demo/02gf070.jpg +p627 +ssg14 +(dp628 +I0 +I0 +ssg16 +g626 +sg17 +(dp629 +g19 +VMechanical organ +p630 +ssa(dp631 +g2 +(dp632 +g4 +Vhttp://freebase.com/view/en/grand_piano +p633 +sg6 +S'A grand piano is the concert form of a piano. A grand piano has the frame and strings placed horizontally, with the strings extending away from the keyboard. Grand pianos are distinguished from upright piano, which have their strings and frame arranged vertically. \n\nGrand Pianos are typically used for concerts and concert hall performances, although a baby grand piano can also be used in a household where space is limited. The strings on a Grand Piano are longer, resulting in a louder and...' +p634 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/064dhsl +p635 +sg10 +VGrand piano +p636 +sg12 +Vhttp://indextank.com/_static/common/demo/064dhsl.jpg +p637 +ssg14 +(dp638 +I0 +I0 +ssg16 +g636 +sg17 +(dp639 +g19 +VPiano +p640 +ssa(dp641 +g2 +(dp642 +g4 +Vhttp://freebase.com/view/en/dabakan +p643 +sg6 +S'The dabakan is a single-headed Philippine drum, primarily used as a supportive instrument in the kulintang ensemble. Among the five main kulintang instruments, it is the only non-gong element of the Maguindanao ensemble.\nThe dabakan is frequently described as either hour-glass, conical, tubular, or goblet in shape Normally, the dabakan is found having a length of more than two feet and a diameter of more than a foot about the widest part of the shell. The shell is carved from wood either...' +p644 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0419j5r +p645 +sg10 +VDabakan +p646 +sg12 +Vhttp://indextank.com/_static/common/demo/0419j5r.jpg +p647 +ssg14 +(dp648 +I0 +I0 +ssg16 +g646 +sg17 +(dp649 +g19 +VPercussion +p650 +ssa(dp651 +g2 +(dp652 +g4 +Vhttp://freebase.com/view/en/kacapi +p653 +sg6 +S'Kacapi is a zither-like Sundanese musical instrument played as the main accompanying instrument in the Tembang Sunda or Mamaos Cianjuran, kacapi suling (tembang Sunda without vocal accompaniment) genre (called kecapi seruling in Indonesian), pantun stories recitation or an additional instrument in Gamelan Degung performance.\nWord kacapi in Sundanese also refers to santol tree, from which initially the wood is believed to be used for building the zither instrument.\nAccording to its form or...' +p654 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029lrtt +p655 +sg10 +VKacapi +p656 +sg12 +Vhttp://indextank.com/_static/common/demo/029lrtt.jpg +p657 +ssg14 +(dp658 +I0 +I0 +ssg16 +g656 +sg17 +(dp659 +g19 +VPlucked string instrument +p660 +ssa(dp661 +g2 +(dp662 +g4 +Vhttp://freebase.com/view/en/doshpuluur +p663 +sg6 +S'The doshpuluur (Tuvan: \xd0\x94\xd0\xbe\xd1\x88\xd0\xbf\xd1\x83\xd0\xbb\xd1\x83\xd1\x83\xd1\x80) is a long-necked Tuvan lute made from wood, usually pine or larch. The doshpuluur is played by plucking and strumming. There are two different versions of the doshpuluur. One version has a trapezoidal soundbox, which is covered on both sides by goat skin and is fretless. The other has a kidney-shaped soundbox mostly of wood with a small goat or snake skin roundel on the front and has frets.\nTraditionally the instrument has only two strings, but there exist...' +p664 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02ddh5l +p665 +sg10 +VDoshpuluur +p666 +sg12 +Vhttp://indextank.com/_static/common/demo/02ddh5l.jpg +p667 +ssg14 +(dp668 +I0 +I0 +ssg16 +g666 +sg17 +(dp669 +g19 +VPlucked string instrument +p670 +ssa(dp671 +g2 +(dp672 +g4 +Vhttp://freebase.com/view/en/piano +p673 +sg6 +S"The piano is a musical instrument played by means of a keyboard. It is one of the most popular instruments in the world. Widely used in classical music for solo performances, ensemble use, chamber music and accompaniment, the piano is also very popular as an aid to composing and rehearsal. Although not portable and often expensive, the piano's versatility and ubiquity have made it one of the world's most familiar musical instruments.\nPressing a key on the piano's keyboard causes a..." +p674 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s9dxr +p675 +sg10 +VPiano +p676 +sg12 +Vhttp://indextank.com/_static/common/demo/03s9dxr.jpg +p677 +ssg14 +(dp678 +I0 +I3420 +ssg16 +g676 +sg17 +(dp679 +g19 +VKeyboard instrument +p680 +ssa(dp681 +g2 +(dp682 +g4 +Vhttp://freebase.com/view/en/bodhran +p683 +sg6 +S'The bodhr\xc3\xa1n ( /\xcb\x88b\xc9\x94r\xc9\x91\xcb\x90n/ or /\xcb\x88ba\xca\x8ar\xc9\x91\xcb\x90n/; plural bodhr\xc3\xa1ns or bodhr\xc3\xa1in) is an Irish frame drum ranging from 25 to 65\xc2\xa0cm (10" to 26") in diameter, with most drums measuring 35 to 45\xc2\xa0cm (14" to 18"). The sides of the drum are 9 to 20\xc2\xa0cm (3\xc2\xbd" to 8") deep. A goatskin head is tacked to one side (synthetic heads, or other animal skins are sometimes used). The other side is open ended for one hand to be placed against the inside of the drum head to control the pitch and timbre.\nOne or two crossbars,...' +p684 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0292qs1 +p685 +sg10 +VBodhrán +p686 +sg12 +Vhttp://indextank.com/_static/common/demo/0292qs1.jpg +p687 +ssg14 +(dp688 +I0 +I14 +ssg16 +g686 +sg17 +(dp689 +g19 +VDrum +p690 +ssa(dp691 +g2 +(dp692 +g4 +Vhttp://freebase.com/view/en/gadulka +p693 +sg6 +S'The gadulka (Bulgarian: \xd0\x93\xd1\x8a\xd0\xb4\xd1\x83\xd0\xbb\xd0\xba\xd0\xb0) is a traditional Bulgarian bowed string instrument. Alternate spellings are "gudulka" and "g\'dulka". Its name comes from a root meaning "to make noise, hum or buzz". The gadulka is an integral part of Bulgarian traditional instrumental ensembles, commonly played in the context of dance music.\nThe gadulka commonly has three (occasionally four) main strings with up to ten sympathetic resonating strings underneath, although there is a smaller variant of the...' +p694 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029qs02 +p695 +sg10 +VGadulka +p696 +sg12 +Vhttp://indextank.com/_static/common/demo/029qs02.jpg +p697 +ssg14 +(dp698 +I0 +I1 +ssg16 +g696 +sg17 +(dp699 +g19 +VBowed string instruments +p700 +ssa(dp701 +g2 +(dp702 +g4 +Vhttp://freebase.com/view/en/kora +p703 +sg6 +S'The kora is a 21-string bridge-harp used extensively by people in West Africa.\nA kora is built from a large calabash cut in half and covered with cow skin to make a resonator, and has a notched bridge like a lute or guitar. It does not fit well into any one category of western instruments and would have to be described as a double bridge harp lute. The sound of a kora resembles that of a harp, though when played in the traditional style, it bears a closer resemblance to flamenco and delta...' +p704 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s1l8p +p705 +sg10 +VKora +p706 +sg12 +Vhttp://indextank.com/_static/common/demo/03s1l8p.jpg +p707 +ssg14 +(dp708 +I0 +I3 +ssg16 +g706 +sg17 +(dp709 +g19 +VPlucked string instrument +p710 +ssa(dp711 +g2 +(dp712 +g4 +Vhttp://freebase.com/view/en/oboe_damore +p713 +sg6 +S"The oboe d'amore (oboe of love in Italian), less commonly oboe d'amour, is a double reed woodwind musical instrument in the oboe family. Slightly larger than the oboe, it has a less assertive and more tranquil and serene tone, and is considered the mezzo-soprano of the oboe family, between the oboe itself (soprano) and the cor anglais, or English horn (alto). It is a transposing instrument, sounding a minor third lower than it is notated, i.e. in A. The bell is pear-shaped and the instrument..." +p714 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d0q36 +p715 +sg10 +VOboe d'amore +p716 +sg12 +Vhttp://indextank.com/_static/common/demo/02d0q36.jpg +p717 +ssg14 +(dp718 +I0 +I1 +ssg16 +g716 +sg17 +(dp719 +g19 +VOboe +p720 +ssa(dp721 +g2 +(dp722 +g4 +Vhttp://freebase.com/view/en/viola_bastarda +p723 +sg6 +S"Viola bastarda refers to a highly virtuosic style of composition or extemporaneous performance, as well as to the altered viols created to maximize players' ability to play in this style. In the viola bastarda style, a polyphonic composition is reduced to a single line, while maintaining the same range as the original, and adding divisions, improvisations, and new counterpoint. The style flourished in Italy in the late 16th and early 17th centuries. Francesco Rognoni, a prominent composer of..." +p724 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02g_6t5 +p725 +sg10 +VViola bastarda +p726 +sg12 +Vhttp://indextank.com/_static/common/demo/02g_6t5.jpg +p727 +ssg14 +(dp728 +I0 +I0 +ssg16 +g726 +sg17 +(dp729 +g19 +VBowed string instruments +p730 +ssa(dp731 +g2 +(dp732 +g4 +Vhttp://freebase.com/view/en/water_organ +p733 +sg6 +S'The water organ or hydraulic organ (early types are sometimes called hydraulis, hydraulos, hydraulus or hydraula) is a type of pipe organ blown by air, where the power source pushing the air is derived by water from a natural source (e.g. by a waterfall) or by a manual pump. Consequently, the water organ lacks a bellows, blower, or compressor.\nOn the water organ, since the 15th century, the water is also used as a source of power to drive a mechanism similar to that of the Barrel organ,...' +p734 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/042v8v8 +p735 +sg10 +VWater organ +p736 +sg12 +Vhttp://indextank.com/_static/common/demo/042v8v8.jpg +p737 +ssg14 +(dp738 +I0 +I0 +ssg16 +g736 +sg17 +(dp739 +g19 +VOrgan +p740 +ssa(dp741 +g2 +(dp742 +g4 +Vhttp://freebase.com/view/en/heckelphone_clarinet +p743 +sg6 +S'The heckelphone-clarinet (or Heckelphon-Klarinette) is a rare woodwind instrument, invented in 1907 by Wilhelm Heckel in Wiesbaden-Biebrich, Germany. Despite its name, it is essentially a wooden saxophone with wide conical bore, built of red-stained maple wood, overblowing the octave, and with clarinet-like fingerings. It has a single-reed mouthpiece attached to a short metal neck, similar to an alto clarinet. The heckelphone-clarinet is a transposing instrument in B\xe2\x99\xad with sounding range of...' +p744 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02h26f8 +p745 +sg10 +VHeckelphone-clarinet +p746 +sg12 +Vhttp://indextank.com/_static/common/demo/02h26f8.jpg +p747 +ssg14 +(dp748 +I0 +I0 +ssg16 +g746 +sg17 +(dp749 +g19 +VSaxophone +p750 +ssa(dp751 +g2 +(dp752 +g4 +Vhttp://freebase.com/view/en/three_hole_pipe +p753 +sg6 +S"The three-hole pipe, also commonly known as Tabor pipe is a wind instrument designed to be played by one hand, leaving the other hand free to play a tabor drum, bell, psalterium or tambourin \xc3\xa0 cordes, bones, triangle or other percussive instrument.\nThe three-hole pipe's origins are not known, but it dates back at least to the 11th Century. \nIt was popular from an early date in France, the Iberian Peninsula and Great Britain and remains in use there today. In the Basque Country it has..." +p754 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03rlw81 +p755 +sg10 +VThree-hole pipe +p756 +sg12 +Vhttp://indextank.com/_static/common/demo/03rlw81.jpg +p757 +ssg14 +(dp758 +I0 +I0 +ssg16 +g756 +sg17 +(dp759 +g19 +VPipe +p760 +ssa(dp761 +g2 +(dp762 +g4 +Vhttp://freebase.com/view/en/natural_trumpet +p763 +sg6 +S'A natural trumpet is a valveless brass instrument that is able to play the notes of the harmonic series.\nThe natural trumpet was used as a military instrument to facilitate communication (e.g. break camp, retreat, etc).\nEven before the early Baroque period the (natural) trumpet had been accepted into Western Art Music. There is evidence, for example, of extensive use of trumpet ensembles in Venetian ceremonial music of the 16th century. Although neither Andrea nor Giovanni Gabrieli wrote...' +p764 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s5hzv +p765 +sg10 +VNatural trumpet +p766 +sg12 +Vhttp://indextank.com/_static/common/demo/03s5hzv.jpg +p767 +ssg14 +(dp768 +I0 +I0 +ssg16 +g766 +sg17 +(dp769 +g19 +VTrumpet +p770 +ssa(dp771 +g2 +(dp772 +g4 +Vhttp://freebase.com/view/en/flugelhorn +p773 +sg6 +S'The flugelhorn (also spelled fluegelhorn, flugel horn or fl\xc3\xbcgelhorn; German: "wing horn") is a brass instrument resembling a trumpet but with a wider, conical bore. Some consider it to be a member of the saxhorn family developed by Adolphe Sax (who also developed the saxophone); however, other historians assert that it derives from the keyed bugle designed by Michael Saurle (father), Munich 1832 (Royal Bavarian privilege for a "chromatic Fl\xc3\xbcgelhorn" 1832), thus predating Adolphe Sax\'s...' +p774 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02914lg +p775 +sg10 +VFlugelhorn +p776 +sg12 +Vhttp://indextank.com/_static/common/demo/02914lg.jpg +p777 +ssg14 +(dp778 +I0 +I36 +ssg16 +g776 +sg17 +(dp779 +g19 +VBrass instrument +p780 +ssa(dp781 +g2 +(dp782 +g4 +Vhttp://freebase.com/view/m/05zsb7k +p783 +sg6 +S'Variants of the bock, a type of bagpipe, were played in Central Europe in what are the modern states of Austria, Germany, Poland and the Czech Republic. The tradition of playing the instrument endured into the 20th century, primarily in the Blata, Chodsko, and Egerland regions of Bohemia, and among the Sorbs of Saxony. The name "Bock" (German for buck, i.e. male goat) refers to the use of goatskins in constructing the bag, similar to the common use of other goat-terms for bagpipes in other...' +p784 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0788zlv +p785 +sg10 +VBock +p786 +sg12 +Vhttp://indextank.com/_static/common/demo/0788zlv.jpg +p787 +ssg14 +(dp788 +I0 +I0 +ssg16 +g786 +sg17 +(dp789 +g19 +VBagpipes +p790 +ssa(dp791 +g2 +(dp792 +g4 +Vhttp://freebase.com/view/en/fretless_guitar +p793 +sg6 +S'A fretless guitar is a guitar without frets. It operates in the same manner as most other stringed instruments and traditional guitars, but does not have any frets to act as the lower end point (node) of the vibrating string. On a fretless guitar, the vibrating string length runs from the bridge, where the strings are attached, all the way up to the point where the fingertip presses the string down on the fingerboard. Fretless guitars are fairly uncommon in most forms of western music and...' +p794 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03qxl43 +p795 +sg10 +VFretless guitar +p796 +sg12 +Vhttp://indextank.com/_static/common/demo/03qxl43.jpg +p797 +ssg14 +(dp798 +I0 +I5 +ssg16 +g796 +sg17 +(dp799 +g19 +VPlucked string instrument +p800 +ssa(dp801 +g2 +(dp802 +g4 +Vhttp://freebase.com/view/en/yueqin +p803 +sg6 +S"This article is about the Chinese Yuequin. The Vietnamese \xc4\x90\xc3\xa0n nguy\xe1\xbb\x87t is also often referred to as a 'moon guitar'.\nThe yueqin (Chinese: \xe6\x9c\x88\xe7\x90\xb4, pinyin: yu\xc3\xa8q\xc3\xadn; also spelled yue qin, or yueh-ch'in; and also called moon guitar, moon-zither, gekkin, la ch'in, or laqin) is a traditional Chinese string instrument. It is a lute with a round, hollow wooden body which gives it the nickname moon guitar. It has a short fretted neck and four strings tuned in courses of two (each pair of strings is tuned to..." +p804 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02ctnn9 +p805 +sg10 +VYueqin +p806 +sg12 +Vhttp://indextank.com/_static/common/demo/02ctnn9.jpg +p807 +ssg14 +(dp808 +I0 +I1 +ssg16 +g806 +sg17 +(dp809 +g19 +VPlucked string instrument +p810 +ssa(dp811 +g2 +(dp812 +g4 +Vhttp://freebase.com/view/en/alto_flute +p813 +sg6 +S"The alto flute is a type of Western concert flute, a musical instrument in the woodwind family. It is the next extension downward of the C flute after the fl\xc3\xbbte d'amour. It is characterized by its distinct, mellow tone in the lower portion of its range. It is a transposing instrument in G and, like the piccolo and bass flute, uses the same fingerings as the C flute.\nThe tube of the alto flute is considerably thicker and longer than a C flute and requires more breath from the player. This..." +p814 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dwyx3 +p815 +sg10 +VAlto flute +p816 +sg12 +Vhttp://indextank.com/_static/common/demo/02dwyx3.jpg +p817 +ssg14 +(dp818 +I0 +I1 +ssg16 +g816 +sg17 +(dp819 +g19 +VFlute (transverse) +p820 +ssa(dp821 +g2 +(dp822 +g4 +Vhttp://freebase.com/view/en/lyra_viol +p823 +sg6 +S'The lyra viol is a small bass viol, used primarily in England in the seventeenth century.\nWhile the instrument itself differs little physically from the standard consort viol, there is a large and important repertoire which was developed specifically for the lyra viol. Due to the number of strings and their rather flat layout, the lyra viol can approximate polyphonic textures, and because of its small size and large range, it is more suited to intricate and quick melodic lines than the...' +p824 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02gw6kc +p825 +sg10 +VLyra viol +p826 +sg12 +Vhttp://indextank.com/_static/common/demo/02gw6kc.jpg +p827 +ssg14 +(dp828 +I0 +I0 +ssg16 +g826 +sg17 +(dp829 +g19 +VViol +p830 +ssa(dp831 +g2 +(dp832 +g4 +Vhttp://freebase.com/view/m/07_hbw +p833 +sg6 +S'Setar (Persian: \xd8\xb3\xd9\x87 \xe2\x80\x8c\xd8\xaa\xd8\xa7\xd8\xb1, from seh, meaning "three" and t\xc4\x81r, meaning "string") is a Persian musical instrument. It is a member of the lute family. Two and a half centuries ago, a fourth string was added to the setar, which has 25 - 27 moveable frets. It originated in Persia before the spread of Islam.' +p834 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c_r2z +p835 +sg10 +VSetar +p836 +sg12 +Vhttp://indextank.com/_static/common/demo/02c_r2z.jpg +p837 +ssg14 +(dp838 +I0 +I10 +ssg16 +g836 +sg17 +(dp839 +g19 +VLute +p840 +ssa(dp841 +g2 +(dp842 +g4 +Vhttp://freebase.com/view/en/alto_horn +p843 +sg6 +S'The alto horn (US English; tenor horn in British English, Althorn in Germany; occasionally referred to as E\xe2\x99\xad horn) is a brass instrument pitched in E\xe2\x99\xad. It has a predominantly conical bore (most tube extents gradually widening), and normally uses a deep, cornet-like mouthpiece.\nIt is most commonly used in marching bands, brass bands and similar ensembles, whereas the horn tends to take the corresponding parts in symphonic groupings and classical brass ensembles.\nThe alto horn is a valved...' +p844 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029hg8p +p845 +sg10 +VAlto horn +p846 +sg12 +Vhttp://indextank.com/_static/common/demo/029hg8p.jpg +p847 +ssg14 +(dp848 +I0 +I1 +ssg16 +g846 +sg17 +(dp849 +g19 +VBrass instrument +p850 +ssa(dp851 +g2 +(dp852 +g4 +Vhttp://freebase.com/view/en/torupill +p853 +sg6 +S"The torupill (literally 'pipe instrument'; also known as kitsepill, lootspill, kotepill) is a type of bagpipe from Estonia.\nIt is not clear when the bagpipe became established in Estonia. It may have arrived with the Germans, but an analysis of the bagpipe tunes in West and North Estonia also show a strong Swedish influence.\nThe instrument was known throughout Estonia. The bagpipe tradition was longest preserved in West and North Estonia where folk music retained archaic characteristics for..." +p854 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/063cblf +p855 +sg10 +VTorupill +p856 +sg12 +Vhttp://indextank.com/_static/common/demo/063cblf.jpg +p857 +ssg14 +(dp858 +I0 +I0 +ssg16 +g856 +sg17 +(dp859 +g19 +VBagpipes +p860 +ssa(dp861 +g2 +(dp862 +g4 +Vhttp://freebase.com/view/m/02wylsp +p863 +sg6 +S'The baglamas (Greek \xce\xbc\xcf\x80\xce\xb1\xce\xb3\xce\xbb\xce\xb1\xce\xbc\xce\xac\xcf\x82) or baglamadaki (Greek \xce\xbc\xcf\x80\xce\xb1\xce\xb3\xce\xbb\xce\xb1\xce\xbc\xce\xb1\xce\xb4\xce\xac\xce\xba\xce\xb9), a long necked bowl-lute, is a plucked string instrument used in Greek music; it is a version of the bouzouki pitched an octave higher (nominally D-A-D), with unison pairs on the four highest strings and an octave pair on the lower D. Musically, the baglamas is most often found supporting the bouzouki in the Piraeus style of rembetika.\nThe body is often hollowed out from a piece of wood (skaftos construction) or else made...' +p864 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05kpl3f +p865 +sg10 +VBaglama +p866 +sg12 +Vhttp://indextank.com/_static/common/demo/05kpl3f.jpg +p867 +ssg14 +(dp868 +I0 +I0 +ssg16 +g866 +sg17 +(dp869 +g19 +VPlucked string instrument +p870 +ssa(dp871 +g2 +(dp872 +g4 +Vhttp://freebase.com/view/en/soprano_clarinet +p873 +sg6 +S'The soprano clarinets are a sub-family of the clarinet family. They include the most common types of clarinets, and indeed are often referred to as simply "clarinets".\nAmong the soprano clarinets are the B\xe2\x99\xad clarinet, the most common type, whose range extends from D below middle C (written E) to about the C three octaves above middle C; the A and C clarinets, sounding respectively a semitone lower and a whole tone higher than the B\xe2\x99\xad clarinet; and the low G clarinet, sounding yet a whole tone...' +p874 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bctx9 +p875 +sg10 +VSoprano clarinet +p876 +sg12 +Vhttp://indextank.com/_static/common/demo/02bctx9.jpg +p877 +ssg14 +(dp878 +I0 +I0 +ssg16 +g876 +sg17 +(dp879 +g19 +VClarinet +p880 +ssa(dp881 +g2 +(dp882 +g4 +Vhttp://freebase.com/view/en/rudra_veena +p883 +sg6 +S'The rudra veena (also spelled rudra vina, and also called been or bin; Hindi: \xe0\xa4\xb0\xe0\xa5\x81\xe0\xa4\xa6\xe0\xa5\x8d\xe0\xa4\xb0\xe0\xa4\xb5\xe0\xa5\x80\xe0\xa4\xa3\xe0\xa4\xbe) is a large plucked string instrument used in Hindustani classical music. It is an ancient instrument rarely played today. The rudra veena declined in popularity in part due to the introduction of the surbahar in the early 19th century which allowed sitarists to more easily present the alap sections of slow dhrupad-style ragas.\nThe rudra veena has a long tubular body with a length ranging between 54 and...' +p884 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02ddhg6 +p885 +sg10 +VRudra veena +p886 +sg12 +Vhttp://indextank.com/_static/common/demo/02ddhg6.jpg +p887 +ssg14 +(dp888 +I0 +I1 +ssg16 +g886 +sg17 +(dp889 +g19 +VPlucked string instrument +p890 +ssa(dp891 +g2 +(dp892 +g4 +Vhttp://freebase.com/view/en/gaida +p893 +sg6 +S'The gaida (bagpipe) is a musical instrument, aerophone, using enclosed reeds fed from a constant reservoir of air in the form of a bag.\nThe gaida, and its variations, is a traditional musical instrument for entire Europe, Northern Africa and the Middle East.\nThe several variations of gaida are in: Albania (Gajde) Czech Republic (bock), Romania (cimpoi), Croatia (diple and surle), Hungary (duda), Slovenia (dude), Poland (duda, gaidu and koza), Russia (mih, sahrb, volinka and shapar), Turkey...' +p894 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dtz_4 +p895 +sg10 +VGaida +p896 +sg12 +Vhttp://indextank.com/_static/common/demo/02dtz_4.jpg +p897 +ssg14 +(dp898 +I0 +I1 +ssg16 +g896 +sg17 +(dp899 +g19 +VBagpipes +p900 +ssa(dp901 +g2 +(dp902 +g4 +Vhttp://freebase.com/view/en/reclam_de_xeremies +p903 +sg6 +S'The reclam de xeremies, also known as the xeremia bessona or xeremieta, is a double clarinet with two single reeds, traditionally found on the Balearic island of Ibiza, off the east coast of Spain.\nIt consists of two cane tubes of equal length, bound together by cord and small pieces of lead to stabilise the tubes. On each tube are several finger holes, traditionally four in the front and one on the back, though in modern instruments the back hole is often omitted. At the top end of each...' +p904 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0786zyh +p905 +sg10 +VReclam de xeremies +p906 +sg12 +Vhttp://indextank.com/_static/common/demo/0786zyh.jpg +p907 +ssg14 +(dp908 +I0 +I0 +ssg16 +g906 +sg17 +(dp909 +g19 +VWoodwind instrument +p910 +ssa(dp911 +g2 +(dp912 +g4 +Vhttp://freebase.com/view/en/chalumeau +p913 +sg6 +S'This article is about the historical musical instrument. For the register on the clarinet that is named for this instrument, see Clarinet#Range.\nThe chalumeau (plural chalumeaux; from Greek: \xce\xba\xce\xac\xce\xbb\xce\xb1\xce\xbc\xce\xbf\xcf\x82, kalamos, meaning "reed") is a woodwind instrument of the late baroque and early classical era, in appearance rather like a recorder, but with a mouthpiece like a clarinet\'s.\nThe word "chalumeau" was in use in French from the twelfth century to refer to various sorts of pipes, some of which were...' +p914 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/07b05t1 +p915 +sg10 +VChalumeau +p916 +sg12 +Vhttp://indextank.com/_static/common/demo/07b05t1.jpg +p917 +ssg14 +(dp918 +I0 +I0 +ssg16 +g916 +sg17 +(dp919 +g19 +VClarinet +p920 +ssa(dp921 +g2 +(dp922 +g4 +Vhttp://freebase.com/view/en/square_piano +p923 +sg6 +S"The square piano is a piano that has horizontal strings arranged diagonally across the rectangular case above the hammers and with the keyboard set in the long side. It is variously attributed to Silbermann and Frederici and was improved by Petzold and Babcock. Built in quantity through the 1890s (in the United States), Steinway's celebrated iron framed over strung squares were more than two and a half times the size of Zumpe's wood framed instruments that were successful a century before...." +p924 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02h0hp7 +p925 +sg10 +VSquare piano +p926 +sg12 +Vhttp://indextank.com/_static/common/demo/02h0hp7.jpg +p927 +ssg14 +(dp928 +I0 +I0 +ssg16 +g926 +sg17 +(dp929 +g19 +VPiano +p930 +ssa(dp931 +g2 +(dp932 +g4 +Vhttp://freebase.com/view/en/gaita_asturiana +p933 +sg6 +S'The gaita asturiana is a type of bagpipe native to the autonomous communities of Asturias and parts of Cantabria on the northern coast of Spain.\nThe first evidence for the existence of the gaita asturiana dates back to the 13th century, as a piper can be seen carved into the capital of the church of Santa Mar\xc3\xada de Villaviciosa. Further evidence includes an illumination of a rabbit playing the gaita in the 14th century text Llibru la regla colorada. An early carving of a wild boar playing the...' +p934 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0638m3c +p935 +sg10 +VGaita asturiana +p936 +sg12 +Vhttp://indextank.com/_static/common/demo/0638m3c.jpg +p937 +ssg14 +(dp938 +I0 +I0 +ssg16 +g936 +sg17 +(dp939 +g19 +VBagpipes +p940 +ssa(dp941 +g2 +(dp942 +g4 +Vhttp://freebase.com/view/en/yamaha_dx7 +p943 +sg6 +S'The Yamaha DX7 is an FM Digital Synthesizer manufactured by the Yamaha Corporation from 1983 to 1986. It was the first commercially successful digital synthesizer. Its distinctive sound can be heard on many recordings, especially Pop music from the 1980s. The DX7 was the moderately priced model of the DX series of FM keyboards that included DX9, the smaller DX100, DX11, and DX21 and the larger DX1 and DX5. Over 160,000 DX7s were made.\nTone generation in the DX7 is based on linear Frequency...' +p944 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03sz4t7 +p945 +sg10 +VYamaha DX7 +p946 +sg12 +Vhttp://indextank.com/_static/common/demo/03sz4t7.jpg +p947 +ssg14 +(dp948 +I0 +I0 +ssg16 +g946 +sg17 +(dp949 +g19 +VSynthesizer +p950 +ssa(dp951 +g2 +(dp952 +g4 +Vhttp://freebase.com/view/en/rebec +p953 +sg6 +S'The rebec (sometimes rebeck, and originally various other spellings) is a bowed string musical instrument. In its most common form, it has narrowboat shaped body, three strings and is played on the arm or under the chin, like a violin.\nThe rebec dates back to the Middle Ages and was particularly popular in the 15th and 16th centuries. The instrument is European and derived from the Arabic bowed instrument rebab and the Byzantine lyra. The rebec was first referred to by that name around the...' +p954 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02b_96j +p955 +sg10 +VRebec +p956 +sg12 +Vhttp://indextank.com/_static/common/demo/02b_96j.jpg +p957 +ssg14 +(dp958 +I0 +I0 +ssg16 +g956 +sg17 +(dp959 +g19 +VBowed string instruments +p960 +ssa(dp961 +g2 +(dp962 +g4 +Vhttp://freebase.com/view/en/tin_whistle +p963 +sg6 +S'The tin whistle also called the penny whistle, English Flageolet, Scottish penny whistle, Tin Flageolet, Irish whistle and Clarke London Flageolet is a simple six-holed woodwind instrument. It is an end blown fipple flute flageolet, putting it in the same category as the recorder, American Indian flute, and other woodwind instruments. A tin whistle player is called a tin whistler or whistler. The tin whistle is closely associated with Celtic music.\nThe penny whistle in its modern form stems...' +p964 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0292329 +p965 +sg10 +VTin whistle +p966 +sg12 +Vhttp://indextank.com/_static/common/demo/0292329.jpg +p967 +ssg14 +(dp968 +I0 +I41 +ssg16 +g966 +sg17 +(dp969 +g19 +VFlute (transverse) +p970 +ssa(dp971 +g2 +(dp972 +g4 +Vhttp://freebase.com/view/en/an_gao +p973 +sg6 +S'The \xc4\x91\xc3\xa0n g\xc3\xa1o is a Vietnamese bowed string instrument with two strings. Its body is made from half of a coconut shell covered with wood, with a small seashell used as bridge. The instrument\'s name literally means "coconut shell dipper string instrument" (\xc4\x91\xc3\xa0n is the generic term for "string instrument" and g\xc3\xa1o means "coconut shell dipper").\nThe \xc4\x91\xc3\xa0n g\xc3\xa1o is closely related to a similar Chinese instrument, the yehu, and was likely introduced to Vietnam by Chaozhou or Cantonese immigrants. It is...' +p974 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04p94q0 +p975 +sg10 +V\u0110àn gáo +p976 +sg12 +Vhttp://indextank.com/_static/common/demo/04p94q0.jpg +p977 +ssg14 +(dp978 +I0 +I0 +ssg16 +g976 +sg17 +(dp979 +g19 +VBowed string instruments +p980 +ssa(dp981 +g2 +(dp982 +g4 +Vhttp://freebase.com/view/en/basler_drum +p983 +sg6 +S'The Basler drum is a kind of snare drum traditionally used in Switzerland for marching music, and notably at the Carnival of Basel.\nIt has a height of between 40 and 60\xc2\xa0cm and a diameter of about 40\xc2\xa0cm.' +p984 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d3fr0 +p985 +sg10 +VBasler drum +p986 +sg12 +Vhttp://indextank.com/_static/common/demo/02d3fr0.jpg +p987 +ssg14 +(dp988 +I0 +I0 +ssg16 +g986 +sg17 +(dp989 +g19 +VPercussion +p990 +ssa(dp991 +g2 +(dp992 +g4 +Vhttp://freebase.com/view/en/baritone_horn +p993 +sg6 +S"The baritone horn is a member of the brass instrument family. The baritone horn has a predominantly cylindrical bore as do the trumpet and trombone. A baritone horn uses a large mouthpiece much like those of a trombone or euphonium. It is pitched in B\xe2\x99\xad, one octave below the B\xe2\x99\xad trumpet. In the UK the baritone is frequently found in brass bands. The baritone horn is also a common instrument in high school and college bands, as older baritones are often found in schools' inventories. However,..." +p994 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0293f7p +p995 +sg10 +VBaritone horn +p996 +sg12 +Vhttp://indextank.com/_static/common/demo/0293f7p.jpg +p997 +ssg14 +(dp998 +I0 +I3 +ssg16 +g996 +sg17 +(dp999 +g19 +VBrass instrument +p1000 +ssa(dp1001 +g2 +(dp1002 +g4 +Vhttp://freebase.com/view/en/temple_block +p1003 +sg6 +S'The temple block is a percussion instrument originating in China, Japan and Korea where it is used in religious ceremonies.\nIt is a carved hollow wooden instrument with a large slit. In its traditional form, the wooden fish, the shape is somewhat bulbous; modern instruments are also used which are rectangular in shape. Several blocks of varying sizes are often used together to give a variety of pitches. In Western music, their use can be traced back to early jazz drummers, and they are not...' +p1004 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0c52jjc +p1005 +sg10 +VTemple block +p1006 +sg12 +Vhttp://indextank.com/_static/common/demo/0c52jjc.jpg +p1007 +ssg14 +(dp1008 +I0 +I0 +ssg16 +g1006 +sg17 +(dp1009 +g19 +VPercussion +p1010 +ssa(dp1011 +g2 +(dp1012 +g4 +Vhttp://freebase.com/view/en/rainstick +p1013 +sg6 +S'A rainstick is a long, hollow tube partially filled with small pebbles or beans, and has small pins or thorns arranged helically on its inside surface. When the stick is upended, the pebbles fall to the other end of the tube, making a sound reminiscent of rain falling. Rainsticks are often sold to tourists visiting parts of Latin America.\nThe rainstick is believed to have been invented in Chile or Peru, and was played in the belief that it could bring about rainstorms. It is also said that...' +p1014 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d3lbb +p1015 +sg10 +VRainstick +p1016 +sg12 +Vhttp://indextank.com/_static/common/demo/02d3lbb.jpg +p1017 +ssg14 +(dp1018 +I0 +I0 +ssg16 +g1016 +sg17 +(dp1019 +g19 +VPercussion +p1020 +ssa(dp1021 +g2 +(dp1022 +g4 +Vhttp://freebase.com/view/en/ophicleide +p1023 +sg6 +S'The ophicleide (pronounced /\xcb\x88\xc9\x92f\xc9\xa8kla\xc9\xaad/) is a family of conical bore, brass keyed-bugles. It has a similar shape to the sudrophone.\nThe ophicleide was invented in 1817 and patented in 1821 by French instrument maker Jean Hilaire Ast\xc3\xa9 (also known as Halary or Haleri) as an extension to the keyed bugle or Royal Kent bugle family. It was the structural cornerstone of the brass section of the Romantic orchestra, often replacing the serpent, a Renaissance instrument which was thought to be...' +p1024 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s7lcd +p1025 +sg10 +VOphicleide +p1026 +sg12 +Vhttp://indextank.com/_static/common/demo/03s7lcd.jpg +p1027 +ssg14 +(dp1028 +I0 +I0 +ssg16 +g1026 +sg17 +(dp1029 +g19 +VWind instrument +p1030 +ssa(dp1031 +g2 +(dp1032 +g4 +Vhttp://freebase.com/view/en/keyed_trumpet +p1033 +sg6 +S'The keyed trumpet is a brass instrument that, contrary to the traditional valved trumpet, uses keys. The keyed trumpet is rarely seen in modern performances, but was relatively common up until the introduction of the valved trumpet in the early nineteenth century. Prior to the invention of the keyed trumpet, the prominent trumpet of the time was the natural trumpet.\nThe keyed trumpet has holes in the wall of the tube that are closed by keys. The experimental E\xe2\x99\xad keyed trumpet was not confined...' +p1034 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0633w7q +p1035 +sg10 +VKeyed trumpet +p1036 +sg12 +Vhttp://indextank.com/_static/common/demo/0633w7q.jpg +p1037 +ssg14 +(dp1038 +I0 +I0 +ssg16 +g1036 +sg17 +(dp1039 +g19 +VTrumpet +p1040 +ssa(dp1041 +g2 +(dp1042 +g4 +Vhttp://freebase.com/view/en/guqin +p1043 +sg6 +S'The guqin (simplified/traditional: \xe5\x8f\xa4\xe7\x90\xb4; pinyin: g\xc7\x94q\xc3\xadn; Wades-Giles ku-ch\'in; pronounced\xc2\xa0[k\xc3\xb9t\xc9\x95\xca\xb0\xc7\x90n]\xc2\xa0 ( listen); literally "ancient stringed instrument") is the modern name for a plucked seven-string Chinese musical instrument of the zither family. It has been played since ancient times, and has traditionally been favored by scholars and literati as an instrument of great subtlety and refinement, as highlighted by the quote "a gentleman does not part with his qin or se without good reason," as...' +p1044 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02b89d9 +p1045 +sg10 +VGuqin +p1046 +sg12 +Vhttp://indextank.com/_static/common/demo/02b89d9.jpg +p1047 +ssg14 +(dp1048 +I0 +I0 +ssg16 +g1046 +sg17 +(dp1049 +g19 +VPlucked string instrument +p1050 +ssa(dp1051 +g2 +(dp1052 +g4 +Vhttp://freebase.com/view/en/tubular_bell +p1053 +sg6 +S'Tubular bells (also known as chimes) are musical instruments in the percussion family. Each bell is a metal tube, 30\xe2\x80\x9338 mm (1\xc2\xbc\xe2\x80\x931\xc2\xbd inches) in diameter, tuned by altering its length. Tubular bells are often replaced by studio chimes, which are a smaller and usually less expensive instrument. Studio chimes are similar in appearance to tubular bells, but each bell has a smaller diameter than the corresponding bell on tubular bells.\nTubular bells are typically struck on the top edge of the tube...' +p1054 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029g1m6 +p1055 +sg10 +VTubular bell +p1056 +sg12 +Vhttp://indextank.com/_static/common/demo/029g1m6.jpg +p1057 +ssg14 +(dp1058 +I0 +I4 +ssg16 +g1056 +sg17 +(dp1059 +g19 +VPercussion +p1060 +ssa(dp1061 +g2 +(dp1062 +g4 +Vhttp://freebase.com/view/en/vox_continental +p1063 +sg6 +S'The Vox Continental is a transistor-based combo organ that was introduced in 1962. Known for its thin, bright, breathy sound, the "Connie," as it was affectionately known, was designed to be used by touring musicians. It was also designed to replace heavy tonewheel organs, such as the Hammond B3.\nWhile this was not entirely accomplished, the Continental was used in many 1960s hit singles, and was probably the most popular and best-known combo organ among major acts. Although phased out of...' +p1064 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03t79cq +p1065 +sg10 +VVox Continental +p1066 +sg12 +Vhttp://indextank.com/_static/common/demo/03t79cq.jpg +p1067 +ssg14 +(dp1068 +I0 +I0 +ssg16 +g1066 +sg17 +(dp1069 +g19 +VElectronic organ +p1070 +ssa(dp1071 +g2 +(dp1072 +g4 +Vhttp://freebase.com/view/en/baryton +p1073 +sg6 +S'The baryton is a bowed string instrument in the viol family, in regular use in Europe up until the end of the 18th century. In London a performance at Marylebone Gardens was announced in 1744, when Mr Ferrand was to perform on "the Pariton, an instrument never played on in publick before." It most likely fell out of favor due to its immense difficulty to play. Its size is comparable to that of a violoncello; it has seven or sometimes six bowed strings of gut, plus ten sympathetic wire...' +p1074 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03t02nm +p1075 +sg10 +VBaryton +p1076 +sg12 +Vhttp://indextank.com/_static/common/demo/03t02nm.jpg +p1077 +ssg14 +(dp1078 +I0 +I0 +ssg16 +g1076 +sg17 +(dp1079 +g19 +VBowed string instruments +p1080 +ssa(dp1081 +g2 +(dp1082 +g4 +Vhttp://freebase.com/view/en/electric_upright_bass +p1083 +sg6 +S"The electric upright bass (abbreviated EUB and sometimes also called stick bass) is an electronically amplified version of the double bass that has a minimal or 'skeleton' body, which greatly reduces the size and weight of the instrument. The EUB retains enough of the features of the double bass so that double bass players are comfortable performing on it. While the EUB retains some of the tonal characteristics of the double bass, its electrically-amplified nature also gives it its own..." +p1084 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s1c6v +p1085 +sg10 +VElectric upright bass +p1086 +sg12 +Vhttp://indextank.com/_static/common/demo/03s1c6v.jpg +p1087 +ssg14 +(dp1088 +I0 +I6 +ssg16 +g1086 +sg17 +(dp1089 +g19 +VDouble bass +p1090 +ssa(dp1091 +g2 +(dp1092 +g4 +Vhttp://freebase.com/view/en/veena +p1093 +sg6 +S"Veena (also spelled 'vina', Sanskrit: \xe0\xa4\xb5\xe0\xa5\x80\xe0\xa4\xa3\xe0\xa4\xbe (v\xc4\xab\xe1\xb9\x87\xc4\x81), Tamil: \xe0\xae\xb5\xe0\xaf\x80\xe0\xae\xa3\xe0\xaf\x88, Kannada: \xe0\xb2\xb5\xe0\xb3\x80\xe0\xb2\xa3\xe0\xb3\x86, Malayalam: \xe0\xb4\xb5\xe0\xb5\x80\xe0\xb4\xa3, Telugu: \xe0\xb0\xb5\xe0\xb1\x80\xe0\xb0\xa3) is a plucked stringed instrument used mostly in Carnatic Indian classical music. There are several variations of the veena, which in its South Indian form is a member of the lute family. One who plays the veena is referred to as a vainika.\nThe veena has a recorded history that dates back to the Vedic period (approximately 1500 BCE)\nIn ancient times, the tone vibrating from the hunter's..." +p1094 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cz_w2 +p1095 +sg10 +VVeena +p1096 +sg12 +Vhttp://indextank.com/_static/common/demo/02cz_w2.jpg +p1097 +ssg14 +(dp1098 +I0 +I2 +ssg16 +g1096 +sg17 +(dp1099 +g19 +VPlucked string instrument +p1100 +ssa(dp1101 +g2 +(dp1102 +g4 +Vhttp://freebase.com/view/en/clarsach +p1103 +sg6 +S"Cl\xc3\xa0rsach or Cl\xc3\xa1irseach (depending on Scottish Gaelic or Irish spellings), is the generic Gaelic word for 'a harp', as derived from Middle Irish. In English, the word is used to refer specifically to a variety of small Irish and Scottish harps.\nThe use of this word in English, and the varieties of harps that it describes, is very complex and is a cause of arguments or disagreements between different groups of harp-lovers.\nBy and large, in English, the word cl\xc3\xa0rsach is equivalent to the term..." +p1104 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dp7mp +p1105 +sg10 +VClàrsach +p1106 +sg12 +Vhttp://indextank.com/_static/common/demo/02dp7mp.jpg +p1107 +ssg14 +(dp1108 +I0 +I3 +ssg16 +g1106 +sg17 +(dp1109 +g19 +VPlucked string instrument +p1110 +ssa(dp1111 +g2 +(dp1112 +g4 +Vhttp://freebase.com/view/en/tamburitza +p1113 +sg6 +S'Tamburica (pronounced /t\xc3\xa6m\xcb\x88b\xca\x8a\xc9\x99r\xc9\xaats\xc9\x99/ or /\xcb\x8ct\xc3\xa6mb\xc9\x99\xcb\x88r\xc9\xaats\xc9\x99/) or Tamboura (Croatian: Tamburica, Serbian: \xd0\xa2\xd0\xb0\xd0\xbc\xd0\xb1\xd1\x83\xd1\x80\xd0\xb8\xd1\x86\xd0\xb0, Tamburica, meaning Little Tamboura, Hungarian: Tambura, Greek: \xce\xa4\xce\xb1\xce\xbc\xcf\x80\xce\xbf\xcf\x85\xcf\x81\xce\xac\xcf\x82, sometimes written tamburrizza) refers to any member of a family of long-necked lutes popular in Eastern and Southern Europe, particularly Croatia (especially Slavonia), northern Serbia (Vojvodina) and Hungary. It is also known in southern Slovenia and Burgenland. All took their name and some characteristics...' +p1114 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04pmtc5 +p1115 +sg10 +VTamburitza +p1116 +sg12 +Vhttp://indextank.com/_static/common/demo/04pmtc5.jpg +p1117 +ssg14 +(dp1118 +I0 +I3 +ssg16 +g1116 +sg17 +(dp1119 +g19 +VPlucked string instrument +p1120 +ssa(dp1121 +g2 +(dp1122 +g4 +Vhttp://freebase.com/view/en/baroque_violin +p1123 +sg6 +S'A baroque violin is, in common usage, any violin whose neck, fingerboard, bridge, and tailpiece are of the type used during the baroque period. Such an instrument may be an original built during the baroque and never changed to modern form; or a modern replica built as a baroque violin; or an older instrument which has been converted (or re-converted) to baroque form. "Baroque cellos" and "baroque violas" also exist, with similar modifications made to their form.\nFollowing period practices,...' +p1124 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bxp8b +p1125 +sg10 +VBaroque violin +p1126 +sg12 +Vhttp://indextank.com/_static/common/demo/02bxp8b.jpg +p1127 +ssg14 +(dp1128 +I0 +I0 +ssg16 +g1126 +sg17 +(dp1129 +g19 +VViolin +p1130 +ssa(dp1131 +g2 +(dp1132 +g4 +Vhttp://freebase.com/view/en/cavaquinho +p1133 +sg6 +S'The cavaquinho (pronounced [kav\xc9\x90\xcb\x88ki\xc9\xb2u] in Portuguese) is a small string instrument of the European guitar family with four wire or gut strings. It is also called machimbo, machim, machete (in the Portuguese Atlantic islands and Brazil), manchete or marchete, braguinha or braguinho, or cavaco.\nThe most common tuning is D-G-B-D (from lower to higher pitches); other tunings include D-A-B-E (Portuguese ancient tuning, made popular by Julio Pereira) and G-G-B-D and A-A-C#-E. Guitarists often use...' +p1134 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d9kx8 +p1135 +sg10 +VCavaquinho +p1136 +sg12 +Vhttp://indextank.com/_static/common/demo/02d9kx8.jpg +p1137 +ssg14 +(dp1138 +I0 +I4 +ssg16 +g1136 +sg17 +(dp1139 +g19 +VPlucked string instrument +p1140 +ssa(dp1141 +g2 +(dp1142 +g4 +Vhttp://freebase.com/view/en/sarinda +p1143 +sg6 +S'A sarinda is a stringed Indian folk musical instrument similar to lutes or fiddles. It is played with a bow and has three strings. The bottom part of the front of its hollow wooden soundbox is covered with animal skin. It is played while sitting on the ground in a vertical orientation.\nThe sarinda seems to have its origin in tribal fiddle instruments called "dhodro banam" found throughout in central, north-western and eastern India. It is an important instrument in the culture of the...' +p1144 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02fwz73 +p1145 +sg10 +VSarinda +p1146 +sg12 +Vhttp://indextank.com/_static/common/demo/02fwz73.jpg +p1147 +ssg14 +(dp1148 +I0 +I0 +ssg16 +g1146 +sg17 +(dp1149 +g19 +VBowed string instruments +p1150 +ssa(dp1151 +g2 +(dp1152 +g4 +Vhttp://freebase.com/view/en/zil +p1153 +sg6 +S'Zills, also zils or finger cymbals, (from Turkish zil, "cymbals" ) are tiny metallic cymbals used in belly dancing and similar performances. They are called s\xc4\x81j\xc4\x81t (\xd8\xb5\xd8\xa7\xd8\xac\xd8\xa7\xd8\xaa) in Arabic. They are similar to Tibetan tingsha bells.\nA set of zills consists of four cymbals, two for each hand. Modern zills come in a range of sizes, the most common having a diameter of about 5\xc2\xa0cm (2\xc2\xa0in). Different sizes and shapes of zills will produce sounds that differ in volume, tone and resonance. For instance, a...' +p1154 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03td8pl +p1155 +sg10 +VZil +p1156 +sg12 +Vhttp://indextank.com/_static/common/demo/03td8pl.jpg +p1157 +ssg14 +(dp1158 +I0 +I3 +ssg16 +g1156 +sg17 +(dp1159 +g19 +VCymbal +p1160 +ssa(dp1161 +g2 +(dp1162 +g4 +Vhttp://freebase.com/view/en/monochord +p1163 +sg6 +S'A monochord is an ancient musical and scientific laboratory instrument. The word "monochord" comes from the Greek and means literally "one string." A misconception of the term lies within its name. Often a monochord has more than one string, most of the time two, one open string and a second string with a movable bridge. In a basic monochord, a single string is stretched over a sound box. The string is fixed at both ends while one or many movable bridges are manipulated to demonstrate...' +p1164 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cx4g8 +p1165 +sg10 +VMonochord +p1166 +sg12 +Vhttp://indextank.com/_static/common/demo/02cx4g8.jpg +p1167 +ssg14 +(dp1168 +I0 +I0 +ssg16 +g1166 +sg17 +(dp1169 +g19 +VPlucked string instrument +p1170 +ssa(dp1171 +g2 +(dp1172 +g4 +Vhttp://freebase.com/view/en/marimba +p1173 +sg6 +S'The marimba ( pronunciation (help\xc2\xb7info)) (also: Marimbaphone) is a musical instrument in the percussion family. Keys or bars (usually made of wood) are struck with mallets to produce musical tones. The keys are arranged as those of a piano, with the accidentals raised vertically and overlapping the natural keys (similar to a piano) to aid the performer both visually and physically.\nThe chromatic marimba was developed in southern Mexico and northern Guatemala from the diatonic marimba, an...' +p1174 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03sw9r1 +p1175 +sg10 +VMarimba +p1176 +sg12 +Vhttp://indextank.com/_static/common/demo/03sw9r1.jpg +p1177 +ssg14 +(dp1178 +I0 +I43 +ssg16 +g1176 +sg17 +(dp1179 +g19 +VPercussion +p1180 +ssa(dp1181 +g2 +(dp1182 +g4 +Vhttp://freebase.com/view/en/ajaeng +p1183 +sg6 +S'The ajaeng is a Korean string instrument. It is a wide zither with strings made of twisted silk, played by means of a slender stick made of forsythia wood, which is scraped against the strings in the manner of a bow. The original version of the instrument, and that used in court music (called the jeongak ajaeng), has seven strings, while the ajaeng used for sanjo and sinawi (called the sanjo ajaeng) has eight strings; some instruments may have up to nine strings.\nThe ajaeng is generally...' +p1184 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044jqcy +p1185 +sg10 +VAjaeng +p1186 +sg12 +Vhttp://indextank.com/_static/common/demo/044jqcy.jpg +p1187 +ssg14 +(dp1188 +I0 +I0 +ssg16 +g1186 +sg17 +(dp1189 +g19 +VBowed string instruments +p1190 +ssa(dp1191 +g2 +(dp1192 +g4 +Vhttp://freebase.com/view/en/cymbal +p1193 +sg6 +S'Cymbals are a common percussion instrument. Cymbals consist of thin, normally round plates of various alloys; see cymbal making for a discussion of their manufacture. The greater majority of cymbals are of indefinite pitch, although small disc-shaped cymbals based on ancient designs sound a definite note (see: crotales). Cymbals are used in many ensembles ranging from the orchestra, percussion ensembles, jazz bands, heavy metal bands, and marching groups. Drum kits usually incorporate a...' +p1194 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0290zn9 +p1195 +sg10 +VCymbal +p1196 +sg12 +Vhttp://indextank.com/_static/common/demo/0290zn9.jpg +p1197 +ssg14 +(dp1198 +I0 +I2 +ssg16 +g1196 +sg17 +(dp1199 +g19 +VPercussion +p1200 +ssa(dp1201 +g2 +(dp1202 +g4 +Vhttp://freebase.com/view/en/bass_oboe +p1203 +sg6 +S'The bass oboe or baritone oboe is a double reed instrument in the woodwind family. It is about twice the size of a regular (soprano) oboe and sounds an octave lower; it has a deep, full tone not unlike that of its higher-pitched cousin, the English horn. The bass oboe is notated in the treble clef, sounding one octave lower than written. Its lowest note is B2 (in scientific pitch notation), one octave and a semitone below middle C, although an extension may be inserted between the lower...' +p1204 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041gwbz +p1205 +sg10 +VBass oboe +p1206 +sg12 +Vhttp://indextank.com/_static/common/demo/041gwbz.jpg +p1207 +ssg14 +(dp1208 +I0 +I0 +ssg16 +g1206 +sg17 +(dp1209 +g19 +VWoodwind instrument +p1210 +ssa(dp1211 +g2 +(dp1212 +g4 +Vhttp://freebase.com/view/en/shamisen +p1213 +sg6 +S'The shamisen or samisen (\xe4\xb8\x89\xe5\x91\xb3\xe7\xb7\x9a, literally "three flavor strings"), also called sangen (\xe4\xb8\x89\xe7\xb5\x83, literally "three strings") is a three-stringed musical instrument played with a plectrum called a bachi. The pronunciation in Japanese is usually "shamisen" (in western Japan, and often in Edo-period sources "samisen") but sometimes "jamisen" when used as a suffix (e.g., Tsugaru-jamisen).\nThe shamisen is similar in length to a guitar, but its neck is much much slimmer and has no frets. Its drum-like...' +p1214 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bg242 +p1215 +sg10 +VShamisen +p1216 +sg12 +Vhttp://indextank.com/_static/common/demo/02bg242.jpg +p1217 +ssg14 +(dp1218 +I0 +I4 +ssg16 +g1216 +sg17 +(dp1219 +g19 +VPlucked string instrument +p1220 +ssa(dp1221 +g2 +(dp1222 +g4 +Vhttp://freebase.com/view/en/stroh_violin +p1223 +sg6 +S'A Stroh violin, Stro(h)viol, violinophone, or horn-violin is a violin that amplifies its sound through a metal resonator and metal horns rather than a wooden sound box as on a standard violin. The instrument is named after its German designer, Johannes Matthias Augustus Stroh, who patented it in 1899. The Stroh violin is also closely related to other horned violins using a mica sheet resonating diaphragm known as Phonofiddles.\nIn the present day, many types of horn-violin exist, especially...' +p1224 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05mbh5x +p1225 +sg10 +VStroh violin +p1226 +sg12 +Vhttp://indextank.com/_static/common/demo/05mbh5x.jpg +p1227 +ssg14 +(dp1228 +I0 +I1 +ssg16 +g1226 +sg17 +(dp1229 +g19 +VViolin +p1230 +ssa(dp1231 +g2 +(dp1232 +g4 +Vhttp://freebase.com/view/en/nose_flute +p1233 +sg6 +S'The nose flute is a popular musical instrument played in Polynesia and the Pacific Rim countries. Other versions are found in Africa, China, and India.\nIn the North Pacific, in the Hawaiian islands the nose flute was a common courting instrument. In Hawaiian, it is variously called hano, "nose flute," (Pukui and Elbert 1986), by the more specific term \'ohe hano ihu, "bamboo flute [for] nose," or `ohe hanu `ihu, "bamboo [for] nose breath" (Nona Beamer lectures).\nIt is made from a single...' +p1234 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03sxt9m +p1235 +sg10 +VNose flute +p1236 +sg12 +Vhttp://indextank.com/_static/common/demo/03sxt9m.jpg +p1237 +ssg14 +(dp1238 +I0 +I0 +ssg16 +g1236 +sg17 +(dp1239 +g19 +VFlute (transverse) +p1240 +ssa(dp1241 +g2 +(dp1242 +g4 +Vhttp://freebase.com/view/en/konghou +p1243 +sg6 +S"The konghou (Chinese: \xe7\xae\x9c\xe7\xaf\x8c; pinyin: k\xc5\x8dngh\xc3\xb3u) is an ancient Chinese harp. The konghou, also known as kanhou, went extinct sometime in the Ming Dynasty, but was revived in the 20th century. The modern version of the instrument does not resemble the ancient one.\nThe main feature that distinguishes the contemporary konghou from the Western concert harp is that the modern konghou's strings are folded over to make two rows, which enables players to use advanced playing techniques such as vibrato and..." +p1244 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03sgm8f +p1245 +sg10 +VKonghou +p1246 +sg12 +Vhttp://indextank.com/_static/common/demo/03sgm8f.jpg +p1247 +ssg14 +(dp1248 +I0 +I0 +ssg16 +g1246 +sg17 +(dp1249 +g19 +VPlucked string instrument +p1250 +ssa(dp1251 +g2 +(dp1252 +g4 +Vhttp://freebase.com/view/en/cello +p1253 +sg6 +S"The cello (pronounced /\xcb\x88t\xca\x83\xc9\x9blo\xca\x8a/ CHEL-oh; plural cellos or celli) is a bowed string instrument with four strings tuned in perfect fifths. It is a member of the violin family of musical instruments, which also includes the violin, viola and the contrabass.\nThe word derives from the Italian 'violoncello'. The word derives ultimately from vitula, meaning a stringed instrument. A person who plays a cello is called a cellist. The cello is used as a solo instrument, in chamber music, in a string..." +p1254 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0290__v +p1255 +sg10 +VVioloncello +p1256 +sg12 +Vhttp://indextank.com/_static/common/demo/0290__v.jpg +p1257 +ssg14 +(dp1258 +I0 +I274 +ssg16 +g1256 +sg17 +(dp1259 +g19 +VBowed string instruments +p1260 +ssa(dp1261 +g2 +(dp1262 +g4 +Vhttp://freebase.com/view/en/buzuq +p1263 +sg6 +S'The buzuq (Arabic: \xd8\xa8\xd8\xb2\xd9\x82\xe2\x80\x8e; also transliterated bozuq, bouzouk, buzuk etc.) is a long-necked fretted lute related to the Greek bouzouki and Turkish saz. It is an essential instrument in the Rahbani repertoire, but it is not classified among the classical instruments of Arab or Turkish music. However, this instrument may be looked upon as a larger and deeper-toned relative of the saz, to which it could be compared in the same way as the viola to the violin in Western music. Before the Rahbanis...' +p1264 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04p8wkb +p1265 +sg10 +VBuzuq +p1266 +sg12 +Vhttp://indextank.com/_static/common/demo/04p8wkb.jpg +p1267 +ssg14 +(dp1268 +I0 +I0 +ssg16 +g1266 +sg17 +(dp1269 +g19 +VPlucked string instrument +p1270 +ssa(dp1271 +g2 +(dp1272 +g4 +Vhttp://freebase.com/view/en/pipe_organ_of_the_lds_conference_center +p1273 +sg6 +S'The Schoenstein Organ at the Conference Center is a pipe organ built by Schoenstein & Co., San Francisco, California located in the Conference Center of The Church of Jesus Christ of Latter-day Saints in Salt Lake City, Utah. The organ was completed in 2003. It is composed of five manuals and pedal. Along with the nearby Salt Lake Tabernacle organ, it is typically used to accompany the Mormon Tabernacle Choir.\nThe Conference Center organ is heard semi-annually each year at the Church\xe2\x80\x99s...' +p1274 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/08bc354 +p1275 +sg10 +VPipe organ of the LDS Conference Center +p1276 +sg12 +Vhttp://indextank.com/_static/common/demo/08bc354.jpg +p1277 +ssg14 +(dp1278 +I0 +I0 +ssg16 +g1276 +sg17 +(dp1279 +g19 +VPipe organ +p1280 +ssa(dp1281 +g2 +(dp1282 +g4 +Vhttp://freebase.com/view/en/cittern +p1283 +sg6 +S'The cittern or cither is a stringed instrument dating from the Renaissance. Modern scholars debate its exact history, but it is generally accepted that it is descended from the Medieval Citole, or Cytole. It looks much like the modern-day flat-back mandolin and the modern Irish bouzouki and cittern. Its flat-back design was simpler and cheaper to construct than the lute. It was also easier to play, smaller, less delicate and more portable. Played by all classes, the cittern was a premier...' +p1284 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02ch7s3 +p1285 +sg10 +VCittern +p1286 +sg12 +Vhttp://indextank.com/_static/common/demo/02ch7s3.jpg +p1287 +ssg14 +(dp1288 +I0 +I2 +ssg16 +g1286 +sg17 +(dp1289 +g19 +VPlucked string instrument +p1290 +ssa(dp1291 +g2 +(dp1292 +g4 +Vhttp://freebase.com/view/en/chord_organ +p1293 +sg6 +S'A chord organ is an organ (a free-reed musical instrument), similar to a small reed organ, in which sound is produced by the flow of air, usually driven by an electric motor, over plastic or metal reeds. Much like the accordion, the chord organ has both a keyboard and a set of chord buttons, enabling the musician to play a melody or lead with one hand and accompanying chords with the other. Chord organs were generally designed as toys, like those made by the Magnus Harmonica Corporation and...' +p1294 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s8cq2 +p1295 +sg10 +VChord organ +p1296 +sg12 +Vhttp://indextank.com/_static/common/demo/03s8cq2.jpg +p1297 +ssg14 +(dp1298 +I0 +I1 +ssg16 +g1296 +sg17 +(dp1299 +g19 +VOrgan +p1300 +ssa(dp1301 +g2 +(dp1302 +g4 +Vhttp://freebase.com/view/en/saz +p1303 +sg6 +S'Saz can be a nickname for the given name Sarah, or may refer to:' +p1304 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s6v1m +p1305 +sg10 +VSaz +p1306 +sg12 +Vhttp://indextank.com/_static/common/demo/03s6v1m.jpg +p1307 +ssg14 +(dp1308 +I0 +I4 +ssg16 +g1306 +sg17 +(dp1309 +g19 +VPlucked string instrument +p1310 +ssa(dp1311 +g2 +(dp1312 +g4 +Vhttp://freebase.com/view/en/clavinet +p1313 +sg6 +S'A Clavinet is an electrophonic keyboard instrument manufactured by the Hohner company. It is essentially an electronically amplified clavichord, analogous to an electric guitar. Its distinctive bright staccato sound has appeared particularly in funk, disco, rock, and reggae songs.\nVarious models were produced over the years, including the models I, II, L, C, D6, and E7. Most models consist of 60 keys and 60 associated strings, giving it a five-octave range from F1 to E6.\nEach key uses a...' +p1314 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029s8y_ +p1315 +sg10 +VClavinet +p1316 +sg12 +Vhttp://indextank.com/_static/common/demo/029s8y_.jpg +p1317 +ssg14 +(dp1318 +I0 +I22 +ssg16 +g1316 +sg17 +(dp1319 +g19 +VClavichord +p1320 +ssa(dp1321 +g2 +(dp1322 +g4 +Vhttp://freebase.com/view/en/paraguayan_harp +p1323 +sg6 +S'The Paraguayan harp is the national instrument of Paraguay, and similar instruments are used elsewhere in South America, particularly Venezuela.\nIt is a diatonic harp with 32, 36, 38 or 40 strings, made from tropical wood, with an exaggerated neck-arch, played with the fingernail. It accompanies songs in the Guarani language.' +p1324 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/07s_y8b +p1325 +sg10 +VParaguayan harp +p1326 +sg12 +Vhttp://indextank.com/_static/common/demo/07s_y8b.jpg +p1327 +ssg14 +(dp1328 +I0 +I0 +ssg16 +g1326 +sg17 +(dp1329 +g19 +VHarp +p1330 +ssa(dp1331 +g2 +(dp1332 +g4 +Vhttp://freebase.com/view/en/timbales +p1333 +sg6 +S'Timbales (or pailas criollas) are shallow single-headed drums with metal casing, invented in Cuba. They are shallower in shape than single-headed tom-toms, and usually much higher tuned. The player (known as a timbalero) uses a variety of stick strokes, rim shots, and rolls on the skins to produce a wide range of percussive expression during solos and at transitional sections of music, and usually plays the shells of the drum or auxiliary percussion such as a cowbell or cymbal to keep time...' +p1334 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041l8m6 +p1335 +sg10 +VTimbales +p1336 +sg12 +Vhttp://indextank.com/_static/common/demo/041l8m6.jpg +p1337 +ssg14 +(dp1338 +I0 +I8 +ssg16 +g1336 +sg17 +(dp1339 +g19 +VPercussion +p1340 +ssa(dp1341 +g2 +(dp1342 +g4 +Vhttp://freebase.com/view/en/post_horn +p1343 +sg6 +S'The post horn (also posthorn, post-horn, or coach horn) is a valveless cylindrical brass or copper instrument with cupped mouthpiece, used to signal the arrival or departure of a post rider or mail coach. It was used especially by postilions of the 18th and 19th centuries.\nThe instrument commonly had a circular or coiled shape with three turns of the tubing, though sometimes it was straight. It is therefore an example of a natural instrument. The cornet was developed from the post horn by...' +p1344 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02g8_mb +p1345 +sg10 +VPost horn +p1346 +sg12 +Vhttp://indextank.com/_static/common/demo/02g8_mb.jpg +p1347 +ssg14 +(dp1348 +I0 +I0 +ssg16 +g1346 +sg17 +(dp1349 +g19 +VBrass instrument +p1350 +ssa(dp1351 +g2 +(dp1352 +g4 +Vhttp://freebase.com/view/en/division_viol +p1353 +sg6 +S'The division viol is an English type of bass viol, which was originally popular in the mid-17th century, but is currently experiencing a renaissance of its own due to the movement for historically informed performance. John Playford mentions the division viol in his A Brief Introduction of 1667, describing it as smaller than a consort bass viol, but larger than a lyra viol.\nAs suggested by its name, (divisions were a type of variations), the division viol is intended for highly ornamented...' +p1354 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02g_6ql +p1355 +sg10 +VDivision viol +p1356 +sg12 +Vhttp://indextank.com/_static/common/demo/02g_6ql.jpg +p1357 +ssg14 +(dp1358 +I0 +I0 +ssg16 +g1356 +sg17 +(dp1359 +g19 +VBowed string instruments +p1360 +ssa(dp1361 +g2 +(dp1362 +g4 +Vhttp://freebase.com/view/en/bass_trumpet +p1363 +sg6 +S"The bass trumpet is a type of low trumpet which was first developed during the 1820s in Germany. It is usually pitched in 8' C or 9' B\xe2\x99\xad today, but is sometimes built in E\xe2\x99\xad and is treated as a transposing instrument sounding either an octave, a sixth or a ninth lower than written, depending on the pitch of the instrument. Although almost identical in length to the trombone, the bass trumpet possesses a tone which is harder and more metallic than that of the trombone. Although it has valves..." +p1364 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02937mn +p1365 +sg10 +VBass trumpet +p1366 +sg12 +Vhttp://indextank.com/_static/common/demo/02937mn.jpg +p1367 +ssg14 +(dp1368 +I0 +I0 +ssg16 +g1366 +sg17 +(dp1369 +g19 +VTrumpet +p1370 +ssa(dp1371 +g2 +(dp1372 +g4 +Vhttp://freebase.com/view/en/plucked_string_instrument +p1373 +sg6 +S'Plucked string instruments are a subcategory of string instruments that are played by plucking the strings. Plucking is a way of pulling and releasing the string in such as way as to give it an impulse that causes the string to vibrate. Plucking can be done with either a finger or a plectrum.\nMost plucked string instruments belong to the lute family (such as guitar, bass guitar, mandolin, banjo, balalaika, sitar, pipa, etc.), which generally consist of a resonating body, and a neck; the...' +p1374 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dd6jj +p1375 +sg10 +VPlucked string instrument +p1376 +sg12 +Vhttp://indextank.com/_static/common/demo/02dd6jj.jpg +p1377 +ssg14 +(dp1378 +I0 +I0 +ssg16 +g1376 +sg17 +(dp1379 +g19 +VString instrument +p1380 +ssa(dp1381 +g2 +(dp1382 +g4 +Vhttp://freebase.com/view/en/castanet +p1383 +sg6 +S'Castanets are percussion instrument (idiophone), used in Moorish, Ottoman, ancient Roman, Italian, Spanish, Portuguese, Latin American music, and Irish Folk Music. The instrument consists of a pair of concave shells joined on one edge by string. These are held in the hand and used to produce clicks for rhythmic accents or a ripping or rattling sound consisting of a rapid series of clicks. They are traditionally made of hardwood, although fibreglass is becoming increasingly popular.\nIn...' +p1384 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03tpmfd +p1385 +sg10 +VCastanet +p1386 +sg12 +Vhttp://indextank.com/_static/common/demo/03tpmfd.jpg +p1387 +ssg14 +(dp1388 +I0 +I2 +ssg16 +g1386 +sg17 +(dp1389 +g19 +VPercussion +p1390 +ssa(dp1391 +g2 +(dp1392 +g4 +Vhttp://freebase.com/view/en/bouzouki +p1393 +sg6 +S'The bouzouki (Greek \xcf\x84\xce\xbf \xce\xbc\xcf\x80\xce\xbf\xcf\x85\xce\xb6\xce\xbf\xcf\x8d\xce\xba\xce\xb9; pl. \xcf\x84\xce\xb1 \xce\xbc\xcf\x80\xce\xbf\xcf\x85\xce\xb6\xce\xbf\xcf\x8d\xce\xba\xce\xb9\xce\xb1) (plural sometimes transliterated as bouzoukia) is a musical instrument in the lute family, with a pear-shaped body and a long neck. A mainstay of modern Greek music, the front of the body is flat and is usually heavily inlaid with mother-of-pearl. The instrument is played with a plectrum and has a sharp metallic sound, reminiscent of a mandolin but pitched lower.\nThere are two main types of bouzouki:\nIn Greece, there had been an instrument...' +p1394 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029gy47 +p1395 +sg10 +VBouzouki +p1396 +sg12 +Vhttp://indextank.com/_static/common/demo/029gy47.jpg +p1397 +ssg14 +(dp1398 +I0 +I25 +ssg16 +g1396 +sg17 +(dp1399 +g19 +VPlucked string instrument +p1400 +ssa(dp1401 +g2 +(dp1402 +g4 +Vhttp://freebase.com/view/en/suspended_cymbal +p1403 +sg6 +S'A suspended cymbal is any single cymbal played with a stick or beater rather than struck against another cymbal. A common abbreviation used is sus. cym., or sus. cymb. (with, or without the period).\nThe term comes from the modern orchestra, in which the term cymbals normally refers to a pair of clash cymbals. The first suspended cymbals used in the modern orchestra were one of a pair of orchestral cymbals, supported by hanging it bell upwards by its strap. This technique is still used, at...' +p1404 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03tdbhm +p1405 +sg10 +VSuspended cymbal +p1406 +sg12 +Vhttp://indextank.com/_static/common/demo/03tdbhm.jpg +p1407 +ssg14 +(dp1408 +I0 +I0 +ssg16 +g1406 +sg17 +(dp1409 +g19 +VCymbal +p1410 +ssa(dp1411 +g2 +(dp1412 +g4 +Vhttp://freebase.com/view/en/electric_violin +p1413 +sg6 +S'An electric violin is a violin equipped with an electronic output of its sound. The term most properly refers to an instrument purposely made to be electrified with built-in pickups, usually with a solid body. It can also refer to a violin fitted with an electric pickup of some type, although "amplified violin" or "electro-acoustic violin" are more accurate in that case.\nElectrically amplified violins have been used in one form or another since the 1920s; jazz and blues artist Stuff Smith is...' +p1414 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03sz_t5 +p1415 +sg10 +VElectric violin +p1416 +sg12 +Vhttp://indextank.com/_static/common/demo/03sz_t5.jpg +p1417 +ssg14 +(dp1418 +I0 +I41 +ssg16 +g1416 +sg17 +(dp1419 +g19 +VViolin +p1420 +ssa(dp1421 +g2 +(dp1422 +g4 +Vhttp://freebase.com/view/en/oboe +p1423 +sg6 +S'The oboe (English pronunciation:\xc2\xa0/\xcb\x88o\xca\x8abo\xca\x8a/) is a double reed musical instrument of the woodwind family. In English, prior to 1770, the instrument was called "hautbois" (French, meaning "high wood"), "hoboy", or "French hoboy". The spelling "oboe" was adopted into English ca. 1770 from the Italian obo\xc3\xa8, a transliteration in that language\'s orthography of the 17th-century pronunciation of the French word hautbois, a compound word made of haut ("high, loud") and bois ("wood, woodwind"). A...' +p1424 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0291jbh +p1425 +sg10 +VOboe +p1426 +sg12 +Vhttp://indextank.com/_static/common/demo/0291jbh.jpg +p1427 +ssg14 +(dp1428 +I0 +I144 +ssg16 +g1426 +sg17 +(dp1429 +g19 +VWoodwind instrument +p1430 +ssa(dp1431 +g2 +(dp1432 +g4 +Vhttp://freebase.com/view/en/kemenche +p1433 +sg6 +S'The term kemenche (Turkish: kemen\xc3\xa7e, Adyghe: \xd0\xa8\xd1\x8b\xd0\xba1\xd1\x8d \xd0\xbf\xd1\x89\xd1\x8b\xd0\xbd, Armenian: \xd6\x84\xd5\xa1\xd5\xb4\xd5\xa1\xd5\xb6\xd5\xb9\xd5\xa1 k\xe2\x80\x99aman\xc4\x8da, Laz: \xc3\x87\'ilili - \xe1\x83\xad\xe1\x83\x98\xe1\x83\x9a\xe1\x83\x98\xe1\x83\x9a\xe1\x83\x98, Azerbaijani: kaman\xc3\xa7a, Persian: \xda\xa9\xd9\x85\xd8\xa7\xd9\x86\xda\x86\xd9\x87, Greek: \xce\xbb\xcf\x8d\xcf\x81\xce\xb1) is used to describe two types of three-stringed bowed musical instruments:\nBoth types of kemenche are played in the downright position, either by resting it on the knee when sitting, or held in front of the player when standing. It is always played "braccio", that is, with the tuning head uppermost. The kemenche bow is called the...' +p1434 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044m0vc +p1435 +sg10 +VKemenche +p1436 +sg12 +Vhttp://indextank.com/_static/common/demo/044m0vc.jpg +p1437 +ssg14 +(dp1438 +I0 +I1 +ssg16 +g1436 +sg17 +(dp1439 +g19 +VBowed string instruments +p1440 +ssa(dp1441 +g2 +(dp1442 +g4 +Vhttp://freebase.com/view/en/ride_cymbal +p1443 +sg6 +S"The ride cymbal is a standard cymbal in most drum kits. It maintains a steady rhythmic pattern, sometimes called a ride pattern, rather than the accent of a crash. It is normally placed on the extreme right (or dominant hand) of a drum kit, above the floor tom.\nThe ride can fulfill any function or rhythm the hi-hat does, with the exclusion of an open and closed sound.\nThe term ride means to ride with the music, describing the cymbal's sustain after it is struck. The term may depict either..." +p1444 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bdbkf +p1445 +sg10 +VRide cymbal +p1446 +sg12 +Vhttp://indextank.com/_static/common/demo/02bdbkf.jpg +p1447 +ssg14 +(dp1448 +I0 +I0 +ssg16 +g1446 +sg17 +(dp1449 +g19 +VCymbal +p1450 +ssa(dp1451 +g2 +(dp1452 +g4 +Vhttp://freebase.com/view/en/kithara +p1453 +sg6 +S'The cithara or kithara (Greek: \xce\xba\xce\xb9\xce\xb8\xce\xac\xcf\x81\xce\xb1, kith\xc4\x81ra, Latin: cithara) was an ancient Greek musical instrument in the lyre or lyra family. In modern Greek the word kithara has come to mean "guitar" (a word whose origins are found in kithara).\nThe kithara was a professional version of the two-stringed lyre. As opposed to the simpler lyre, which was a folk-instrument, the cithara was primarily used by professional musicians, called citharedes. The barbiton was a bass version of the kithara popular in...' +p1454 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f7_fw +p1455 +sg10 +VKithara +p1456 +sg12 +Vhttp://indextank.com/_static/common/demo/02f7_fw.jpg +p1457 +ssg14 +(dp1458 +I0 +I0 +ssg16 +g1456 +sg17 +(dp1459 +g19 +VLyre +p1460 +ssa(dp1461 +g2 +(dp1462 +g4 +Vhttp://freebase.com/view/en/scottish_smallpipes +p1463 +sg6 +S"The Scottish smallpipe, in its modern form, is a bellows-blown bagpipe developed by Colin Ross and others, to be playable according to the Great Highland Bagpipe fingering system. There are surviving examples of similar historical instruments such as the mouth-blown Montgomery smallpipes in E, dated 1757, which are now in the National Museum of Scotland. There is some discussion of the historical Scottish smallpipes in Collinson's history of the bagpipes. Some instruments are being built as..." +p1464 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/042smv6 +p1465 +sg10 +VScottish smallpipes +p1466 +sg12 +Vhttp://indextank.com/_static/common/demo/042smv6.jpg +p1467 +ssg14 +(dp1468 +I0 +I0 +ssg16 +g1466 +sg17 +(dp1469 +g19 +VBagpipes +p1470 +ssa(dp1471 +g2 +(dp1472 +g4 +Vhttp://freebase.com/view/en/contrabassoon +p1473 +sg6 +S'The contrabassoon, also known as the double bassoon or double-bassoon, is a larger version of the bassoon, sounding an octave lower. Its technique is similar to its smaller cousin, with a few notable differences.\nThe reed is considerably larger, at 65\xe2\x80\x9375\xc2\xa0mm in total length as compared to 53\xe2\x80\x9358\xc2\xa0mm for most bassoon reeds. Fingering is slightly different, particularly at the register change and in the extreme high range. The instrument is twice as long, curves around on itself twice, and, due...' +p1474 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044ln56 +p1475 +sg10 +VContrabassoon +p1476 +sg12 +Vhttp://indextank.com/_static/common/demo/044ln56.jpg +p1477 +ssg14 +(dp1478 +I0 +I0 +ssg16 +g1476 +sg17 +(dp1479 +g19 +VBassoon +p1480 +ssa(dp1481 +g2 +(dp1482 +g4 +Vhttp://freebase.com/view/en/rmi_368_electra-piano_and_harpsichord +p1483 +sg6 +S"The RMI 368 Electra-Piano and Harpsichord was an electronic piano and the most popular instrument created by RMI. Often serving as a substitute for a grand piano in live performance, it didn't actually sound like one. It had its own distinctive sound that separated it from other electric alternatives to the piano, like the Fender Rhodes or the Wurlitzer, in that its sound was generated by transistors (like an electronic organ, which RMI started out making), instead of a hammer hitting a reed..." +p1484 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044yr_6 +p1485 +sg10 +VRMI 368 Electra-Piano and Harpsichord +p1486 +sg12 +Vhttp://indextank.com/_static/common/demo/044yr_6.jpg +p1487 +ssg14 +(dp1488 +I0 +I0 +ssg16 +g1486 +sg17 +(dp1489 +g19 +VElectronic piano +p1490 +ssa(dp1491 +g2 +(dp1492 +g4 +Vhttp://freebase.com/view/en/barrel_piano +p1493 +sg6 +S'A barrel piano (also known as a "roller piano") is a forerunner of the modern player piano. Unlike the pneumatic player piano, a barrel piano is usually powered by turning a hand crank, though coin operated models powered by clockwork were used to provide music in establishments such as pubs and caf\xc3\xa9s. Barrel pianos were popular with street musicians, who sought novel instruments that were also highly portable. They are frequently confused with barrel organs, but are quite different...' +p1494 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02gsf7c +p1495 +sg10 +VBarrel piano +p1496 +sg12 +Vhttp://indextank.com/_static/common/demo/02gsf7c.jpg +p1497 +ssg14 +(dp1498 +I0 +I0 +ssg16 +g1496 +sg17 +(dp1499 +g19 +VPiano +p1500 +ssa(dp1501 +g2 +(dp1502 +g4 +Vhttp://freebase.com/view/en/langeleik +p1503 +sg6 +S'The langeleik also called langleik is a Norwegian stringed folklore musical instrument, a droned zither\nThe langeleik has only one melody string and up to 8 drone strings. Under the melody string there are seven frets per octave, forming a diatonic major scale. The drone strings are tuned to a triad. The langeleik is tuned to about an A, though on score the C major key is used, as if the instrument were tuned in C. This is for simplification of both writing and reading, by circumventing the...' +p1504 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04rh8gk +p1505 +sg10 +VLangeleik +p1506 +sg12 +Vhttp://indextank.com/_static/common/demo/04rh8gk.jpg +p1507 +ssg14 +(dp1508 +I0 +I0 +ssg16 +g1506 +sg17 +(dp1509 +g19 +VPlucked string instrument +p1510 +ssa(dp1511 +g2 +(dp1512 +g4 +Vhttp://freebase.com/view/en/ektara +p1513 +sg6 +S'Ektara (Bengali: \xe0\xa6\x8f\xe0\xa6\x95\xe0\xa6\xa4\xe0\xa6\xbe\xe0\xa6\xb0\xe0\xa6\xbe, Punjabi: \xe0\xa8\x87\xe0\xa8\x95 \xe0\xa8\xa4\xe0\xa8\xbe\xe0\xa8\xb0; literally "one-string", also called iktar, ektar, yaktaro gopichand) is a one-string instrument used in Bangladesh, India, Egypt, and Pakistan.\nIn origin the ektara was a regular string instrument of wandering bards and minstrels from India and is plucked with one finger. The ektara usually has a stretched single string, an animal skin over a head (made of dried pumpkin/gourd, wood or coconut) and pole neck or split bamboo cane neck.\nPressing the two...' +p1514 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041ykv2 +p1515 +sg10 +VEktara +p1516 +sg12 +Vhttp://indextank.com/_static/common/demo/041ykv2.jpg +p1517 +ssg14 +(dp1518 +I0 +I0 +ssg16 +g1516 +sg17 +(dp1519 +g19 +VPlucked string instrument +p1520 +ssa(dp1521 +g2 +(dp1522 +g4 +Vhttp://freebase.com/view/en/flexatone +p1523 +sg6 +S"The flexatone is a modern percussion instrument (an indirectly struck idiophone) consisting of a small flexible metal sheet suspended in a wire frame ending in a handle. \nAn invention for a flexatone occurs in the British Patent Records of 1922 and 1923. In 1924 the 'Flex-a-tone' was patented in the USA by the Playatone Company of New York.\nA wooden knob mounted on a strip of spring steel lies on each side of the metal sheet. The player holds the flexatone in one hand with the palm around..." +p1524 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029ydt8 +p1525 +sg10 +VFlexatone +p1526 +sg12 +Vhttp://indextank.com/_static/common/demo/029ydt8.jpg +p1527 +ssg14 +(dp1528 +I0 +I0 +ssg16 +g1526 +sg17 +(dp1529 +g19 +VPercussion +p1530 +ssa(dp1531 +g2 +(dp1532 +g4 +Vhttp://freebase.com/view/en/psalmodicon +p1533 +sg6 +S'The psalmodicon, or psalmodikon, is a single-stringed musical instrument. It was developed in Scandinavia for simplifying music in churches and schools. Beginning in the early 19th century, it was adopted by many rural churches in Scandinavia; later, immigrants brought the instrument to the United States. At the time, many congregations could not afford organs. Dance instruments were considered inappropriate for sacred settings, so violins were not allowed. The psalmodikon, on the other...' +p1534 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03sf05n +p1535 +sg10 +VPsalmodicon +p1536 +sg12 +Vhttp://indextank.com/_static/common/demo/03sf05n.jpg +p1537 +ssg14 +(dp1538 +I0 +I0 +ssg16 +g1536 +sg17 +(dp1539 +g19 +VBowed string instruments +p1540 +ssa(dp1541 +g2 +(dp1542 +g4 +Vhttp://freebase.com/view/en/santur +p1543 +sg6 +S'The santur (also sant\xc5\xabr, santour, santoor ) (Persian: \xd8\xb3\xd9\x86\xd8\xaa\xd9\x88\xd8\xb1) is a hammered dulcimer, of Persian origin it has strong resemblances to the Indian santoor. It is a trapezoid-shaped box often made of walnut or different exotic woods. The original classical santur has 72 strings. The can be roughly described as one hundred strings in Persian. The oval-shaped mallets (Mezrabs) are feather-weight and are held between the index and middle fingers. A typical santur has two sets of bridges, providing...' +p1544 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f61nd +p1545 +sg10 +VSantur +p1546 +sg12 +Vhttp://indextank.com/_static/common/demo/02f61nd.jpg +p1547 +ssg14 +(dp1548 +I0 +I4 +ssg16 +g1546 +sg17 +(dp1549 +g19 +VStruck string instruments +p1550 +ssa(dp1551 +g2 +(dp1552 +g4 +Vhttp://freebase.com/view/en/tromba_marina +p1553 +sg6 +S'A tromba marina, or marine trumpet (Fr. trompette marine; Ger. Marientrompete, Trompetengeige, Nonnengeige or Trumscheit, Pol. tubmaryna) is a triangular bowed string instrument used in medieval and Renaissance Europe that was highly popular in the 15th century in England and survived into the 18th century. The tromba marina consists of a body and neck in the shape of a truncated cone resting on a triangular base. It is usually four to seven feet long, and is a monochord (although some...' +p1554 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02g9ls1 +p1555 +sg10 +VTromba marina +p1556 +sg12 +Vhttp://indextank.com/_static/common/demo/02g9ls1.jpg +p1557 +ssg14 +(dp1558 +I0 +I0 +ssg16 +g1556 +sg17 +(dp1559 +g19 +VBowed string instruments +p1560 +ssa(dp1561 +g2 +(dp1562 +g4 +Vhttp://freebase.com/view/en/uilleann_pipes +p1563 +sg6 +S'The uilleann (pronounced /\xcb\x88\xc9\xaal\xc9\x99n/) pipes are the characteristic national bagpipe of Ireland. Their current name (they were earlier known in English as "union pipes") is a part translation of the Irish-language term p\xc3\xadoba uilleann (literally, "pipes of the elbow"), from their method of inflation. The bag of the uilleann pipes is inflated by means of a small set of bellows strapped around the waist and the right arm. The bellows not only relieve the player from the effort needed to blow into a...' +p1564 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03twg5s +p1565 +sg10 +VUilleann pipes +p1566 +sg12 +Vhttp://indextank.com/_static/common/demo/03twg5s.jpg +p1567 +ssg14 +(dp1568 +I0 +I5 +ssg16 +g1566 +sg17 +(dp1569 +g19 +VBagpipes +p1570 +ssa(dp1571 +g2 +(dp1572 +g4 +Vhttp://freebase.com/view/en/electronic_keyboard +p1573 +sg6 +S'An electronic keyboard (also called digital keyboard, portable keyboard and home keyboard) is an electronic or digital keyboard instrument.\nThe major components of a typical modern electronic keyboard are:\nElectronic keyboards typically use MIDI signals to send and receive data, a standard format now universally used across most digital electronic musical instruments. On the simplest example of an electronic keyboard, MIDI messages would be sent when a note is pressed on the keyboard, and...' +p1574 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029n2xk +p1575 +sg10 +VElectronic keyboard +p1576 +sg12 +Vhttp://indextank.com/_static/common/demo/029n2xk.jpg +p1577 +ssg14 +(dp1578 +I0 +I98 +ssg16 +g1576 +sg17 +(dp1579 +g19 +VKeyboard instrument +p1580 +ssa(dp1581 +g2 +(dp1582 +g4 +Vhttp://freebase.com/view/en/barrel_organ +p1583 +sg6 +S'A barrel organ (or roller organ) is a mechanical musical instrument consisting of bellows and one or more ranks of pipes housed in a case, usually of wood, and often highly decorated. The basic principle is the same as a traditional pipe organ, but rather than being played by an organist, the barrel organ is activated either by a person turning a crank, or by clockwork driven by weights or springs. The pieces of music are encoded onto wooden barrels (or cylinders), which are analogous to the...' +p1584 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bcy90 +p1585 +sg10 +VBarrel organ +p1586 +sg12 +Vhttp://indextank.com/_static/common/demo/02bcy90.jpg +p1587 +ssg14 +(dp1588 +I0 +I0 +ssg16 +g1586 +sg17 +(dp1589 +g19 +VMechanical organ +p1590 +ssa(dp1591 +g2 +(dp1592 +g4 +Vhttp://freebase.com/view/en/subcontrabass_flute +p1593 +sg6 +S'The subcontrabass flute is one of the largest instruments in the flute family, measuring over 15\xc2\xa0feet (4.6 m) long. The instrument can be made in the key of G, pitched a fourth below the contrabass flute in C and two octaves below the alto flute in G; which is sometimes also called double contra-alto flute, or in C, which will sound three octaves lower than the C flute.\nThe subcontrabass flute is rarely used outside of flute ensembles. It is sometimes called the "gentle giant" of the flute...' +p1594 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05m243v +p1595 +sg10 +VSubcontrabass flute +p1596 +sg12 +Vhttp://indextank.com/_static/common/demo/05m243v.jpg +p1597 +ssg14 +(dp1598 +I0 +I0 +ssg16 +g1596 +sg17 +(dp1599 +g19 +VFlute (transverse) +p1600 +ssa(dp1601 +g2 +(dp1602 +g4 +Vhttp://freebase.com/view/en/gaita_de_saco +p1603 +sg6 +S'The gaita de saco (or de bota) is a type of bagpipe native to the provinces of Soria, La Rioja, Alava, and Burgos in north-central Spain. In the past, it may also have been played in Segovia and \xc3\x81vila. According to some experts, the gaita de boto is the same as the gaita de fuelle of Old Castile.\nIt consists of a single chanter (puntero) holding a double reed which plays the melody, and single drone (ronco), which has a single reed and plays a constant bass note.\nIn La Rioja, the instrument...' +p1604 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/063850_ +p1605 +sg10 +VGaita de saco +p1606 +sg12 +Vhttp://indextank.com/_static/common/demo/063850_.jpg +p1607 +ssg14 +(dp1608 +I0 +I0 +ssg16 +g1606 +sg17 +(dp1609 +g19 +VBagpipes +p1610 +ssa(dp1611 +g2 +(dp1612 +g4 +Vhttp://freebase.com/view/en/gong +p1613 +sg6 +S'A gong is an East and South East Asian musical percussion instrument that takes the form of a flat metal disc which is hit with a malleta.\nGongs are broadly of three types. Suspended gongs are more or less flat, circular discs of metal suspended vertically by means of a cord passed through holes near to the top rim. Bossed or nipple gongs have a raised center boss and are often suspended and played horizontally. Bowl gongs are bowl-shaped, and rest on cushions and belong more to bells than...' +p1614 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041nzgy +p1615 +sg10 +VGong +p1616 +sg12 +Vhttp://indextank.com/_static/common/demo/041nzgy.jpg +p1617 +ssg14 +(dp1618 +I0 +I3 +ssg16 +g1616 +sg17 +(dp1619 +g19 +VPercussion +p1620 +ssa(dp1621 +g2 +(dp1622 +g4 +Vhttp://freebase.com/view/en/vielle +p1623 +sg6 +S'The vielle is a European bowed stringed instrument used in the Medieval period, similar to a modern violin but with a somewhat longer and deeper body, five (rather than four) gut strings, and a leaf-shaped pegbox with frontal tuning pegs. The instrument was also known as a fidel or a viuola, although the French name for the instrument, vielle, is generally used. It was one of the most popular instruments of the Medieval period, and was used by troubadours and jongleurs from the 13th through...' +p1624 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bgn1z +p1625 +sg10 +VVielle +p1626 +sg12 +Vhttp://indextank.com/_static/common/demo/02bgn1z.jpg +p1627 +ssg14 +(dp1628 +I0 +I0 +ssg16 +g1626 +sg17 +(dp1629 +g19 +VBowed string instruments +p1630 +ssa(dp1631 +g2 +(dp1632 +g4 +Vhttp://freebase.com/view/en/horn +p1633 +sg6 +S'The horn is a brass instrument consisting of about 12\xe2\x80\x9313 feet (3.66\xe2\x80\x933.96 meters) of tubing wrapped into a coil with a flared bell. A musician who plays the horn is called a horn player (or less frequently, a hornist).\nDescended from the natural horn, the instrument is often informally and incorrectly known as the French horn. Since 1971 the International Horn Society has recommended the use of the word horn alone, as the commonly played instrument is not, in fact, the French horn, but rather...' +p1634 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0593gps +p1635 +sg10 +VHorn +p1636 +sg12 +Vhttp://indextank.com/_static/common/demo/0593gps.jpg +p1637 +ssg14 +(dp1638 +I0 +I58 +ssg16 +g1636 +sg17 +(dp1639 +g19 +VBrass instrument +p1640 +ssa(dp1641 +g2 +(dp1642 +g4 +Vhttp://freebase.com/view/en/irish_flute +p1643 +sg6 +S'The term Irish Flute refers to a conical-bore, simple-system wooden flute of the type favored by classical flautists of the early 19th century, or to a flute of modern manufacture derived from this design (often with modifications to optimize its use in Irish Traditional Music or Scottish Traditional Music).\nThe Irish flute is a simple system, transverse flute which plays a diatonic (Major) scale as the tone holes are successively uncovered. Most flutes from the Classical era, and some of...' +p1644 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05tr2mh +p1645 +sg10 +VIrish flute +p1646 +sg12 +Vhttp://indextank.com/_static/common/demo/05tr2mh.jpg +p1647 +ssg14 +(dp1648 +I0 +I1 +ssg16 +g1646 +sg17 +(dp1649 +g19 +VFlute (transverse) +p1650 +ssa(dp1651 +g2 +(dp1652 +g4 +Vhttp://freebase.com/view/en/cumbus +p1653 +sg6 +S'The c\xc3\xbcmb\xc3\xbc\xc5\x9f (Turkish pronunciation:\xc2\xa0[d\xca\x92ym\xcb\x88by\xca\x83]; sometimes approximated as /d\xca\x92u\xcb\x90m\xcb\x88bu\xcb\x90\xca\x83/ by English speakers) is a Turkish stringed instrument of relatively modern origin. Developed in the early 20th century by Zeynelabidin C\xc3\xbcmb\xc3\xbc\xc5\x9f as an oud-like instrument that could be heard as part of a larger ensemble. In construction it resembles both the American banjo and the Middle Eastern oud. A fretless instrument, it has six courses of doubled-strings, and is generally tuned like an oud. In shape,...' +p1654 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02b8q91 +p1655 +sg10 +VCümbü\u015f +p1656 +sg12 +Vhttp://indextank.com/_static/common/demo/02b8q91.jpg +p1657 +ssg14 +(dp1658 +I0 +I4 +ssg16 +g1656 +sg17 +(dp1659 +g19 +VPlucked string instrument +p1660 +ssa(dp1661 +g2 +(dp1662 +g4 +Vhttp://freebase.com/view/en/gaita_de_boto +p1663 +sg6 +S'The gaita de boto is a type of bagpipe native to the Aragon region of northern Spain.\nIts use and construction were nearly extinct by the 1970s, when a revival of folk music began. Today there are various gaita builders, various schools and associations for gaita players, and more than a dozen Aragonese folk music groups which include the instrument in their ensemble. Most importantly, there are now several hundred gaiteros within Aragon.\nThe gaita de boto consists of\nThe bag is...' +p1664 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0636rx_ +p1665 +sg10 +VGaita de boto +p1666 +sg12 +Vhttp://indextank.com/_static/common/demo/0636rx_.jpg +p1667 +ssg14 +(dp1668 +I0 +I0 +ssg16 +g1666 +sg17 +(dp1669 +g19 +VBagpipes +p1670 +ssa(dp1671 +g2 +(dp1672 +g4 +Vhttp://freebase.com/view/en/cristal_baschet +p1673 +sg6 +S'The Cristal Baschet is a musical instrument that produces sound from oscillating glass cylinders. The Cristal Baschet is also known as the Crystal Organ and the Crystal Baschet, and composed of 54 chromatically-tuned glass rods. The glass rods are rubbed with moistened fingers to produce vibrations. The sound of the Cristal Baschet is similar to that of the glass harmonica.\nThe vibration of the glass rods in the Cristal Baschet is passed to a heavy block of metal by a metal stem whose...' +p1674 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041d9tn +p1675 +sg10 +VCristal baschet +p1676 +sg12 +Vhttp://indextank.com/_static/common/demo/041d9tn.jpg +p1677 +ssg14 +(dp1678 +I0 +I1 +ssg16 +g1676 +sg17 +(dp1679 +g19 +VCrystallophone +p1680 +ssa(dp1681 +g2 +(dp1682 +g4 +Vhttp://freebase.com/view/en/lauterbach_stradivarius +p1683 +sg6 +S'The Lauterbach Stradivarius of 1719 is an antique violin fabricated by Italian luthier, Antonio Stradivari of Cremona (1644-1737). The instrument derives its name from previous owner, German virtuoso, Johann Christoph Lauterbach.\nComposer and violinist Charles Philippe Lafont owned the violin. On his death, the violin was acquired by luthier and expert Jean-Baptiste Vuillaume. Vuillaume sold the violin to Johann Christoph Lauterbach.\nPolish textile manufacturer Henryk Grohman acquired the...' +p1684 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0bbdvgl +p1685 +sg10 +VLauterbach Stradivarius +p1686 +sg12 +Vhttp://indextank.com/_static/common/demo/0bbdvgl.jpg +p1687 +ssg14 +(dp1688 +I0 +I0 +ssg16 +g1686 +sg17 +(dp1689 +g19 +VViolin +p1690 +ssa(dp1691 +g2 +(dp1692 +g4 +Vhttp://freebase.com/view/en/goblet_drum +p1693 +sg6 +S'The Goblet drum (also Chalice drum, Darbuka Doumbek or Tablah"\') is a goblet shaped hand drum used mostly in the Middle East, North Africa, and Eastern Europe.\nThough it is not known exactly when these drums were first made, they are known to be of ancient origin. Some say that it has been around for thousands of years, used in Mesopotamian and Ancient Egyptian cultures.There has also has been some debates that it has actually originated in Europe and was brought to the Middle East by...' +p1694 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03sll6p +p1695 +sg10 +VGoblet drum +p1696 +sg12 +Vhttp://indextank.com/_static/common/demo/03sll6p.jpg +p1697 +ssg14 +(dp1698 +I0 +I9 +ssg16 +g1696 +sg17 +(dp1699 +g19 +VDrum +p1700 +ssa(dp1701 +g2 +(dp1702 +g4 +Vhttp://freebase.com/view/en/synthesizer +p1703 +sg6 +S'A synthesizer (often abbreviated "synth") is an electronic instrument capable of producing sounds by generating electrical signals of different frequencies. These electrical signals are played through a loudspeaker or set of headphones. Synthesizers can usually produce a wide range of sounds, which may either imitate other instruments ("imitative synthesis") or generate new timbres.\nSynthesizers use a number of different technologies or programmed algorithms, each with their own strengths...' +p1704 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03q_gc1 +p1705 +sg10 +VSynthesizer +p1706 +sg12 +Vhttp://indextank.com/_static/common/demo/03q_gc1.jpg +p1707 +ssg14 +(dp1708 +I0 +I442 +ssg16 +g1706 +sg17 +(dp1709 +g19 +VKeyboard instrument +p1710 +ssa(dp1711 +g2 +(dp1712 +g4 +Vhttp://freebase.com/view/en/ceng +p1713 +sg6 +S'The \xc3\xa7eng is a Turkish harp. Descended from ancient Near Eastern instruments, it was a popular Ottoman instrument until the last quarter of the 17th century. The word comes from the Persian word "chang," which means "harp" (and also "five fingers").\nThe ancestor of the Ottoman harp is thought to be an instrument seen in ancient Assyrian tablets. While a similar instrument also appears in Egyptian drawings.\nIn the late 20th century, instrument makers and performers began to revive the \xc3\xa7eng,...' +p1714 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d6s41 +p1715 +sg10 +VÇeng +p1716 +sg12 +Vhttp://indextank.com/_static/common/demo/02d6s41.jpg +p1717 +ssg14 +(dp1718 +I0 +I0 +ssg16 +g1716 +sg17 +(dp1719 +g19 +VPlucked string instrument +p1720 +ssa(dp1721 +g2 +(dp1722 +g4 +Vhttp://freebase.com/view/en/banhu +p1723 +sg6 +S'The banhu (\xe6\x9d\xbf\xe8\x83\xa1, pinyin: b\xc7\x8enh\xc3\xba) is a Chinese traditional bowed string instrument in the huqin family of instruments. It is used primarily in northern China. Ban means a piece of wood and hu is short for huqin.\nLike the more familiar erhu and gaohu, the banhu has two strings, is held vertically, and the bow hair passes in between the two strings. The banhu differs in construction from the erhu in that its soundbox is generally made from a coconut shell rather than wood, and instead of a...' +p1724 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0bch7yq +p1725 +sg10 +VBanhu +p1726 +sg12 +Vhttp://indextank.com/_static/common/demo/0bch7yq.jpg +p1727 +ssg14 +(dp1728 +I0 +I0 +ssg16 +g1726 +sg17 +(dp1729 +g19 +VBowed string instruments +p1730 +ssa(dp1731 +g2 +(dp1732 +g4 +Vhttp://freebase.com/view/en/contrabass_bugle +p1733 +sg6 +S"The contrabass bugle, usually shortened to contra, is the lowest-pitched instrument in the drum and bugle corps hornline. It is essentially the drum corps' counterpart to the marching band's sousaphone: the lowest-pitched member of the hornline, and a replacement for the concert tuba on the marching field.\nIt is different from the other members of the marching band and drum corps hornlines in that it rests on the shoulder of the player, rather than being held in front of the body. Because..." +p1734 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03tb_w9 +p1735 +sg10 +VContrabass Bugle +p1736 +sg12 +Vhttp://indextank.com/_static/common/demo/03tb_w9.jpg +p1737 +ssg14 +(dp1738 +I0 +I0 +ssg16 +g1736 +sg17 +(dp1739 +g19 +VBrass instrument +p1740 +ssa(dp1741 +g2 +(dp1742 +g4 +Vhttp://freebase.com/view/en/sallaneh +p1743 +sg6 +S'The sallaneh (\xd8\xb3\xd9\x84\xd8\xa7\xd9\x86\xd9\x87) is a newly developed plucked string instrument made under the supervision of the Iranian musician Hossein Alizadeh, and constructed by Siamak Afshari. It is inspired by the ancient Persian lute called barbat. The barbat used to have three strings but Sallaneh has six melody and six harmonic strings giving Alizadeh a new realm in lower tones.' +p1744 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03svz1c +p1745 +sg10 +VSallaneh +p1746 +sg12 +Vhttp://indextank.com/_static/common/demo/03svz1c.jpg +p1747 +ssg14 +(dp1748 +I0 +I1 +ssg16 +g1746 +sg17 +(dp1749 +g19 +VPlucked string instrument +p1750 +ssa(dp1751 +g2 +(dp1752 +g4 +Vhttp://freebase.com/view/en/vichitra_veena +p1753 +sg6 +S'The vichitra veena (Sanskrit: \xe0\xa4\xb5\xe0\xa4\xbf\xe0\xa4\x9a\xe0\xa4\xbf\xe0\xa4\xa4\xe0\xa5\x8d\xe0\xa4\xb0 \xe0\xa4\xb5\xe0\xa5\x80\xe0\xa4\xa3\xe0\xa4\xbe) is a plucked string instrument used in Hindustani music. It is similar to the Carnatic gottuvadhyam (chitra vina). It has no frets and is played with a slide.\nThe Vichitra Veena is the modern form of ancient Ektantri Veena. It is made of a broad, fretless, horizontal arm or crossbar (dand) around three feet long and six inches wide, with two large resonating gourds (tumba), which are inlaid with ivory and attached underneath at either end. The...' +p1754 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dz2m_ +p1755 +sg10 +VVichitra veena +p1756 +sg12 +Vhttp://indextank.com/_static/common/demo/02dz2m_.jpg +p1757 +ssg14 +(dp1758 +I0 +I0 +ssg16 +g1756 +sg17 +(dp1759 +g19 +VPlucked string instrument +p1760 +ssa(dp1761 +g2 +(dp1762 +g4 +Vhttp://freebase.com/view/en/mellophone +p1763 +sg6 +S'The mellophone is a brass instrument that is typically used in place of the horn (sometimes called a French horn) in marching bands or drum and bugle corps.\nOwing to its use primarily outside of concert music, there is not much solo literature for the mellophone, other than that used within drum and bugle corps.\nThe present-day mellophone has three valves, operated with the right hand. Mellophone fingering is identical to that of a trumpet. Mellophones are typically pitched in the key of F....' +p1764 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029hskr +p1765 +sg10 +VMellophone +p1766 +sg12 +Vhttp://indextank.com/_static/common/demo/029hskr.jpg +p1767 +ssg14 +(dp1768 +I0 +I3 +ssg16 +g1766 +sg17 +(dp1769 +g19 +VBrass instrument +p1770 +ssa(dp1771 +g2 +(dp1772 +g4 +Vhttp://freebase.com/view/en/flute +p1773 +sg6 +S'The flute is a musical instrument of the woodwind family. Unlike woodwind instruments with reeds, a flute is an aerophone or reedless wind instrument that produces its sound from the flow of air across an opening. According to the instrument classification of Hornbostel-Sachs, flutes are categorized as Edge-blown aerophones.\nA musician who plays the flute can be referred to as a flute player, a flautist, a flutist, or less commonly a fluter.\nAside from the voice, flutes are the earliest...' +p1774 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bdpv9 +p1775 +sg10 +VFlute (transverse) +p1776 +sg12 +Vhttp://indextank.com/_static/common/demo/02bdpv9.jpg +p1777 +ssg14 +(dp1778 +I0 +I197 +ssg16 +g1776 +sg17 +(dp1779 +g19 +VWoodwind instrument +p1780 +ssa(dp1781 +g2 +(dp1782 +g4 +Vhttp://freebase.com/view/en/serinette +p1783 +sg6 +S'A serinette is a type of mechanical musical instrument consisting of a small barrel organ. It appeared in the first half of the 18th century in eastern France, and was used to teach tunes to canaries. Its name is derived from the French serin, meaning \xe2\x80\x9ccanary.\xe2\x80\x9d\nSerinettes are housed in a wooden case, normally of walnut, and typically measuring 265 \xc3\x97 200 \xc3\x97 150 mm. The instrument is played by turning a crank mounted on the front. The crank pumps a bellows to supply air to the pipes, and also...' +p1784 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02frq4g +p1785 +sg10 +VSerinette +p1786 +sg12 +Vhttp://indextank.com/_static/common/demo/02frq4g.jpg +p1787 +ssg14 +(dp1788 +I0 +I0 +ssg16 +g1786 +sg17 +(dp1789 +g19 +VOrgan +p1790 +ssa(dp1791 +g2 +(dp1792 +g4 +Vhttp://freebase.com/view/en/psaltery +p1793 +sg6 +S'A psaltery is a stringed musical instrument of the harp or the zither family. The psaltery of Ancient Greece (Epigonion) dates from at least 2800 BC, when it was a harp-like instrument. Etymologically the word derives from the Ancient Greek \xcf\x88\xce\xb1\xce\xbb\xcf\x84\xce\xae\xcf\x81\xce\xb9\xce\xbf\xce\xbd (psalterion) \xe2\x80\x9cstringed instrument, psaltery, harp\xe2\x80\x9d and that from the verb \xcf\x88\xce\xac\xce\xbb\xce\xbb\xcf\x89 (psallo) \xe2\x80\x9cto touch sharply, to pluck, pull, twitch\xe2\x80\x9d and in the case of the strings of musical instruments, \xe2\x80\x9cto play a stringed instrument with the fingers, and not...' +p1794 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cd9nz +p1795 +sg10 +VPsaltery +p1796 +sg12 +Vhttp://indextank.com/_static/common/demo/02cd9nz.jpg +p1797 +ssg14 +(dp1798 +I0 +I2 +ssg16 +g1796 +sg17 +(dp1799 +g19 +VZither +p1800 +ssa(dp1801 +g2 +(dp1802 +g4 +Vhttp://freebase.com/view/en/novachord +p1803 +sg6 +S"The Novachord is often considered to be the world's first commercial polyphonic synthesizer. All-electronic, incorporating many circuit and control elements found in modern synths, and using subtractive synthesis to generate tones, it was designed by John Hanert, Laurens Hammond and C. N. Williams and manufactured by the Hammond company. Only some 1069 examples were built over a period from 1939 to 1942. It was one of very few electronic products released by Hammond that was not intended to..." +p1804 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0bd1s41 +p1805 +sg10 +VNovachord +p1806 +sg12 +Vhttp://indextank.com/_static/common/demo/0bd1s41.jpg +p1807 +ssg14 +(dp1808 +I0 +I0 +ssg16 +g1806 +sg17 +(dp1809 +g19 +VElectronic keyboard +p1810 +ssa(dp1811 +g2 +(dp1812 +g4 +Vhttp://freebase.com/view/en/steel-string_acoustic_guitar +p1813 +sg6 +S'A steel-string acoustic guitar is a modern form of guitar descended from the classical guitar, but strung with steel strings for a brighter, louder sound. It is often referred to simply as an acoustic guitar, although strictly speaking the nylon-strung classical guitar is acoustic as well.\nThe most common type can be called a flat-top guitar to distinguish it from the more specialized archtop guitar and other variations.\nThe standard tuning for an acoustic guitar is E-A-D-G-B-E (low to...' +p1814 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0290vfk +p1815 +sg10 +VSteel-string acoustic guitar +p1816 +sg12 +Vhttp://indextank.com/_static/common/demo/0290vfk.jpg +p1817 +ssg14 +(dp1818 +I0 +I34 +ssg16 +g1816 +sg17 +(dp1819 +g19 +VGuitar +p1820 +ssa(dp1821 +g2 +(dp1822 +g4 +Vhttp://freebase.com/view/en/electric_harp +p1823 +sg6 +S'Like electric guitars, electric harps are based on their acoustic originals, and there are both solid-body and electro-acoustic models available.\nA solid-body electric harp has no hollow soundbox, and thus makes very little noise when not amplified. Alan Stivell writes in his book Telenn, la harpe bretonne of his first dreams of electric harps going back to the late 1950s. He designed and had made a solid body (after different electric-acoustic harps) electric harp at the turn of the...' +p1824 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cqk_x +p1825 +sg10 +VElectric harp +p1826 +sg12 +Vhttp://indextank.com/_static/common/demo/02cqk_x.jpg +p1827 +ssg14 +(dp1828 +I0 +I0 +ssg16 +g1826 +sg17 +(dp1829 +g19 +VHarp +p1830 +ssa(dp1831 +g2 +(dp1832 +g4 +Vhttp://freebase.com/view/m/04czcxw +p1833 +sg6 +S'Fue (\xe7\xac\x9b, hiragana: \xe3\x81\xb5\xe3\x81\x88) is the Japanese word for flute, and refers to a class of flutes native to Japan. Fue come in many varieties, but are generally high-pitched and made of a bamboo called shinobue. The most popular of the fue is the shakuhachi.\nFue are traditionally broken up into two basic categories \xe2\x80\x93 the transverse flute and the end-blown flute. Transverse flutes are held to the side, with the musician blowing across a hole near one end; end-blown flutes are held vertically and the...' +p1834 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0cckk27 +p1835 +sg10 +VFue +p1836 +sg12 +Vhttp://indextank.com/_static/common/demo/0cckk27.jpg +p1837 +ssg14 +(dp1838 +I0 +I0 +ssg16 +g1836 +sg17 +(dp1839 +g19 +VFlute (transverse) +p1840 +ssa(dp1841 +g2 +(dp1842 +g4 +Vhttp://freebase.com/view/m/02npsp +p1843 +sg6 +S"A ratchet, also called a noisemaker (or, when used in Judaism, a gragger or grogger (etymologically from Yiddish: \xd7\x92\xd7\xa8\xd7\x90\xd6\xb7\xd7\x92\xd7\xa2\xd7\xa8) or ra'ashan (Hebrew: \xd7\xa8\xd7\xa2\xd7\xa9\xd7\x9f\xe2\x80\x8e)), is an orchestral musical instrument played by percussionists. Operating on the principle of the ratchet device, a gearwheel and a stiff board is mounted on a handle, which can be freely rotated. The handle is held and the whole mechanism is swung around, the momentum causing the board to click against the gearwheel, making a clicking and..." +p1844 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c4758 +p1845 +sg10 +VRatchet +p1846 +sg12 +Vhttp://indextank.com/_static/common/demo/02c4758.jpg +p1847 +ssg14 +(dp1848 +I0 +I0 +ssg16 +g1846 +sg17 +(dp1849 +g19 +VPercussion +p1850 +ssa(dp1851 +g2 +(dp1852 +g4 +Vhttp://freebase.com/view/en/ondes_martenot +p1853 +sg6 +S'The ondes Martenot (pronounced:\xc2\xa0[\xc9\x94\xcc\x83d ma\xca\x81t\xc9\x99no], OHND mar-t\xc9\x99-NOH, French for "Martenot waves"), also known as the ondium Martenot, Martenot and ondes musicales, is an early electronic musical instrument invented in 1928 by Maurice Martenot. The original design was similar in sound to the theremin. The sonic capabilities of the instrument were later expanded by the addition of timbral controls and switchable loudspeakers.\nThe instrument\'s eerie wavering notes are produced by varying the...' +p1854 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044vh28 +p1855 +sg10 +VOndes Martenot +p1856 +sg12 +Vhttp://indextank.com/_static/common/demo/044vh28.jpg +p1857 +ssg14 +(dp1858 +I0 +I1 +ssg16 +g1856 +sg17 +(dp1859 +g19 +VElectronic keyboard +p1860 +ssa(dp1861 +g2 +(dp1862 +g4 +Vhttp://freebase.com/view/en/yehu +p1863 +sg6 +S"The yehu (\xe6\xa4\xb0\xe8\x83\xa1; pinyin: y\xc4\x93h\xc3\xba) is a Chinese bowed string instrument in the huqin family of musical instruments. Ye means coconut and hu is short for huqin. It is used particularly in the southern coastal provinces of China and in Taiwan. The instrument's soundbox is made from a coconut shell, which is cut on the playing end and covered with a piece of coconut wood instead of the snakeskin commonly used on other huqin instruments such as the erhu or gaohu. As with most huqin the bow hair passes..." +p1864 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d9q5z +p1865 +sg10 +VYehu +p1866 +sg12 +Vhttp://indextank.com/_static/common/demo/02d9q5z.jpg +p1867 +ssg14 +(dp1868 +I0 +I0 +ssg16 +g1866 +sg17 +(dp1869 +g19 +VBowed string instruments +p1870 +ssa(dp1871 +g2 +(dp1872 +g4 +Vhttp://freebase.com/view/m/05zl9hy +p1873 +sg6 +S'The Sarangi (Nepali/Hindi: \xe0\xa4\xb8\xe0\xa4\xbe\xe0\xa4\xb0\xe0\xa4\x99\xe0\xa5\x8d\xe0\xa4\x97\xe0\xa5\x80) is a folk Nepalese string instrument. Unlike Classical Indian Sarangi, it has four strings and all of them are played. Traditionally, in Nepal, Sarangi was only played by people of Gandarva or Gaine cast, who sings narrative tales and folk song. However, in present days, its widely used and played by many.\nTraditional Nepali Sarangi is made up of single piece of wood having a neck and hollowed out body. Sarangi is carved out from a very light wood, locally...' +p1874 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/08bgn5k +p1875 +sg10 +VSarangi +p1876 +sg12 +Vhttp://indextank.com/_static/common/demo/08bgn5k.jpg +p1877 +ssg14 +(dp1878 +I0 +I0 +ssg16 +g1876 +sg17 +(dp1879 +g19 +VBowed string instruments +p1880 +ssa(dp1881 +g2 +(dp1882 +g4 +Vhttp://freebase.com/view/en/tambura +p1883 +sg6 +S'The tambura, tanpura, or tambora is a long-necked plucked lute (a stringed instrument found in different forms and in many places). The body shape of the tambura somewhat resembles that of the sitar, but it has no frets \xe2\x80\x93 only the open strings are played to accompany other musicians. It has four or five (rarely six) wire strings, which are plucked one after another in a regular pattern to create a harmonic resonance on the basic note (bourdon or drone function).\nTamburas come in different...' +p1884 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05lng4p +p1885 +sg10 +VTambura +p1886 +sg12 +Vhttp://indextank.com/_static/common/demo/05lng4p.jpg +p1887 +ssg14 +(dp1888 +I0 +I11 +ssg16 +g1886 +sg17 +(dp1889 +g19 +VPlucked string instrument +p1890 +ssa(dp1891 +g2 +(dp1892 +g4 +Vhttp://freebase.com/view/en/welsh_pipes +p1893 +sg6 +S'Welsh bagpipes (Welsh pibau, pipa c\xc5\xb5d, pibau c\xc5\xb5d, pibgod, cotbib, pibau cyrn, chwibanogl a chod, sachbib, backpipes or bacbib) have been documented, represented or described in Wales since the fourteenth century. In 1376, the poet Iolo Goch describes the instrument in his Cywydd to Syr Hywel y Fwyall.. Also, in the same century, Brut y Tywysogion ("Chronicle of the Princes"), written around 1330 AD, states that there are three types of wind instrument: Organ, a Phibeu a Cherd y got ("organ,...' +p1894 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05mfh7c +p1895 +sg10 +VWelsh bagpipes +p1896 +sg12 +Vhttp://indextank.com/_static/common/demo/05mfh7c.jpg +p1897 +ssg14 +(dp1898 +I0 +I0 +ssg16 +g1896 +sg17 +(dp1899 +g19 +VBagpipes +p1900 +ssa(dp1901 +g2 +(dp1902 +g4 +Vhttp://freebase.com/view/en/roman_tuba +p1903 +sg6 +S'The tuba of ancient Rome is a military signal trumpet, quite different from the modern tuba. The tuba (from Latin tubus, "tube") was produced around 500 BC. Its shape was straight, in contrast to the military buccina or cornu, which was more like the modern tuba in curving around the body. Its origin is thought to be Etruscan, and it is similar to the Greek salpinx. About four feet in length, it was made usually of bronze, and was played with a detachable bone mouthpiece.' +p1904 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02ghk4m +p1905 +sg10 +VRoman tuba +p1906 +sg12 +Vhttp://indextank.com/_static/common/demo/02ghk4m.jpg +p1907 +ssg14 +(dp1908 +I0 +I0 +ssg16 +g1906 +sg17 +(dp1909 +g19 +VBrass instrument +p1910 +ssa(dp1911 +g2 +(dp1912 +g4 +Vhttp://freebase.com/view/en/musette_de_cour +p1913 +sg6 +S'The musette de cour or baroque musette is a musical instrument of the bagpipe family. Visually, the musette is characterised by the short, cylindrical shuttle-drone and the two chalumeaux. Both the chanters and the drones have a cylindrical bore and use a double reed, giving a quiet tone similar to the oboe. The instrument is always bellows-blown.\nNote: the qualified name de cour does not appear in original music for the instrument; title-pages refer to it simply as musette, allowing...' +p1914 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dxqsx +p1915 +sg10 +VMusette de cour +p1916 +sg12 +Vhttp://indextank.com/_static/common/demo/02dxqsx.jpg +p1917 +ssg14 +(dp1918 +I0 +I0 +ssg16 +g1916 +sg17 +(dp1919 +g19 +VBagpipes +p1920 +ssa(dp1921 +g2 +(dp1922 +g4 +Vhttp://freebase.com/view/en/silent_piano +p1923 +sg6 +S'A silent piano is an acoustic piano where there is an option to silence the strings by means of an interposing hammer bar. A silent piano is designed for private silent practice. In the silent mode, sensors pick up the piano key movement. Older models used mechanical sensors which affected the touch and produced a clicking sound, whereas newer models use optical sensors which do not affect the feel or sound of the piano. The key movement is then converted to a midi signal and can link to a...' +p1924 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/09hq9r4 +p1925 +sg10 +VSilent piano +p1926 +sg12 +Vhttp://indextank.com/_static/common/demo/09hq9r4.jpg +p1927 +ssg14 +(dp1928 +I0 +I0 +ssg16 +g1926 +sg17 +(dp1929 +g19 +VPiano +p1930 +ssa(dp1931 +g2 +(dp1932 +g4 +Vhttp://freebase.com/view/en/electronic_drum +p1933 +sg6 +S'An electronic drum is a percussion instrument in which the sound is generated by an electronic waveform generator or sampler instead of by acoustic vibration.\nWhen an electronic drum pad is struck, a voltage change is triggered in the embedded piezoelectric transducer (piezo) or force sensitive resistor (FSR). The resultant signals are transmitted to an electronic "drum brain" via TS or TRS cables, and are translated into digital waveforms, which produce the desired percussion sound assigned...' +p1934 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f88sx +p1935 +sg10 +VElectronic drum +p1936 +sg12 +Vhttp://indextank.com/_static/common/demo/02f88sx.jpg +p1937 +ssg14 +(dp1938 +I0 +I12 +ssg16 +g1936 +sg17 +(dp1939 +g19 +VPercussion +p1940 +ssa(dp1941 +g2 +(dp1942 +g4 +Vhttp://freebase.com/view/en/piano_accordion +p1943 +sg6 +S'A piano accordion is an accordion equipped with a right-hand keyboard similar to a piano or organ. Its acoustic mechanism is more similar to that of an organ than a piano, as they are both wind instruments, but the term "piano accordion"\xe2\x80\x94coined by Guido Deiro in 1910\xe2\x80\x94has remained the popular nomenclature. It may be equipped with any of the available systems for the left-hand manual.\nIn comparison to a piano keyboard, the keys are more rounded, smaller, and lighter to the touch. These go...' +p1944 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04xzfth +p1945 +sg10 +VPiano accordion +p1946 +sg12 +Vhttp://indextank.com/_static/common/demo/04xzfth.jpg +p1947 +ssg14 +(dp1948 +I0 +I1 +ssg16 +g1946 +sg17 +(dp1949 +g19 +VAccordion +p1950 +ssa(dp1951 +g2 +(dp1952 +g4 +Vhttp://freebase.com/view/en/zetland_pipes +p1953 +sg6 +S'The Zetland pipes were a type of bagpipe designed and crafted by Pipe Major Royce Lerwick in the 1990s.\nLerwick believed that the bagpipes had been introduced to the British Isles by the Vikings. His "Zetland pipes" were intended to resemble single-drone, single-reeded pipes such as might have been brought to the Shetland Islands by the Vikings. The term "Zetland" is an antiquated variant of "Shetland".\nThe original impetus for the design, according to Lerwick, was the Lady Maket pipes, or...' +p1954 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/063bbz6 +p1955 +sg10 +VZetland pipes +p1956 +sg12 +Vhttp://indextank.com/_static/common/demo/063bbz6.jpg +p1957 +ssg14 +(dp1958 +I0 +I0 +ssg16 +g1956 +sg17 +(dp1959 +g19 +VBagpipes +p1960 +ssa(dp1961 +g2 +(dp1962 +g4 +Vhttp://freebase.com/view/en/igil +p1963 +sg6 +S'An igil (Tuvan- \xd0\xb8\xd0\xb3\xd0\xb8\xd0\xbb) is a two-stringed Tuvan musical instrument, played by bowing the strings. (It is called "ikili" in Western Mongolia.) The neck and lute-shaped sound box are usually made of a solid piece of pine or larch. The top of the sound box may be covered with skin or a thin wooden plate. The strings, and those of the bow, are traditionally made of hair from a horse\'s tail (strung parallel), but may also be made of nylon. Like the morin khuur of Mongolia, the igil typically...' +p1964 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03tck74 +p1965 +sg10 +VIgil +p1966 +sg12 +Vhttp://indextank.com/_static/common/demo/03tck74.jpg +p1967 +ssg14 +(dp1968 +I0 +I0 +ssg16 +g1966 +sg17 +(dp1969 +g19 +VBowed string instruments +p1970 +ssa(dp1971 +g2 +(dp1972 +g4 +Vhttp://freebase.com/view/en/alto_clarinet +p1973 +sg6 +S'The alto clarinet is a wind instrument of the clarinet family. It is a transposing instrument pitched in the key of E\xe2\x99\xad, though instruments in F (and in the 19th century, E) have been made. It is sometimes known as a tenor clarinet; this name especially is applied to the instrument in F. In size it lies between the soprano clarinet and the bass clarinet, to which it bears a greater resemblance in that it typically has a straight body (made of Grenadilla or other wood, hard rubber, or...' +p1974 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bvl5n +p1975 +sg10 +VAlto clarinet +p1976 +sg12 +Vhttp://indextank.com/_static/common/demo/02bvl5n.jpg +p1977 +ssg14 +(dp1978 +I0 +I3 +ssg16 +g1976 +sg17 +(dp1979 +g19 +VClarinet +p1980 +ssa(dp1981 +g2 +(dp1982 +g4 +Vhttp://freebase.com/view/en/hellier_stradivari +p1983 +sg6 +S'The Hellier Stradivarius of circa 1679 is a violin made by Antonio Stradivari of Cremona, Italy. It derives its name from the Hellier family, who might well have bought it directly from the luthier himself.\nThe Hellier Stradivarius has had a convoluted ownership history. It seems to have been in the possession of the Hellier family from the beginning of the 18th century. Sir Samuel Hellier, High Sheriff of Staffordshire 1745-1749, brought the violin to England, and through various wills it...' +p1984 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03rb6yt +p1985 +sg10 +VHellier Stradivarius +p1986 +sg12 +Vhttp://indextank.com/_static/common/demo/03rb6yt.jpg +p1987 +ssg14 +(dp1988 +I0 +I0 +ssg16 +g1986 +sg17 +(dp1989 +g19 +VViolin +p1990 +ssa(dp1991 +g2 +(dp1992 +g4 +Vhttp://freebase.com/view/en/rubab +p1993 +sg6 +S'Rubab or robab (Persian: \xd8\xb1\xd9\x8f\xd8\xa8\xd8\xa7\xd8\xa8 rub\xc4\x81b, Urdu And Pashto \xd8\xb1\xd8\xa8\xd8\xa7\xd8\xa8, Tajik and Uzbek \xd1\x80\xd1\x83\xd0\xb1\xd0\xbe\xd0\xb1) is a lute-like musical instrument originally from Afghanistan but is also played in the neighbouring countries, especially Pakistan. It derives its name from the Arab rebab which means "played with a bow" but the Central Asian instrument is plucked, and is distinctly different in construction. The rubab is mainly used by Pashtun, Tajik, Kashmiri and Iranian Kurdish classical musicians.\nThe rubab is a...' +p1994 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02h1k1w +p1995 +sg10 +VRubab +p1996 +sg12 +Vhttp://indextank.com/_static/common/demo/02h1k1w.jpg +p1997 +ssg14 +(dp1998 +I0 +I1 +ssg16 +g1996 +sg17 +(dp1999 +g19 +VPlucked string instrument +p2000 +ssa(dp2001 +g2 +(dp2002 +g4 +Vhttp://freebase.com/view/en/great_irish_warpipes +p2003 +sg6 +S'The Great Irish Warpipes (Irish: p\xc3\xadob mh\xc3\xb3r; literally "great pipes") are an instrument that in modern practice is identical, and historically was analogous or identical to the Great Highland Bagpipe. "Warpipes" is an English term; The first use of the Gaelic term in Ireland is recorded in a poem by John O\'Naughton (c. 1650-1728), in which the bagpipes are referred to as p\xc3\xadb mh\xc3\xb3r. The p\xc3\xadob mh\xc3\xb3r has a long and significant history in Ireland.\nIn Gaelic Ireland and Scotland, the bagpipe seems to...' +p2004 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s1xj3 +p2005 +sg10 +VGreat Irish Warpipes +p2006 +sg12 +Vhttp://indextank.com/_static/common/demo/03s1xj3.jpg +p2007 +ssg14 +(dp2008 +I0 +I0 +ssg16 +g2006 +sg17 +(dp2009 +g19 +VBagpipes +p2010 +ssa(dp2011 +g2 +(dp2012 +g4 +Vhttp://freebase.com/view/en/flageolet +p2013 +sg6 +S'A flageolet is a woodwind musical instrument and a member of the fipple flute family. Its invention is ascribed to the 16th century Sieur Juvigny in 1581. It had 4 holes on the front and 2 on the back. The English instrument maker William Bainbridge developed it further and patented the "improved English flageolet" in 1803 as well as the double flageolet around 1805. They were continued to be made until the 19th century when it was succeeded by the tin whistle.\nFlageolets have varied greatly...' +p2014 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f9ht0 +p2015 +sg10 +VFlageolet +p2016 +sg12 +Vhttp://indextank.com/_static/common/demo/02f9ht0.jpg +p2017 +ssg14 +(dp2018 +I0 +I0 +ssg16 +g2016 +sg17 +(dp2019 +g19 +VDuct flutes +p2020 +ssa(dp2021 +g2 +(dp2022 +g4 +Vhttp://freebase.com/view/en/arp_string_ensemble +p2023 +sg6 +S"The ARP String Ensemble, also known as the Solina String Ensemble, is a fully polyphonic multi-orchestral ARP Instruments, Inc. synthesizer with a 49-key keyboard, produced by Solina from 1974 to 1981. The sounds it incorporates are violin, viola, trumpet, horn, cello and contrabass. The keyboard uses 'organ style' divide-down technology to make it polyphonic. The built-in chorus effect gives the instrument its famous sound.\nThe core technology is based on the String Section of the Eminent..." +p2024 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bv07p +p2025 +sg10 +VARP String Ensemble +p2026 +sg12 +Vhttp://indextank.com/_static/common/demo/02bv07p.jpg +p2027 +ssg14 +(dp2028 +I0 +I0 +ssg16 +g2026 +sg17 +(dp2029 +g19 +VSynthesizer +p2030 +ssa(dp2031 +g2 +(dp2032 +g4 +Vhttp://freebase.com/view/en/crash_cymbal +p2033 +sg6 +S'A crash cymbal is a type of cymbal that produces a loud, sharp "crash" and is used mainly for occasional accents, as opposed to in ostinato. The term "crash" may have been first used by Zildjian in 1928. They can be mounted on a stand and played with a drum stick, or by hand in pairs. One or two crash cymbals are a standard part of a drum kit. Suspended crash cymbals are also used in bands and orchestras, either played with a drumstick or rolled with a pair of mallets to produce a slower,...' +p2034 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bdbkf +p2035 +sg10 +VCrash cymbal +p2036 +sg12 +Vhttp://indextank.com/_static/common/demo/02bdbkf.jpg +p2037 +ssg14 +(dp2038 +I0 +I0 +ssg16 +g2036 +sg17 +(dp2039 +g19 +VCymbal +p2040 +ssa(dp2041 +g2 +(dp2042 +g4 +Vhttp://freebase.com/view/en/trautonium +p2043 +sg6 +S"The trautonium is a monophonic electronic musical instrument invented about 1929 by Friedrich Trautwein in Berlin at the Musikhochschule's music and radio lab, the Rundfunkversuchstelle. Soon Oskar Sala joined him, continuing development until Sala's death in 2002. Instead of a keyboard, its manual is made of a resistor wire over a metal plate which is pressed to create a sound. Expressive playing was possible with this wire by gliding on it, creating vibrato with small movements. Volume was..." +p2044 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05kqj3y +p2045 +sg10 +VTrautonium +p2046 +sg12 +Vhttp://indextank.com/_static/common/demo/05kqj3y.jpg +p2047 +ssg14 +(dp2048 +I0 +I0 +ssg16 +g2046 +sg17 +(dp2049 +g19 +VElectronic keyboard +p2050 +ssa(dp2051 +g2 +(dp2052 +g4 +Vhttp://freebase.com/view/m/0151b0 +p2053 +sg6 +S'The triangle is an idiophone type of musical instrument in the percussion family. It is a bar of metal, usually steel but sometimes other metals such as beryllium copper, bent into a triangle shape. The instrument is usually held by a loop of some form of thread or wire at the top curve. It was first made around the 16th century.\nOn a triangle instrument, one of the angles is left open, with the ends of the bar not quite touching. This causes the instrument to be of indeterminate or not...' +p2054 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c091h +p2055 +sg10 +VTriangle +p2056 +sg12 +Vhttp://indextank.com/_static/common/demo/02c091h.jpg +p2057 +ssg14 +(dp2058 +I0 +I2 +ssg16 +g2056 +sg17 +(dp2059 +g19 +VPercussion +p2060 +ssa(dp2061 +g2 +(dp2062 +g4 +Vhttp://freebase.com/view/en/glass_harmonica +p2063 +sg6 +S'The glass harmonica, also known as the glass armonica, bowl organ, hydrocrystalophone, or simply the armonica (derived from "harmonia," the Greek word for harmony), is a type of musical instrument that uses a series of glass bowls or goblets graduated in size to produce musical tones by means of friction (instruments of this type are known as friction idiophones).\nBecause its sounding portion is made of glass, the glass harmonica is a crystallophone. The phenomenon of rubbing a wet finger...' +p2064 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cl0sp +p2065 +sg10 +VGlass harmonica +p2066 +sg12 +Vhttp://indextank.com/_static/common/demo/02cl0sp.jpg +p2067 +ssg14 +(dp2068 +I0 +I2 +ssg16 +g2066 +sg17 +(dp2069 +g19 +VCrystallophone +p2070 +ssa(dp2071 +g2 +(dp2072 +g4 +Vhttp://freebase.com/view/en/shakuhachi +p2073 +sg6 +S'The shakuhachi (\xe5\xb0\xba\xe5\x85\xab, pronounced\xc2\xa0[\xc9\x95ak\xc9\xafhat\xc9\x95i]) is a Japanese end-blown flute. It is traditionally made of bamboo, but versions now exist in ABS and hardwoods. It was used by the monks of the Fuke school of Zen Buddhism in the practice of suizen (\xe5\x90\xb9\xe7\xa6\x85, blowing meditation). Its soulful sound made it popular in 1980s pop music in the English-speaking world.\nThey are often made in the minor pentatonic scale.\nThe name shakuhachi means "1.8 shaku", referring to its size. It is a compound of two...' +p2074 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c3rxm +p2075 +sg10 +VShakuhachi +p2076 +sg12 +Vhttp://indextank.com/_static/common/demo/02c3rxm.jpg +p2077 +ssg14 +(dp2078 +I0 +I3 +ssg16 +g2076 +sg17 +(dp2079 +g19 +VFlute (transverse) +p2080 +ssa(dp2081 +g2 +(dp2082 +g4 +Vhttp://freebase.com/view/en/treble_flute +p2083 +sg6 +S'The treble flute is a member of the flute family. It is in the key of G, pitched a fifth above the concert flute and is a transposing instrument, sounding a fifth up from the written note. The instrument is rare today, only occasionally found in flute choirs, some marching bands or private collections. Some 19th century operas, such as Ivanhoe include the instrument in their orchestrations.\nA limited number of manufacturers produce G treble flute, including Myall-Allen and Flutemakers Guild....' +p2084 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05mg4hy +p2085 +sg10 +VTreble flute +p2086 +sg12 +Vhttp://indextank.com/_static/common/demo/05mg4hy.jpg +p2087 +ssg14 +(dp2088 +I0 +I0 +ssg16 +g2086 +sg17 +(dp2089 +g19 +VFlute (transverse) +p2090 +ssa(dp2091 +g2 +(dp2092 +g4 +Vhttp://freebase.com/view/en/jinghu +p2093 +sg6 +S'The jinghu (\xe4\xba\xac\xe8\x83\xa1; pinyin: j\xc4\xabngh\xc3\xba) is a Chinese bowed string instrument in the huqin family, used primarily in Beijing opera. It is the smallest and highest pitched instrument in the huqin family.\nLike most of its relatives, the jinghu has two strings that are customarily tuned to the interval of a fifth which the hair of the non-detachable bow passes in between. The strings were formerly made of silk, but in modern times are increasingly made of steel or nylon. Unlike other huqin instruments...' +p2094 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02fgb_1 +p2095 +sg10 +VJinghu +p2096 +sg12 +Vhttp://indextank.com/_static/common/demo/02fgb_1.jpg +p2097 +ssg14 +(dp2098 +I0 +I0 +ssg16 +g2096 +sg17 +(dp2099 +g19 +VBowed string instruments +p2100 +ssa(dp2101 +g2 +(dp2102 +g4 +Vhttp://freebase.com/view/en/maraca +p2103 +sg6 +S'Maracas ( pronunciation (help\xc2\xb7info), sometimes called rumba shakers) are a native instrument of Puerto Rico, Cuba, Colombia, Guatemala and several nations of the Caribbean and Latin America. They are simple percussion instruments (idiophones), usually played in pairs, consisting of a dried calabash or gourd shell (cuia "cue-ya") or coconut shell filled with seeds or dried beans. They may also be made of leather, wood, or plastic.\nOften one ball is pitched high and the other is pitched low....' +p2104 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05mgb3m +p2105 +sg10 +VMaracas +p2106 +sg12 +Vhttp://indextank.com/_static/common/demo/05mgb3m.jpg +p2107 +ssg14 +(dp2108 +I0 +I7 +ssg16 +g2106 +sg17 +(dp2109 +g19 +VPercussion +p2110 +ssa(dp2111 +g2 +(dp2112 +g4 +Vhttp://freebase.com/view/en/khim +p2113 +sg6 +S'The khim (Thai: \xe0\xb8\x82\xe0\xb8\xb4\xe0\xb8\xa1, Thai pronunciation:\xc2\xa0[k\xca\xb0\xc7\x90m]; Khmer: \xe1\x9e\x83\xe1\x9e\xb9\xe1\x9e\x98) is a hammered dulcimer from Thailand and Cambodia. It is made of wood and trapezoidal in shape, with brass strings that are laid across the instrument. There are 14 groups of strings on the khim, and each group has 3 strings. Overall, the khim has a total of 42 strings. It is played with two flexible bamboo sticks with soft leather at the tips to produce the soft tone. It is used as both a solo and ensemble instrument.\nThe...' +p2114 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dbwq_ +p2115 +sg10 +VKhim +p2116 +sg12 +Vhttp://indextank.com/_static/common/demo/02dbwq_.jpg +p2117 +ssg14 +(dp2118 +I0 +I0 +ssg16 +g2116 +sg17 +(dp2119 +g19 +VStruck string instruments +p2120 +ssa(dp2121 +g2 +(dp2122 +g4 +Vhttp://freebase.com/view/en/guzheng +p2123 +sg6 +S'The guzheng, also spelled gu zheng or gu-zheng (Chinese: \xe5\x8f\xa4\xe7\xae\x8f; pinyin: g\xc7\x94zh\xc4\x93ng, with gu \xe5\x8f\xa4 meaning "ancient"); and also called zheng (\xe7\xae\x8f) is a Chinese plucked zither. It has 13-21 strings and movable bridges.\nThe guzheng is a similar instrument to many Asian instruments such as the Japanese koto, the Mongolian yatga, the Korean gayageum and the Vietnamese \xc4\x91\xc3\xa0n tranh.\nThe guzheng should not to be confused with the guqin (another ancient Chinese zither but a fewer number of strings and without...' +p2124 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02b89jl +p2125 +sg10 +VGuzheng +p2126 +sg12 +Vhttp://indextank.com/_static/common/demo/02b89jl.jpg +p2127 +ssg14 +(dp2128 +I0 +I2 +ssg16 +g2126 +sg17 +(dp2129 +g19 +VPlucked string instrument +p2130 +ssa(dp2131 +g2 +(dp2132 +g4 +Vhttp://freebase.com/view/en/gaita_sanabresa +p2133 +sg6 +S'The gaita sanabresa is a type of bagpipe native to Sanabria, a comarca of the province of Zamora in northwestern Spain.\nThe gaita sanabresa features a single drone. The scale of this chanter is distinct from others in Spain, more resembling the gaita transmontana in the neighboring regions of Portugal, as well as the gaita alistana of Aliste. In playing, the fingering is generally open, though some players use semi-closed touches.\nThe instrument was in decline in the 20th century and nearly...' +p2134 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0637vmv +p2135 +sg10 +VGaita sanabresa +p2136 +sg12 +Vhttp://indextank.com/_static/common/demo/0637vmv.jpg +p2137 +ssg14 +(dp2138 +I0 +I0 +ssg16 +g2136 +sg17 +(dp2139 +g19 +VBagpipes +p2140 +ssa(dp2141 +g2 +(dp2142 +g4 +Vhttp://freebase.com/view/en/pastoral_pipes +p2143 +sg6 +S'The Pastoral Pipe (also known as the Scottish Pastoral pipes, Hybrid Union pipes, Organ pipe and Union pipe) was a bellows-blown bagpipe, widely recognised as the forerunner and ancestor of the 19th-century Union pipes, which became the Uilleann Pipes of today. Similar in design and construction, it had a foot joint in order to play a low leading note and plays a two octave chromatic scale. There is a tutor for the "Pastoral or New Bagpipe" by J. Geoghegan, published in London in 1745.. It...' +p2144 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04rvbry +p2145 +sg10 +VPastoral pipes +p2146 +sg12 +Vhttp://indextank.com/_static/common/demo/04rvbry.jpg +p2147 +ssg14 +(dp2148 +I0 +I0 +ssg16 +g2146 +sg17 +(dp2149 +g19 +VBagpipes +p2150 +ssa(dp2151 +g2 +(dp2152 +g4 +Vhttp://freebase.com/view/en/gusli +p2153 +sg6 +S'Gusli (Russian: \xd0\x93\xd1\x83\xd1\x81\xd0\xbb\xd0\xb8) is the oldest Russian multi-string plucked instrument. Its exact history is unknown, but it may have derived from a Byzantine form of the Greek kythare, which in turn derived from the ancient lyre. It has its relatives throughout the world - kantele in Finland, kannel in Estonia, kankles and kokle in Lithuania and Latvia. Furthermore, we can find kanun in Arabic countries and the autoharp in the USA. It is also related to such ancient instruments as Chinese gu zheng...' +p2154 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04252vn +p2155 +sg10 +VGusli +p2156 +sg12 +Vhttp://indextank.com/_static/common/demo/04252vn.jpg +p2157 +ssg14 +(dp2158 +I0 +I0 +ssg16 +g2156 +sg17 +(dp2159 +g19 +VZither +p2160 +ssa(dp2161 +g2 +(dp2162 +g4 +Vhttp://freebase.com/view/en/piccolo_clarinet +p2163 +sg6 +S'The piccolo clarinets are members of the clarinet family, smaller and higher pitched than the more familiar high soprano clarinets in E\xe2\x99\xad and D. None are common, but the most often used piccolo clarinet is the A\xe2\x99\xad clarinet, sounding a minor seventh higher than the B\xe2\x99\xad clarinet. Shackleton also lists obsolete instruments in C, B\xe2\x99\xad, and A\xe2\x99\xae. Some writers call these sopranino clarinets or octave clarinets. The boundary between the piccolo and soprano clarinets is not well-defined, and the rare...' +p2164 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0425c82 +p2165 +sg10 +VPiccolo clarinet +p2166 +sg12 +Vhttp://indextank.com/_static/common/demo/0425c82.jpg +p2167 +ssg14 +(dp2168 +I0 +I0 +ssg16 +g2166 +sg17 +(dp2169 +g19 +VClarinet +p2170 +ssa(dp2171 +g2 +(dp2172 +g4 +Vhttp://freebase.com/view/en/tabla +p2173 +sg6 +S'The tabla (or tabl, tabla) (Hindi: \xe0\xa4\xa4\xe0\xa4\xac\xe0\xa4\xb2\xe0\xa4\xbe, Marathi: \xe0\xa4\xa4\xe0\xa4\xac\xe0\xa4\xb2\xe0\xa4\xbe, Kannada: \xe0\xb2\xa4\xe0\xb2\xac\xe0\xb2\xb2, Telugu: \xe0\xb0\xa4\xe0\xb0\xac\xe0\xb0\xb2, Tamil: \xe0\xae\xa4\xe0\xae\xaa\xe0\xaf\x87\xe0\xae\xb2\xe0\xae\xbe, Bengali: \xe0\xa6\xa4\xe0\xa6\xac\xe0\xa6\xb2\xe0\xa6\xbe, Nepali: \xe0\xa4\xa4\xe0\xa4\xac\xe0\xa4\xb2\xe0\xa4\xbe, Urdu: \xd8\xb7\xd8\xa8\xd9\x84\xdb\x81, Arabic: \xd8\xb7\xd8\xa8\xd9\x84\xd8\x8c \xd8\xb7\xd8\xa8\xd9\x84\xd8\xa9\xe2\x80\x8e) is a popular Indian percussion instrument (of the membranophone family) used in Hindustani classical music and in popular and devotional music of the Indian subcontinent. The instrument consists of a pair of hand drums of contrasting sizes and timbres. The term \'tabla is derived from an Arabic word, tabl, which simply means "drum."...' +p2174 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029l5st +p2175 +sg10 +VTabla +p2176 +sg12 +Vhttp://indextank.com/_static/common/demo/029l5st.jpg +p2177 +ssg14 +(dp2178 +I0 +I23 +ssg16 +g2176 +sg17 +(dp2179 +g19 +VPercussion +p2180 +ssa(dp2181 +g2 +(dp2182 +g4 +Vhttp://freebase.com/view/en/gandingan_a_kayo +p2183 +sg6 +S'The gandingan a kayo (translated means, \xe2\x80\x9cwooden gandingan,\xe2\x80\x9d or \xe2\x80\x9cgandingan made of wood\xe2\x80\x9d) is a Philippine xylophone and considered the wooden version of the real gandingan. This instrument is a relatively new instrument coming of age due to the increasing popularity of the \xe2\x80\x9cwooden kulintang ensemble,\xe2\x80\x9d but unfortunately, there is nothing traditional about it and they cannot be used for apad, communicating long distances like the real gandingan.' +p2184 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0428rh2 +p2185 +sg10 +VGandingan a Kayo +p2186 +sg12 +Vhttp://indextank.com/_static/common/demo/0428rh2.jpg +p2187 +ssg14 +(dp2188 +I0 +I0 +ssg16 +g2186 +sg17 +(dp2189 +g19 +VXylophone +p2190 +ssa(dp2191 +g2 +(dp2192 +g4 +Vhttp://freebase.com/view/en/gusle +p2193 +sg6 +S'The gusle or lahuta (or gusla) (Albanian: lahuta, Bulgarian: \xd0\xb3\xd1\x83\xd1\x81\xd0\xbb\xd0\xb0, Croatian: gusle, Romanian: guzl\xc4\x83, Serbian: \xd0\xb3\xd1\x83\xd1\x81\xd0\xbb\xd0\xb5), is a single-stringed musical instrument used in the Balkans and in the Dinarides region.\nThe term gusle/gusli/husli/husla is common term to all Slavic languages and denotes a musical instrument with strings. The gusle should, however, not be confused with the Russian gusli, which is a psaltery-like instrument; nor with the Czech term for violin, housle.\nThe Gusle has many...' +p2194 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029tqpb +p2195 +sg10 +VGusle +p2196 +sg12 +Vhttp://indextank.com/_static/common/demo/029tqpb.jpg +p2197 +ssg14 +(dp2198 +I0 +I0 +ssg16 +g2196 +sg17 +(dp2199 +g19 +VString instrument +p2200 +ssa(dp2201 +g2 +(dp2202 +g4 +Vhttp://freebase.com/view/en/bugle +p2203 +sg6 +S"The bugle is one of the simplest brass instruments, having no valves or other pitch-altering devices. All pitch control is done by varying the player's embouchure, since the bugle has no other mechanism for controlling pitch. Consequently, the bugle is limited to notes within the harmonic series. See bugle call for scores to standard bugle calls, which all consist of only five notes.\nThe bugle developed from early musical or communication instruments made of animal horns, with the word..." +p2204 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02994bd +p2205 +sg10 +VBugle +p2206 +sg12 +Vhttp://indextank.com/_static/common/demo/02994bd.jpg +p2207 +ssg14 +(dp2208 +I0 +I3 +ssg16 +g2206 +sg17 +(dp2209 +g19 +VBrass instrument +p2210 +ssa(dp2211 +g2 +(dp2212 +g4 +Vhttp://freebase.com/view/en/qinqin +p2213 +sg6 +S'The qinqin (\xe7\xa7\xa6\xe7\x90\xb4; pinyin: q\xc3\xadnq\xc3\xadn) is a plucked Chinese lute. It was originally manufactured with a wooden body, a slender fretted neck, and three strings. Its body can be either round, hexagonal (with rounded sides), or octagonal. Often, only two strings were used, as in certain regional silk-and-bamboo ensembles. In its hexagonal form (with rounded sides), it is also referred to as meihuaqin (\xe6\xa2\x85\xe8\x8a\xb1\xe7\x90\xb4, literally "plum blossom instrument"). \nThe qinqin is particularly popular in southern China: in...' +p2214 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05mmdp1 +p2215 +sg10 +VQinqin +p2216 +sg12 +Vhttp://indextank.com/_static/common/demo/05mmdp1.jpg +p2217 +ssg14 +(dp2218 +I0 +I0 +ssg16 +g2216 +sg17 +(dp2219 +g19 +VPlucked string instrument +p2220 +ssa(dp2221 +g2 +(dp2222 +g4 +Vhttp://freebase.com/view/en/masenqo +p2223 +sg6 +S'The masenqo (also spelled masenko or masinqo) is a single-string violin . The square- or diamond-shaped resonator is normally covered with parchment or rawhide. The instrument is tuned by means of a large tuning peg. As with the krar, this is an instrument used by Ethiopian minstrels, or azmaris ("singer" in Amharic). The masenqo requires considerable virtuosity.' +p2224 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f0470 +p2225 +sg10 +VMasenqo +p2226 +sg12 +Vhttp://indextank.com/_static/common/demo/02f0470.jpg +p2227 +ssg14 +(dp2228 +I0 +I0 +ssg16 +g2226 +sg17 +(dp2229 +g19 +VBowed string instruments +p2230 +ssa(dp2231 +g2 +(dp2232 +g4 +Vhttp://freebase.com/view/en/subcontrabass_saxophone +p2233 +sg6 +S'The subcontrabass saxophone is a type of saxophone that Adolphe Sax patented and planned to build but never constructed. Sax called this imagined instrument saxophone bourdon (named after the lowest stop on the pipe organ). It would have been a transposing instrument pitched in B\xe2\x99\xad, one octave below the bass saxophone and two octaves below the tenor saxophone.\nUntil 1999, no genuine, playable subcontrabass saxophones were made, though at least two gigantic saxophones seem to have been built...' +p2234 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02fx9nl +p2235 +sg10 +VSubcontrabass saxophone +p2236 +sg12 +Vhttp://indextank.com/_static/common/demo/02fx9nl.jpg +p2237 +ssg14 +(dp2238 +I0 +I0 +ssg16 +g2236 +sg17 +(dp2239 +g19 +VSaxophone +p2240 +ssa(dp2241 +g2 +(dp2242 +g4 +Vhttp://freebase.com/view/en/galician_gaita +p2243 +sg6 +S'The (Galician) gaita or gaita de foles is a traditional bagpipe of Galicia, Asturias and northern Portugal.\nThe name gaita is used in Galician, Spanish, Leonese and Portuguese languages as a generic term for "bagpipe".\nJust like "Northumbrian smallpipe"\' or "Great Highland Bagpipe", each country and region attributes its toponym to the respective gaita name: gaita galega (Galicia), gaita transmontana (Tr\xc3\xa1s-os-Montes), gaita asturiana (Asturias), gaita sanabresa (Sanabria), sac de gemecs...' +p2244 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c9kgg +p2245 +sg10 +VGalician gaita +p2246 +sg12 +Vhttp://indextank.com/_static/common/demo/02c9kgg.jpg +p2247 +ssg14 +(dp2248 +I0 +I0 +ssg16 +g2246 +sg17 +(dp2249 +g19 +VBagpipes +p2250 +ssa(dp2251 +g2 +(dp2252 +g4 +Vhttp://freebase.com/view/en/celesta +p2253 +sg6 +S'The celesta ( /s\xc9\xaa\xcb\x88l\xc9\x9bst\xc9\x99/) or celeste ( /s\xc9\xaa\xcb\x88l\xc9\x9bst/) is a struck idiophone operated by a keyboard. Its appearance is similar to that of an upright piano (four- or five-octave) or of a large wooden music box (three-octave). The keys are connected to hammers which strike a graduated set of metal (usually steel) plates suspended over wooden resonators. On four or five octave models one pedal is usually available to sustain or dampen the sound. The three-octave instruments do not have a pedal, due...' +p2254 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0292dqw +p2255 +sg10 +VCelesta +p2256 +sg12 +Vhttp://indextank.com/_static/common/demo/0292dqw.jpg +p2257 +ssg14 +(dp2258 +I0 +I7 +ssg16 +g2256 +sg17 +(dp2259 +g19 +VPercussion +p2260 +ssa(dp2261 +g2 +(dp2262 +g4 +Vhttp://freebase.com/view/en/conga +p2263 +sg6 +S'The conga (pronounced c\xc5\xabnga), or more properly the tumbadora, is a tall, narrow, single-headed Cuban drum with African antecedents. It is thought to be derived from the Makuta drums or similar drums associated with Afro-Cubans of Central African descent. A person who plays conga is called a conguero. Although ultimately derived from African drums made from hollowed logs, the Cuban conga is staved, like a barrel. These drums were probably made from salvaged barrels originally. They are used...' +p2264 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029w55f +p2265 +sg10 +VConga +p2266 +sg12 +Vhttp://indextank.com/_static/common/demo/029w55f.jpg +p2267 +ssg14 +(dp2268 +I0 +I19 +ssg16 +g2266 +sg17 +(dp2269 +g19 +VPercussion +p2270 +ssa(dp2271 +g2 +(dp2272 +g4 +Vhttp://freebase.com/view/en/archlute +p2273 +sg6 +S"The archlute (Spanish archila\xc3\xbad, Italian arciliuto, German Erzlaute, Russian \xd0\x90\xd1\x80\xd1\x85\xd0\xb8\xd0\xbb\xd1\x8e\xd1\x82\xd0\xbd\xd1\x8f) is a European plucked string instrument developed around 1600 as a compromise between the very large theorbo, the size and re-entrant tuning of which made for difficulties in the performance of solo music, and the Renaissance tenor lute, which lacked the bass range of the theorbo. Essentially a tenor lute with the theorbo's neck-extension, the archlute lacks the power in the tenor and the bass that the..." +p2274 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d1d7j +p2275 +sg10 +VArchlute +p2276 +sg12 +Vhttp://indextank.com/_static/common/demo/02d1d7j.jpg +p2277 +ssg14 +(dp2278 +I0 +I0 +ssg16 +g2276 +sg17 +(dp2279 +g19 +VLute +p2280 +ssa(dp2281 +g2 +(dp2282 +g4 +Vhttp://freebase.com/view/en/kantele +p2283 +sg6 +S'A kantele (pronounced [\xcb\x88k\xc9\x91ntele] in Finnish) or kannel ([\xcb\x88k\xc9\x91n\xcb\x90el] in Estonian) is a traditional plucked string instrument of the zither family native to Finland, Estonia, and Karelia. It is related to the Russian gusli, the Latvian kokle and the Lithuanian kankl\xc4\x97s. Together these instruments make up the family known as Baltic psalteries.\nThe oldest forms of kantele have 5 or 6 horsehair strings and a wooden body carved from one piece; more modern instruments have metal strings and often a...' +p2284 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0291bv5 +p2285 +sg10 +VKantele +p2286 +sg12 +Vhttp://indextank.com/_static/common/demo/0291bv5.jpg +p2287 +ssg14 +(dp2288 +I0 +I0 +ssg16 +g2286 +sg17 +(dp2289 +g19 +VPlucked string instrument +p2290 +ssa(dp2291 +g2 +(dp2292 +g4 +Vhttp://freebase.com/view/en/bongo_drum +p2293 +sg6 +S'Bongo or bongos are a Cuban percussion instrument consisting of a pair of single-headed, open-ended drums attached to each other. The drums are of different size: the larger drum is called in Spanish the hembra (female) and the smaller the macho (male). It is most often played by hand and is especially associated in Cuban music with a steady pattern or ostinato of eighth-notes known as the martillo or "hammer". They are membranophones, or instruments that create sound by a vibration against...' +p2294 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029sm8s +p2295 +sg10 +VBongo drum +p2296 +sg12 +Vhttp://indextank.com/_static/common/demo/029sm8s.jpg +p2297 +ssg14 +(dp2298 +I0 +I13 +ssg16 +g2296 +sg17 +(dp2299 +g19 +VPercussion +p2300 +ssa(dp2301 +g2 +(dp2302 +g4 +Vhttp://freebase.com/view/en/fairground_organ +p2303 +sg6 +S'A fairground organ is a pipe organ designed for use in a commercial public fairground setting to provide loud music to accompany fairground rides and attractions. Unlike organs intended for indoor use, they are designed to produce a large volume of sound to be heard over and above the noise of crowds of people and fairground machinery.\nFrom the development of street entertainment and fairs, musicians and entertainers had both mixed and creatively bounced ideas off one other. As the industry...' +p2304 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f0g8z +p2305 +sg10 +VFairground organ +p2306 +sg12 +Vhttp://indextank.com/_static/common/demo/02f0g8z.jpg +p2307 +ssg14 +(dp2308 +I0 +I0 +ssg16 +g2306 +sg17 +(dp2309 +g19 +VMechanical organ +p2310 +ssa(dp2311 +g2 +(dp2312 +g4 +Vhttp://freebase.com/view/m/074z58 +p2313 +sg6 +S'The regal was a small portable organ, furnished with beating reeds and having two bellows. The instrument enjoyed its greatest popularity during the Renaissance. The name was also sometimes given to the reed stops of a pipe organ, and more especially the vox humana stop.\nThe sound of the regal was produced by brass reeds held in resonators. The length of the vibrating portion of the reed determined its pitch and was regulated by means of a wire passing through the socket, the other end...' +p2314 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/07bjnmm +p2315 +sg10 +VRegal +p2316 +sg12 +Vhttp://indextank.com/_static/common/demo/07bjnmm.jpg +p2317 +ssg14 +(dp2318 +I0 +I0 +ssg16 +g2316 +sg17 +(dp2319 +g19 +VOrgan +p2320 +ssa(dp2321 +g2 +(dp2322 +g4 +Vhttp://freebase.com/view/en/recorder +p2323 +sg6 +S'The recorder or English flute is a woodwind musical instrument of the family known as fipple flutes or internal duct flutes\xe2\x80\x94whistle-like instruments which include the tin whistle and ocarina. The recorder is end-blown and the mouth of the instrument is constricted by a wooden plug, known as a block or fipple. It is distinguished from other members of the family by having holes for seven fingers (the lower one or two often doubled to facilitate the production of semitones) and one for the...' +p2324 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0291nst +p2325 +sg10 +VRecorder +p2326 +sg12 +Vhttp://indextank.com/_static/common/demo/0291nst.jpg +p2327 +ssg14 +(dp2328 +I0 +I75 +ssg16 +g2326 +sg17 +(dp2329 +g19 +VWoodwind instrument +p2330 +ssa(dp2331 +g2 +(dp2332 +g4 +Vhttp://freebase.com/view/en/hammered_dulcimer +p2333 +sg6 +S"The hammered dulcimer is a stringed musical instrument with the strings stretched over a trapezoidal sounding board. Typically, the hammered dulcimer is set on a stand, at an angle, before the musician, who holds small mallet hammers in each hand to strike the strings (cf. Appalachian dulcimer). The Graeco-Roman dulcimer (sweet song), derives from the Latin dulcis (sweet) and the Greek melos (song). The dulcimer's origin is uncertain, but tradition holds it was invented in Persia (Iran), as..." +p2334 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bfht8 +p2335 +sg10 +VHammered dulcimer +p2336 +sg12 +Vhttp://indextank.com/_static/common/demo/02bfht8.jpg +p2337 +ssg14 +(dp2338 +I0 +I44 +ssg16 +g2336 +sg17 +(dp2339 +g19 +VStruck string instruments +p2340 +ssa(dp2341 +g2 +(dp2342 +g4 +Vhttp://freebase.com/view/en/oval_spinet +p2343 +sg6 +S'The oval spinet is a type of harpsichord invented in the late 17th century by Bartolomeo Cristofori, the Italian instrument maker who later achieved fame for inventing the piano. The oval spinet was unusual for its shape, the arrangement of its strings, and for its mechanism for changing registration.\nThe two oval spinets built by Cristofori survive today. One, built in 1690, is kept in the Museo degli strumenti musicali, part of the Galleria del Accademia in Florence. The other, from 1693,...' +p2344 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041ng_b +p2345 +sg10 +VOval spinet +p2346 +sg12 +Vhttp://indextank.com/_static/common/demo/041ng_b.jpg +p2347 +ssg14 +(dp2348 +I0 +I0 +ssg16 +g2346 +sg17 +(dp2349 +g19 +VHarpsichord +p2350 +ssa(dp2351 +g2 +(dp2352 +g4 +Vhttp://freebase.com/view/m/03cljxs +p2353 +sg6 +S'The helicon is a brass musical instrument in the tuba family. Most are BB\xe2\x99\xad basses, but they also commonly exist in EE\xe2\x99\xad, F, and tenor sizes, as well as other types to a lesser extent.\nThe sousaphone is a specialized version of helicon, differing primarily in two ways: a Sousaphone bell is shaped to face forwards and has a larger flare, a Sousaphone has a "goose-neck" leadpipe which offers greater adjustability of mouthpiece position at the expense of tone quality, while both instruments have...' +p2354 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d7_c9 +p2355 +sg10 +VHelicon +p2356 +sg12 +Vhttp://indextank.com/_static/common/demo/02d7_c9.jpg +p2357 +ssg14 +(dp2358 +I0 +I0 +ssg16 +g2356 +sg17 +(dp2359 +g19 +VBrass instrument +p2360 +ssa(dp2361 +g2 +(dp2362 +g4 +Vhttp://freebase.com/view/en/english_concertina +p2363 +sg6 +S'The English concertina is a free-reed instrument invented by Charles Wheatstone in 1829. It is fully chromatic, and plays the same tones whether contracting or expanding the bellows.\nKeys are arranged in four horizontal rows on each side. The natural notes are in the middle two rows; low notes are closest to the player, and alternate between the left and right side as they ascend (e.g., C may be on the left, then D on the right). Any two notes next to each other in the middle two rows form a...' +p2364 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0292mbm +p2365 +sg10 +VEnglish concertina +p2366 +sg12 +Vhttp://indextank.com/_static/common/demo/0292mbm.jpg +p2367 +ssg14 +(dp2368 +I0 +I5 +ssg16 +g2366 +sg17 +(dp2369 +g19 +VConcertina +p2370 +ssa(dp2371 +g2 +(dp2372 +g4 +Vhttp://freebase.com/view/en/chamberlin +p2373 +sg6 +S"The Chamberlin is an electro-mechanical keyboard instrument that was a precursor to the Mellotron. It was developed and patented by Iowa, Wisconsin inventor Harry Chamberlin from 1949 to 1956, when the first model was introduced. Various models and versions of these Chamberlin music instruments exist. While most are keyboard-based instruments, there were also early drum machines produced and sold. Some of these drums patterns feature Harry Chamberlin's son Richard on them.\nHarry Chamberlin's..." +p2374 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02b965p +p2375 +sg10 +VChamberlin +p2376 +sg12 +Vhttp://indextank.com/_static/common/demo/02b965p.jpg +p2377 +ssg14 +(dp2378 +I0 +I0 +ssg16 +g2376 +sg17 +(dp2379 +g19 +VElectric piano +p2380 +ssa(dp2381 +g2 +(dp2382 +g4 +Vhttp://freebase.com/view/en/bass_guitar +p2383 +sg6 +S'The bass guitar (also called electric bass, or simply bass; /\xcb\x88be\xc9\xaas/) is a stringed instrument played primarily with the fingers or thumb (by plucking, slapping, popping, tapping, or thumping), or by using a pick.\nThe bass guitar is similar in appearance and construction to an electric guitar, but with a longer neck and scale length, and four, five, or six strings. The four-string bass\xe2\x80\x94by far the most common\xe2\x80\x94is usually tuned the same as the double bass, which corresponds to pitches one...' +p2384 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bc8zj +p2385 +sg10 +VBass guitar +p2386 +sg12 +Vhttp://indextank.com/_static/common/demo/02bc8zj.jpg +p2387 +ssg14 +(dp2388 +I0 +I2274 +ssg16 +g2386 +sg17 +(dp2389 +g19 +VGuitar +p2390 +ssa(dp2391 +g2 +(dp2392 +g4 +Vhttp://freebase.com/view/en/saxophone +p2393 +sg6 +S'The saxophone (also referred to as the sax) is a conical-bore transposing musical instrument that is a member of the woodwind family. Saxophones are usually made of brass and played with a single-reed mouthpiece similar to that of the clarinet. The saxophone was invented by the Belgian Adolphe Sax in 1841. He wanted to create an instrument that would both be the most powerful and vocal of the woodwinds and the most adaptive of the brass, which would fill the then vacant middle ground between...' +p2394 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05kh825 +p2395 +sg10 +VSaxophone +p2396 +sg12 +Vhttp://indextank.com/_static/common/demo/05kh825.jpg +p2397 +ssg14 +(dp2398 +I0 +I814 +ssg16 +g2396 +sg17 +(dp2399 +g19 +VWoodwind instrument +p2400 +ssa(dp2401 +g2 +(dp2402 +g4 +Vhttp://freebase.com/view/en/pipe_and_tabor +p2403 +sg6 +S"Pipe and tabor is a pair of instruments played by a single player, consisting of a three-hole pipe played with one hand, and a small drum played with the other. The tabor (drum) hangs on the performer's left arm or around the neck, leaving the hands free to beat the drum with a stick in the right hand and play the pipe with thumb and first two fingers of the left hand.\nThe pipe is made out of wood, metal or plastic and consists of a cylindrical tube of narrow bore (1:40 diameter:length..." +p2404 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02g57g5 +p2405 +sg10 +VPipe and Tabor +p2406 +sg12 +Vhttp://indextank.com/_static/common/demo/02g57g5.jpg +p2407 +ssg14 +(dp2408 +I0 +I0 +ssg16 +g2406 +sg17 +(dp2409 +g19 +VFlute (transverse) +p2410 +ssa(dp2411 +g2 +(dp2412 +g4 +Vhttp://freebase.com/view/en/melodica +p2413 +sg6 +S'The melodica, also known as the "blow-organ" or "key-flute", is a free-reed instrument similar to the melodeon and harmonica. It has a musical keyboard on top, and is played by blowing air through a mouthpiece that fits into a hole in the side of the instrument. Pressing a key opens a hole, allowing air to flow through a reed. The keyboard is usually two or three octaves long. Melodicas are small, light, and portable. They are popular in music education, especially in Asia.\nThe modern form...' +p2414 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bpzhj +p2415 +sg10 +VMelodica +p2416 +sg12 +Vhttp://indextank.com/_static/common/demo/02bpzhj.jpg +p2417 +ssg14 +(dp2418 +I0 +I50 +ssg16 +g2416 +sg17 +(dp2419 +g19 +VWoodwind instrument +p2420 +ssa(dp2421 +g2 +(dp2422 +g4 +Vhttp://freebase.com/view/m/0fwc52 +p2423 +sg6 +S'The triple harp, or more often referred to as the Welsh Triple Harp, (Welsh: Telyn deires) is a type of harp employing three rows of strings instead of the common single row. The Welsh triple harp today is found mainly among players of traditional Welsh folk music.\nThe triple harp first originated in Italy, under the form of two rows of strings and later three, as the baroque harp (Italian: Arpa Doppia). It appeared in the British Isles early in the 17th century. In 1629, the French harpist...' +p2424 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0bcj7yk +p2425 +sg10 +VTriple Harp +p2426 +sg12 +Vhttp://indextank.com/_static/common/demo/0bcj7yk.jpg +p2427 +ssg14 +(dp2428 +I0 +I0 +ssg16 +g2426 +sg17 +(dp2429 +g19 +VHarp +p2430 +ssa(dp2431 +g2 +(dp2432 +g4 +Vhttp://freebase.com/view/en/glockenspiel +p2433 +sg6 +S"A glockenspiel (German pronunciation:\xc2\xa0[\xcb\x88\xc9\xa1l\xc9\x94k\xc9\x99n\xcb\x8c\xca\x83pi\xcb\x90l]) is a percussion instrument composed of a set of tuned keys arranged in the fashion of the keyboard of a piano. In this way, it is similar to the xylophone; however, the xylophone's bars are made of wood, while the glockenspiel's are metal plates or tubes, thus making it a metallophone. The glockenspiel, moreover, is usually smaller and higher in pitch.\nIn German, a carillon is also called a Glockenspiel.\nWhen used in a marching or..." +p2434 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bnygk +p2435 +sg10 +VGlockenspiel +p2436 +sg12 +Vhttp://indextank.com/_static/common/demo/02bnygk.jpg +p2437 +ssg14 +(dp2438 +I0 +I15 +ssg16 +g2436 +sg17 +(dp2439 +g19 +VTuned percussion +p2440 +ssa(dp2441 +g2 +(dp2442 +g4 +Vhttp://freebase.com/view/m/02f5t8 +p2443 +sg6 +S'A fife is a small, high-pitched, transverse flute that is similar to the piccolo, but louder and shriller due to its narrower bore. The fife originated in medieval Europe and is often used in military and marching bands. Someone who plays the fife is called a fifer. The word fife comes from the German Pfeife, or pipe, ultimately derived from the Latin word pipare.\nThe fife is a simple instrument usually consisting of a tube with 6 finger holes, and diatonically tuned. Some have 10 or 11...' +p2444 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cw2vb +p2445 +sg10 +VFife +p2446 +sg12 +Vhttp://indextank.com/_static/common/demo/02cw2vb.jpg +p2447 +ssg14 +(dp2448 +I0 +I1 +ssg16 +g2446 +sg17 +(dp2449 +g19 +VFlute (transverse) +p2450 +ssa(dp2451 +g2 +(dp2452 +g4 +Vhttp://freebase.com/view/en/cimpoi +p2453 +sg6 +S'Cimpoi, the Romanian bagpipe, has a single drone and straight bore chanter and is less strident than its Balkan relatives.\nThe number of finger holes varies from five to eight and there are two types of cimpoi with a double chanter. The bag is often covered with embroidered cloth. The bagpipe can be found in most of Romania apart from the central, northern and eastern parts of Transylvania, but at present (the early 21st century) is only played by a few elderly people.\nA well-known player of...' +p2454 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041xg96 +p2455 +sg10 +VCimpoi +p2456 +sg12 +Vhttp://indextank.com/_static/common/demo/041xg96.jpg +p2457 +ssg14 +(dp2458 +I0 +I0 +ssg16 +g2456 +sg17 +(dp2459 +g19 +VBagpipes +p2460 +ssa(dp2461 +g2 +(dp2462 +g4 +Vhttp://freebase.com/view/en/kulintang_a_kayo +p2463 +sg6 +S'The kulintang a kayo (literally, \xe2\x80\x9cwooden kulintang\xe2\x80\x9d) is a Philippine xylophone of the Maguindanaon people with eight tuned slabs arranged horizontally atop a wooden antangan (rack). Made of soft wood such as bayug, the kulintang a kayo is a common found among Maguindanaon households with a musical background. Traditionally, it was used for self-entertainment purpose inside the house, so beginners could practice kulintang pieces before performing them on the real kulintang and only recently...' +p2464 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041rgl8 +p2465 +sg10 +VKulintang a Kayo +p2466 +sg12 +Vhttp://indextank.com/_static/common/demo/041rgl8.jpg +p2467 +ssg14 +(dp2468 +I0 +I0 +ssg16 +g2466 +sg17 +(dp2469 +g19 +VXylophone +p2470 +ssa(dp2471 +g2 +(dp2472 +g4 +Vhttp://freebase.com/view/en/ranat_ek +p2473 +sg6 +S'The ranat ek (Thai: \xe0\xb8\xa3\xe0\xb8\xb0\xe0\xb8\x99\xe0\xb8\xb2\xe0\xb8\x94\xe0\xb9\x80\xe0\xb8\xad\xe0\xb8\x81, pronounced\xc2\xa0[ran\xc3\xa2\xcb\x90t \xca\x94\xc3\xa8\xcb\x90k], "alto xylophone") is a Thai xylophone. It has 21 or 22 wooden bars suspended by cords over a boat-shaped trough resonator, and is played with two mallets. It is used as a leading instrument in the piphat ensemble.\nThe ranat ek is played by two types of mallets. The hard mallets create the sharp bright sound when they keys are hit.The hard mallets are used for more faster playing. The soft mallets create a mellow and more softer tone...' +p2474 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05t3hzy +p2475 +sg10 +VRanat ek +p2476 +sg12 +Vhttp://indextank.com/_static/common/demo/05t3hzy.jpg +p2477 +ssg14 +(dp2478 +I0 +I0 +ssg16 +g2476 +sg17 +(dp2479 +g19 +VXylophone +p2480 +ssa(dp2481 +g2 +(dp2482 +g4 +Vhttp://freebase.com/view/en/mandola +p2483 +sg6 +S'The mandola (US and Canada) or tenor mandola (Ireland, and UK) is a fretted, stringed musical instrument. It is to the mandolin what the viola is to the violin: the four double courses of strings tuned in fifths to the same pitches as the viola (C-G-D-A low-to-high), a fifth lower than a mandolin. However, the mandola, though now rarer, is the ancestor of the mandolin, the name of which means simply "little mandola".\nThe name mandola may originate with the ancient pandura, and was also...' +p2484 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f0bjz +p2485 +sg10 +VMandola +p2486 +sg12 +Vhttp://indextank.com/_static/common/demo/02f0bjz.jpg +p2487 +ssg14 +(dp2488 +I0 +I6 +ssg16 +g2486 +sg17 +(dp2489 +g19 +VMandolin +p2490 +ssa(dp2491 +g2 +(dp2492 +g4 +Vhttp://freebase.com/view/en/clarinet +p2493 +sg6 +S'The clarinet is a musical instrument of woodwind type. The name derives from adding the suffix -et (meaning little) to the Italian word clarino (meaning a type of trumpet), as the first clarinets had a strident tone similar to that of a trumpet. The instrument has an approximately cylindrical bore, and uses a single reed. In jazz contexts, it has sometimes been informally referred to as the "licorice stick."\nClarinets comprise a family of instruments of differing sizes and pitches. The...' +p2494 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bctx9 +p2495 +sg10 +VClarinet +p2496 +sg12 +Vhttp://indextank.com/_static/common/demo/02bctx9.jpg +p2497 +ssg14 +(dp2498 +I0 +I291 +ssg16 +g2496 +sg17 +(dp2499 +g19 +VWoodwind instrument +p2500 +ssa(dp2501 +g2 +(dp2502 +g4 +Vhttp://freebase.com/view/en/gayageum +p2503 +sg6 +S'The gayageum or kayagum is a traditional Korean zither-like string instrument, with 12 strings, although more recently variants have been constructed with 21 or other numbers of strings. It is probably the best known traditional Korean musical instrument. It is in the zither family. It is very similar to the Chinese guzheng, or table harp, from which it is derived.\nAccording to the Samguksagi (1145), a history of the Three Kingdoms of Korea, the gayageum is supposed to have been developed...' +p2504 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cp5bd +p2505 +sg10 +VGayageum +p2506 +sg12 +Vhttp://indextank.com/_static/common/demo/02cp5bd.jpg +p2507 +ssg14 +(dp2508 +I0 +I0 +ssg16 +g2506 +sg17 +(dp2509 +g19 +VPlucked string instrument +p2510 +ssa(dp2511 +g2 +(dp2512 +g4 +Vhttp://freebase.com/view/en/mohan_veena +p2513 +sg6 +S'The Mohan veena (or Vishwa veena) is a stringed musical instrument used in Indian classical music. It derives its name from its inventor Pandit Vishwa Mohan Bhatt\nThe instrument is actually a modified Archtop guitar and consists of 20 strings viz. three melody strings, five drone strings strung to the peghead, and twelve sympathetic strings strung to the tuners mounted on the side of the neck. A gourd (or the tumba) is screwed into the back of the neck for improved sound quality and...' +p2514 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cnzd7 +p2515 +sg10 +VMohan veena +p2516 +sg12 +Vhttp://indextank.com/_static/common/demo/02cnzd7.jpg +p2517 +ssg14 +(dp2518 +I0 +I2 +ssg16 +g2516 +sg17 +(dp2519 +g19 +VPlucked string instrument +p2520 +ssa(dp2521 +g2 +(dp2522 +g4 +Vhttp://freebase.com/view/en/cornet +p2523 +sg6 +S"The cornet is a brass instrument very similar to the trumpet, distinguished by its conical bore, compact shape, and mellower tone quality. The most common cornet is a transposing instrument in B\xe2\x99\xad. It is not related to the renaissance and early baroque cornett or cornetto.\nThe cornet was originally derived from the post horn around 1820 in France. Among the first manufacturers of modern cornets were Parisian Jean Aste' in 1928. Cornets first appear as separate instrumental parts in 19th..." +p2524 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02910xc +p2525 +sg10 +VCornet +p2526 +sg12 +Vhttp://indextank.com/_static/common/demo/02910xc.jpg +p2527 +ssg14 +(dp2528 +I0 +I64 +ssg16 +g2526 +sg17 +(dp2529 +g19 +VBrass instrument +p2530 +ssa(dp2531 +g2 +(dp2532 +g4 +Vhttp://freebase.com/view/en/semi-acoustic_guitar +p2533 +sg6 +S'A semi-acoustic guitar or hollow-body electric is a type of electric guitar with both a sound box and one or more electric pickups. This is not the same as an electric acoustic guitar, which is an acoustic guitar with the addition of pickups or other means of amplification, either added by the manufacturer or the player.\nOther semi-acoustic or acoustic electric instruments include basses and mandolins. These are similarly constructed to semi-acoustic guitars, and are used in the same ways...' +p2534 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044njy7 +p2535 +sg10 +VSemi-acoustic guitar +p2536 +sg12 +Vhttp://indextank.com/_static/common/demo/044njy7.jpg +p2537 +ssg14 +(dp2538 +I0 +I0 +ssg16 +g2536 +sg17 +(dp2539 +g19 +VPlucked string instrument +p2540 +ssa(dp2541 +g2 +(dp2542 +g4 +Vhttp://freebase.com/view/en/daxophone +p2543 +sg6 +S'The daxophone, invented by Hans Reichel, is a experimental musical instrument of the friction idiophones category. It consists of a thin wooden blade fixed in a wooden block (often attached to a tripod), which holds one or more contact microphones. Normally, it is played by bowing the free end, but it can also be struck or plucked, which propagates sound in the same way a ruler halfway off a table does. These vibrations then continue to the wooden-block base, which in turn is amplified by...' +p2544 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/07876hm +p2545 +sg10 +VDaxophone +p2546 +sg12 +Vhttp://indextank.com/_static/common/demo/07876hm.jpg +p2547 +ssg14 +(dp2548 +I0 +I0 +ssg16 +g2546 +sg17 +(dp2549 +g19 +VIdiophone +p2550 +ssa(dp2551 +g2 +(dp2552 +g4 +Vhttp://freebase.com/view/en/cor_anglais +p2553 +sg6 +S'The cor anglais (British English and French), or English horn (American English), is a double-reed woodwind instrument in the oboe family.\nThe cor anglais is a transposing instrument pitched in F, a perfect fifth lower than the oboe (a C instrument), and is consequently approximately one and a half times the length of the oboe. The fingering and playing technique used for the cor anglais are essentially the same as those of the oboe. Music for the cor anglais is thus written a perfect fifth...' +p2554 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02btjlp +p2555 +sg10 +VCor anglais +p2556 +sg12 +Vhttp://indextank.com/_static/common/demo/02btjlp.jpg +p2557 +ssg14 +(dp2558 +I0 +I4 +ssg16 +g2556 +sg17 +(dp2559 +g19 +VWoodwind instrument +p2560 +ssa(dp2561 +g2 +(dp2562 +g4 +Vhttp://freebase.com/view/en/chemnitzer_concertina +p2563 +sg6 +S'A Chemnitzer concertina is a musical instrument of the hand-held bellows-driven free-reed category, sometimes called squeezeboxes. The Chemnitzer concertina is most closely related to the Bandone\xc3\xb3n (German spelling: Bandonion), more distantly to the other concertinas, and accordions.\nIt is roughly square in cross-section, with the keyboards consisting of cylindrical buttons on each end arranged in curving rows. Like other concertinas, the buttons travel in a direction approximately parallel...' +p2564 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bldk5 +p2565 +sg10 +VChemnitzer concertina +p2566 +sg12 +Vhttp://indextank.com/_static/common/demo/02bldk5.jpg +p2567 +ssg14 +(dp2568 +I0 +I0 +ssg16 +g2566 +sg17 +(dp2569 +g19 +VConcertina +p2570 +ssa(dp2571 +g2 +(dp2572 +g4 +Vhttp://freebase.com/view/en/ghaychak +p2573 +sg6 +S'The Ghaychak or Ghijak is a round-bodied musical instrument with 3 or 4 metal strings and a short fretless neck. It is used by Iranians, Afghans, Uzbeks, Uyghurs, Tajiks, Turkmens and Qaraqalpaks.\nThe ghidjak, or violin, is the only bow instrument found in the Pamirs. The ghidjak is very popular throughout Central Asia. Its sound box is metal or wooden, and it has three or four metal strings and a neck made of willow, apricot or mulberry wood. It is tuned in intervals of fourths.\nThe...' +p2574 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02gfwwq +p2575 +sg10 +VGhaychak +p2576 +sg12 +Vhttp://indextank.com/_static/common/demo/02gfwwq.jpg +p2577 +ssg14 +(dp2578 +I0 +I1 +ssg16 +g2576 +sg17 +(dp2579 +g19 +VBowed string instruments +p2580 +ssa(dp2581 +g2 +(dp2582 +g4 +Vhttp://freebase.com/view/en/contrabass_flute +p2583 +sg6 +S'The contrabass flute is one of the rarer members of the flute family. It is used mostly in flute ensembles. Its range is similar to that of the regular concert flute, except that it is pitched two octaves lower; the lowest performable note is two octaves below middle C (the lowest C on the cello). Many contrabass flutes in C are also equipped with a low B, (in the same manner as many modern standard sized flutes are.) Contrabass flutes are only available from select flute makers.\nSometimes...' +p2584 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cffny +p2585 +sg10 +VContrabass flute +p2586 +sg12 +Vhttp://indextank.com/_static/common/demo/02cffny.jpg +p2587 +ssg14 +(dp2588 +I0 +I0 +ssg16 +g2586 +sg17 +(dp2589 +g19 +VFlute (transverse) +p2590 +ssa(dp2591 +g2 +(dp2592 +g4 +Vhttp://freebase.com/view/en/bass_drum +p2593 +sg6 +S'The bass drums are of variable sizes and are used in several musical genres (see usage below). Three major types of bass drums can be distinguished. The type usually seen or heard in orchestral, ensemble or concert band music is the orchestral, or concert bass drum (in Italian: gran cassa, gran tamburo). It is the largest drum of the orchestra. The kick drum, struck with a beater attached to a pedal, is usually seen on drum kits. The third type, the pitched bass drum, is generally used in...' +p2594 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bdbkf +p2595 +sg10 +VBass drum +p2596 +sg12 +Vhttp://indextank.com/_static/common/demo/02bdbkf.jpg +p2597 +ssg14 +(dp2598 +I0 +I6 +ssg16 +g2596 +sg17 +(dp2599 +g19 +VPercussion +p2600 +ssa(dp2601 +g2 +(dp2602 +g4 +Vhttp://freebase.com/view/en/surbahar +p2603 +sg6 +S"Surbahar (Urdu: \xd8\xb3\xd8\xb1\xd8\xa8\xdb\x81\xd8\xa7\xd8\xb1; Hindi: \xe0\xa4\xb8\xe0\xa5\x81\xe0\xa4\xb0 \xe0\xa4\xac\xe0\xa4\xb9\xe0\xa4\xbe\xe0\xa4\xb0), sometimes known as bass sitar, is a plucked string instrument used in the Hindustani classical music of North India. It is closely related to sitar, but it has a lower tone. Depending on the instrument's size, it is usually pitched two to five whole steps below the standard sitar, but Indian classical music having no concept of absolute pitch, this may vary. The instrument can emit frequencies lower than 20\xc2\xa0Hz.\nSurbahar is over 130\xc2\xa0cm (51\xc2\xa0inches). It..." +p2604 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d411z +p2605 +sg10 +VSurbahar +p2606 +sg12 +Vhttp://indextank.com/_static/common/demo/02d411z.jpg +p2607 +ssg14 +(dp2608 +I0 +I4 +ssg16 +g2606 +sg17 +(dp2609 +g19 +VPlucked string instrument +p2610 +ssa(dp2611 +g2 +(dp2612 +g4 +Vhttp://freebase.com/view/en/crwth +p2613 +sg6 +S'The crwth is an archaic stringed musical instrument, associated particularly with Welsh music, once widely-played in Europe.\nCrwth is originally a Welsh word, in English pronounced /\xcb\x88kru\xcb\x90\xce\xb8/ (rhyming with "truth") or /\xcb\x88kr\xca\x8a\xce\xb8/ (with the vowel of "push"). The traditional English name is crowd (or rote), and the variants crwd, crout and crouth are little used today. In Medieval Latin it is called the chorus or crotta. The Welsh word crythor means a performer on the crwth. The Irish word is cruit,...' +p2614 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029gk01 +p2615 +sg10 +VCrwth +p2616 +sg12 +Vhttp://indextank.com/_static/common/demo/029gk01.jpg +p2617 +ssg14 +(dp2618 +I0 +I0 +ssg16 +g2616 +sg17 +(dp2619 +g19 +VBowed string instruments +p2620 +ssa(dp2621 +g2 +(dp2622 +g4 +Vhttp://freebase.com/view/en/cymbalum +p2623 +sg6 +S'The cimbalom is a concert hammered dulcimer: a type of chordophone composed of a large, trapezoidal box with metal strings stretched across its top. It is a musical instrument commonly found throughout the group of East European nations and cultures which composed Austria-Hungary (1867\xe2\x80\x931918), namely contemporary Belarus, Hungary, Romania, Moldova, Ukraine, Poland, the Czech Republic and Slovakia. The cimbalom is (typically) played by striking two beaters against the strings. The steel treble...' +p2624 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029sh3x +p2625 +sg10 +VCymbalum +p2626 +sg12 +Vhttp://indextank.com/_static/common/demo/029sh3x.jpg +p2627 +ssg14 +(dp2628 +I0 +I0 +ssg16 +g2626 +sg17 +(dp2629 +g19 +VStruck string instruments +p2630 +ssa(dp2631 +g2 +(dp2632 +g4 +Vhttp://freebase.com/view/en/kutiyapi +p2633 +sg6 +S'The kutiyapi, is a Philippine two-stringed, fretted boat-lute. It is the only stringed instrument among the Maguindanao people, and one of several among other groups such as the Maranao and Manobo. It is four to six feet long with nine frets made of hardened beeswax. The instrument is carved out of solid soft wood such as from the jackfruit tree.\nCommon to all kutiyapi instruments, a constant drone is played with one string while the other, an octave above the drone, plays the melody with a...' +p2634 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/042m46_ +p2635 +sg10 +VKutiyapi +p2636 +sg12 +Vhttp://indextank.com/_static/common/demo/042m46_.jpg +p2637 +ssg14 +(dp2638 +I0 +I0 +ssg16 +g2636 +sg17 +(dp2639 +g19 +VPlucked string instrument +p2640 +ssa(dp2641 +g2 +(dp2642 +g4 +Vhttp://freebase.com/view/en/sac_de_gemecs +p2643 +sg6 +S'The sac de gemecs (Catalan: literally "bag of moans", alternately buna in Andorra or coixinera, gaita or botella) is a type of bagpipe found in Catalonia (eastern Spain spilling over into southern France).\nThe instrument consists of a chanter, a mouthblown blowpipe, and three drones. The lowest drone (bord\xc3\xb3 llarg) plays a note two octaves below the tonic of the chanter. The middle drone (bord\xc3\xb3 mitj\xc3\xa0 ) plays a fifth above the bass. The high drone (bord\xc3\xb3 petit) plays an octave below the...' +p2644 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0635qpl +p2645 +sg10 +VSac de gemecs +p2646 +sg12 +Vhttp://indextank.com/_static/common/demo/0635qpl.jpg +p2647 +ssg14 +(dp2648 +I0 +I0 +ssg16 +g2646 +sg17 +(dp2649 +g19 +VBagpipes +p2650 +ssa(dp2651 +g2 +(dp2652 +g4 +Vhttp://freebase.com/view/en/octobass +p2653 +sg6 +S'The octobass is an extremely large bowed string instrument constructed about 1850 in Paris by the French luthier Jean Baptiste Vuillaume (1798-1875). It has three strings and is essentially a larger version of the double bass (the specimen in the collection of the Mus\xc3\xa9e de la Musique in Paris measures 3.48 meters in length, whereas a full size double bass is generally approximately 2 meters in length). Because of the impractically large size of its fingerboard and thickness of its strings,...' +p2654 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cmycm +p2655 +sg10 +VOctobass +p2656 +sg12 +Vhttp://indextank.com/_static/common/demo/02cmycm.jpg +p2657 +ssg14 +(dp2658 +I0 +I0 +ssg16 +g2656 +sg17 +(dp2659 +g19 +VBowed string instruments +p2660 +ssa(dp2661 +g2 +(dp2662 +g4 +Vhttp://freebase.com/view/en/dulcian +p2663 +sg6 +S'The dulcian is a Renaissance bass woodwind instrument, with a double reed and a folded conical bore. Equivalent terms include "curtal" in English, "dulzian" in German, "baj\xc3\xb3n" in Spanish, "dou\xc3\xa7aine"\' in French, "dulciaan" in Dutch, and "dulciana" in Italian.\nThe predecessor of the modern bassoon, it flourished between 1550 and 1700, but was probably invented earlier. Towards the end of this period it co-existed with, and was then superseded by the baroque bassoon, although it continued to be...' +p2664 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03r1glg +p2665 +sg10 +VDulcian +p2666 +sg12 +Vhttp://indextank.com/_static/common/demo/03r1glg.jpg +p2667 +ssg14 +(dp2668 +I0 +I0 +ssg16 +g2666 +sg17 +(dp2669 +g19 +VBassoon +p2670 +ssa(dp2671 +g2 +(dp2672 +g4 +Vhttp://freebase.com/view/en/huemmelchen +p2673 +sg6 +S'The h\xc3\xbcmmelchen is a type of small German bagpipe, attested in Syntagma Musicum by Michael Praetorius during the Renaissance.\nEarly versions are believed to have double-reeded chanters, most likely with single-reeded drones.\nThe word "h\xc3\xbcmmelchen" probably comes from the Low German word h\xc3\xa4meln meaning "trim". This may refer to the h\xc3\xbcmmelchen\'s small size, resembling a trimmed-down version of a larger bagpipe. Another possibly etymology comes from the word hummel ("bumble-bee"), referring to...' +p2674 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/07blxb9 +p2675 +sg10 +VHuemmelchen +p2676 +sg12 +Vhttp://indextank.com/_static/common/demo/07blxb9.jpg +p2677 +ssg14 +(dp2678 +I0 +I0 +ssg16 +g2676 +sg17 +(dp2679 +g19 +VBagpipes +p2680 +ssa(dp2681 +g2 +(dp2682 +g4 +Vhttp://freebase.com/view/en/hi-hat +p2683 +sg6 +S'A hi-hat, or hihat, is a type of cymbal and stand used as a typical part of a drum kit by percussionists in R&B, hip-hop, disco, jazz, rock and roll, house, reggae and other forms of contemporary popular music.\nThe hi-hat consists of two cymbals that are mounted on a stand, one on top of the other, and clashed together using a pedal on the stand. A narrow metal shaft or rod runs through both cymbals into a hollow tube and connects to the pedal. The top cymbal is connected to the rod with a...' +p2684 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bdbkf +p2685 +sg10 +VHi-hat +p2686 +sg12 +Vhttp://indextank.com/_static/common/demo/02bdbkf.jpg +p2687 +ssg14 +(dp2688 +I0 +I1 +ssg16 +g2686 +sg17 +(dp2689 +g19 +VCymbal +p2690 +ssa(dp2691 +g2 +(dp2692 +g4 +Vhttp://freebase.com/view/en/sihu +p2693 +sg6 +S'The sihu (\xe5\x9b\x9b\xe8\x83\xa1; pinyin: s\xc3\xach\xc3\xba) is a Chinese bowed string instrument with four strings. It is a member of the huqin family of instruments.\nThe instrument\'s name comes from the words s\xc3\xac (\xe5\x9b\x9b, meaning "four" in Chinese, referring to the instrument\'s number of strings) and h\xc3\xba (\xe8\x83\xa1, short for huqin, the family of instruments of which the sihu is a member). Its soundbox and neck are made from hardwood and the playing end of the soundbox is covered with python, cow, or sheep skin.\nThere are several sizes...' +p2694 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04pr37b +p2695 +sg10 +VSihu +p2696 +sg12 +Vhttp://indextank.com/_static/common/demo/04pr37b.jpg +p2697 +ssg14 +(dp2698 +I0 +I0 +ssg16 +g2696 +sg17 +(dp2699 +g19 +VBowed string instruments +p2700 +ssa(dp2701 +g2 +(dp2702 +g4 +Vhttp://freebase.com/view/en/tricordia +p2703 +sg6 +S'A tricordia (also trichordia or tricordio) or mandriola is a twelve-stringed variation of the mandolin. The tricordia is used in Mexican folk music, while its European cousin, the mandriola, is used identically to the mandolin. It differs from a standard mandolin in that it has three strings per course. Tricordias only use unison tuning (ggg d\'d\'d\' a\'a\'a\' e"e"e"), while mandriolas use either unison tuning or octave tuning (Ggg dd\'d\' aa\'a\' e\'e"e").' +p2704 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04rymhg +p2705 +sg10 +VTricordia +p2706 +sg12 +Vhttp://indextank.com/_static/common/demo/04rymhg.jpg +p2707 +ssg14 +(dp2708 +I0 +I0 +ssg16 +g2706 +sg17 +(dp2709 +g19 +VPlucked string instrument +p2710 +ssa(dp2711 +g2 +(dp2712 +g4 +Vhttp://freebase.com/view/m/04gtkv6 +p2713 +sg6 +S'The word la\xc3\xbad is the Spanish word for lute. It is most commonly used to refer to a plectrum-plucked chordophone from Spain. It belongs to the cittern family of instruments. It has six double courses (i.e. twelve strings in pairs), similarly to the bandurria, but its neck is longer. Traditionally it is used folk string musical groups, together with the guitar and the bandurria.\nLike the bandurria, it is tuned in fourths, but its range is 1 octave lower.\nTuning:\nThere is also a Cuban variety...' +p2714 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05m1djr +p2715 +sg10 +VLaúd +p2716 +sg12 +Vhttp://indextank.com/_static/common/demo/05m1djr.jpg +p2717 +ssg14 +(dp2718 +I0 +I0 +ssg16 +g2716 +sg17 +(dp2719 +g19 +VPlucked string instrument +p2720 +ssa(dp2721 +g2 +(dp2722 +g4 +Vhttp://freebase.com/view/en/kobza +p2723 +sg6 +S'The kobza (Ukrainian: \xd0\xba\xd0\xbe\xd0\xb1\xd0\xb7\xd0\xb0) is a Ukrainian folk music instrument of the lute family (Hornbostel-Sachs classification number 321.321-5+6), a relative of the Central European mandora. The term kobza however, has also been applied to a number of other Eastern European instruments distinct from the Ukrainian kobza.\nThe Ukrainian kobza was traditionally gut-strung, with a body hewn from a single block of wood. Instruments with a staved assembly also exist. The kobza has a medium length neck...' +p2724 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bnwxh +p2725 +sg10 +VKobza +p2726 +sg12 +Vhttp://indextank.com/_static/common/demo/02bnwxh.jpg +p2727 +ssg14 +(dp2728 +I0 +I0 +ssg16 +g2726 +sg17 +(dp2729 +g19 +VPlucked string instrument +p2730 +ssa(dp2731 +g2 +(dp2732 +g4 +Vhttp://freebase.com/view/en/gudok +p2733 +sg6 +S"The gudok or hudok (Russian: \xd0\xb3\xd1\x83\xd0\xb4\xd0\xbe\xd0\xba, Ukrainian: \xd0\xb3\xd1\x83\xd0\xb4\xd0\xbe\xd0\xba) is an ancient Eastern Slavic string musical instrument, played with a bow.\nA gudok usually had three strings, two of them tuned in unison and played as a drone, the third tuned a fifth higher. All three strings were in the same plane at the bridge, so that a bow could make them all sound simultaneously. Sometimes the gudok also had several sympathetic strings (up to eight) under the sounding board. These made the gudok's sound warm and..." +p2734 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cmhjg +p2735 +sg10 +VGudok +p2736 +sg12 +Vhttp://indextank.com/_static/common/demo/02cmhjg.jpg +p2737 +ssg14 +(dp2738 +I0 +I0 +ssg16 +g2736 +sg17 +(dp2739 +g19 +VBowed string instruments +p2740 +ssa(dp2741 +g2 +(dp2742 +g4 +Vhttp://freebase.com/view/en/clavichord +p2743 +sg6 +S'The clavichord is a European stringed keyboard instrument known from the late Medieval, through the Renaissance, Baroque and Classical eras. Historically, it was widely used as a practice instrument and as an aid to composition, not being loud enough for larger performances. The clavichord produces sound by striking brass or iron strings with small metal blades called tangents. Vibrations are transmitted through the bridge(s) to the soundboard. The name is derived from the Latin word clavis,...' +p2744 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041bht2 +p2745 +sg10 +VClavichord +p2746 +sg12 +Vhttp://indextank.com/_static/common/demo/041bht2.jpg +p2747 +ssg14 +(dp2748 +I0 +I3 +ssg16 +g2746 +sg17 +(dp2749 +g19 +VKeyboard instrument +p2750 +ssa(dp2751 +g2 +(dp2752 +g4 +Vhttp://freebase.com/view/en/double_bass +p2753 +sg6 +S'The double bass, also called the string bass, upright bass or contrabass, is the largest and lowest-pitched bowed string instrument in the modern symphony orchestra, with strings usually tuned to E1, A1, D2 and G2 (see standard tuning). The double bass is a standard member of the string section of the symphony orchestra and smaller string ensembles in Western classical music. In addition, it is used in other genres such as jazz, 1950s-style blues and rock and roll, rockabilly/psychobilly,...' +p2754 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s_2wc +p2755 +sg10 +VDouble bass +p2756 +sg12 +Vhttp://indextank.com/_static/common/demo/03s_2wc.jpg +p2757 +ssg14 +(dp2758 +I0 +I454 +ssg16 +g2756 +sg17 +(dp2759 +g19 +VBowed string instruments +p2760 +ssa(dp2761 +g2 +(dp2762 +g4 +Vhttp://freebase.com/view/en/kanun +p2763 +sg6 +S'The Qanun (Azerbaijani\xc2\xa0; Turkish: kanun; qan\xc3\xban or kanun (Arabic: \xd9\x82\xd8\xa7\xd9\x86\xd9\x88\xd9\x86 q\xc4\x81n\xc5\xabn, plural \xd9\x82\xd9\x88\xd8\xa7\xd9\x86\xd9\x8a\xd9\x86 qaw\xc4\x81n\xc4\xabn; Persian: \xd9\x82\xd8\xa7\xd9\x86\xd9\x88\xd9\x86 q\xc4\x81n\xc5\xabn;Armenian: \xd6\x84\xd5\xa1\xd5\xb6\xd5\xb8\xd5\xb6 k\xe2\x80\x99anon; Greek: \xce\xba\xce\xb1\xce\xbd\xce\xbf\xce\xbd\xce\xac\xce\xba\xce\xb9) is a string instrument found in the 10th century in Farab in Turkestan. The name derives from the Greek word \xce\xba\xce\xb1\xce\xbd\xcf\x8e\xce\xbd, "kanon," which means rule, principle, and also "mode." Its traditional music is based on Maqamat. It is essentially a zither with a narrow trapezoidal soundboard. Nylon or PVC strings are stretched over a single...' +p2764 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cd8d4 +p2765 +sg10 +VKanun +p2766 +sg12 +Vhttp://indextank.com/_static/common/demo/02cd8d4.jpg +p2767 +ssg14 +(dp2768 +I0 +I1 +ssg16 +g2766 +sg17 +(dp2769 +g19 +VPlucked string instrument +p2770 +ssa(dp2771 +g2 +(dp2772 +g4 +Vhttp://freebase.com/view/en/tres +p2773 +sg6 +S'The tres is a 3-course, 6-string chordophone which was created in Cuba. \nIn Cuba, among the criollo class, the son was created as a song and a salon dance genre. Originally, a guitar, tiple or bandola, played rhythm and lead in the son, but ultimately these were replaced by a new native-born instrument which was a fusion of all three called the Cuban Tres.\nThe Cuban tres has three courses (groups) of two strings each for a total of six strings. From the low pitch to the highest, the...' +p2774 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0cmv5ww +p2775 +sg10 +VTres +p2776 +sg12 +Vhttp://indextank.com/_static/common/demo/0cmv5ww.jpg +p2777 +ssg14 +(dp2778 +I0 +I2 +ssg16 +g2776 +sg17 +(dp2779 +g19 +VPlucked string instrument +p2780 +ssa(dp2781 +g2 +(dp2782 +g4 +Vhttp://freebase.com/view/en/great_highland_bagpipe +p2783 +sg6 +S"The Great Highland Bagpipe (Scottish Gaelic: a' ph\xc3\xacob mh\xc3\xb2r; often abbreviated GHB in English) is a type of bagpipe native to Scotland. It has achieved widespread recognition through its usage in the British military and in pipe bands throughout the world. It is closely related to the Great Irish Warpipes.\nThe bagpipe is first attested in Scotland around 1400 AD, having previously appeared in European artwork in Spain in the 13th century. The earliest references to bagpipes in Scotland are in..." +p2784 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c9k5h +p2785 +sg10 +VGreat Highland Bagpipe +p2786 +sg12 +Vhttp://indextank.com/_static/common/demo/02c9k5h.jpg +p2787 +ssg14 +(dp2788 +I0 +I17 +ssg16 +g2786 +sg17 +(dp2789 +g19 +VBagpipes +p2790 +ssa(dp2791 +g2 +(dp2792 +g4 +Vhttp://freebase.com/view/en/holztrompete +p2793 +sg6 +S'A Holztrompete (Wooden Trumpet), an instrument somewhat resembling the Alpenhorn in tone-quality, designed by Richard Wagner for representing the natural pipe of the peasant in Tristan und Isolde. This instrument is not unlike the cor anglais in rough outline, being a conical tube of approximately the same length, terminating in a small globular bell, but having neither holes nor keys; it is blown through a cupshaped mouthpiece made of horn. The Holztrompete is in the key of C; the scale is...' +p2794 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/062_x0j +p2795 +sg10 +VHolztrompete +p2796 +sg12 +Vhttp://indextank.com/_static/common/demo/062_x0j.jpg +p2797 +ssg14 +(dp2798 +I0 +I0 +ssg16 +g2796 +sg17 +(dp2799 +g19 +VAlphorn +p2800 +ssa(dp2801 +g2 +(dp2802 +g4 +Vhttp://freebase.com/view/en/classical_guitar +p2803 +sg6 +S'The classical guitar \xe2\x80\x94 (also called the "Spanish guitar" or "nylon string guitar") \xe2\x80\x94 is a 6-stringed plucked string instrument from the family of instruments called chordophones. The classical guitar is well known for its comprehensive right hand technique, which allows the soloist to perform complex melodic and polyphonic material, in much the same manner as the piano.\nThe name classical guitar does not mean that only classical repertoire is performed on it, although classical music is a...' +p2804 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02ns86s +p2805 +sg10 +VClassical guitar +p2806 +sg12 +Vhttp://indextank.com/_static/common/demo/02ns86s.jpg +p2807 +ssg14 +(dp2808 +I0 +I30 +ssg16 +g2806 +sg17 +(dp2809 +g19 +VPlucked string instrument +p2810 +ssa(dp2811 +g2 +(dp2812 +g4 +Vhttp://freebase.com/view/en/contrabass_clarinet +p2813 +sg6 +S"The contrabass clarinet is the largest member of the clarinet family that has ever been in regular production or significant use. Modern contrabass clarinets are pitched in BB\xe2\x99\xad, sounding two octaves lower than the common B\xe2\x99\xad soprano clarinet and one octave lower than the B\xe2\x99\xad bass clarinet. Some contrabass clarinet models have a range extending down to low (written) E\xe2\x99\xad, while others can play down to low D or further to low C. Some early instruments were pitched in C; Arnold Schoenberg's F\xc3\xbcnf..." +p2814 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029xqmr +p2815 +sg10 +VContrabass clarinet +p2816 +sg12 +Vhttp://indextank.com/_static/common/demo/029xqmr.jpg +p2817 +ssg14 +(dp2818 +I0 +I1 +ssg16 +g2816 +sg17 +(dp2819 +g19 +VClarinet +p2820 +ssa(dp2821 +g2 +(dp2822 +g4 +Vhttp://freebase.com/view/en/dc_3 +p2823 +sg6 +S'DC-3 guitars were manufactured by Danelectro. A small number of DC-3\'s were manufactured in the late 1990s. The DC-3\'s design is based on classical Danelectro models, such as the DC-59. The DC-3 has three pickups, whereas the DC-59, only two. The "DC" stands for \'double cutaway\'.' +p2824 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029rynx +p2825 +sg10 +VDC-3 +p2826 +sg12 +Vhttp://indextank.com/_static/common/demo/029rynx.jpg +p2827 +ssg14 +(dp2828 +I0 +I0 +ssg16 +g2826 +sg17 +(dp2829 +g19 +VElectric guitar +p2830 +ssa(dp2831 +g2 +(dp2832 +g4 +Vhttp://freebase.com/view/en/krar +p2833 +sg6 +S"The krar is a five- or six-stringed bowl-shaped lyre from Eritrea and Ethiopia. The instrument is tuned to a pentatonic scale. A modern krar may be amplified, much in the same way as an electric guitar or violin.\nThe krar, a chordophone, is usually decorated with wood, cloth, and beads. Its five or six strings determine the available pitches. The instrument's tone depends on the musician's playing technique: bowing, strumming or plucking. If plucked, the instrument will produce a soft tone...." +p2834 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029gh0r +p2835 +sg10 +VKrar +p2836 +sg12 +Vhttp://indextank.com/_static/common/demo/029gh0r.jpg +p2837 +ssg14 +(dp2838 +I0 +I0 +ssg16 +g2836 +sg17 +(dp2839 +g19 +VPlucked string instrument +p2840 +ssa(dp2841 +g2 +(dp2842 +g4 +Vhttp://freebase.com/view/en/harp +p2843 +sg6 +S'The harp is a multi-stringed instrument which has the plane of its strings positioned perpendicularly to the soundboard. Organologically, it falls in the general category of chordophones (stringed instruments) and occupies its own sub category (the harps). All harps have a neck, resonator, and strings. Some, known as frame harps, also have a pillar; those lacking the pillar are referred to as open harps. Depending on its size (which varies considerably), a harp may be played while held in...' +p2844 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d48ns +p2845 +sg10 +VHarp +p2846 +sg12 +Vhttp://indextank.com/_static/common/demo/02d48ns.jpg +p2847 +ssg14 +(dp2848 +I0 +I48 +ssg16 +g2846 +sg17 +(dp2849 +g19 +VPlucked string instrument +p2850 +ssa(dp2851 +g2 +(dp2852 +g4 +Vhttp://freebase.com/view/en/dutar +p2853 +sg6 +S'The dutar ( Persian: \xd8\xaf\xd9\x88 \xd8\xaa\xd8\xa7\xd8\xb1 , Uzbek: dutor) (also dotar or doutar) is a traditional long-necked two-stringed lute found in Iran, Central Asia and South Asia. Its name comes from the Persian word for "two strings", \xd8\xaf\xd9\x88 \xd8\xaa\xd8\xa7\xd8\xb1 dot\xc4\x81r (' +p2854 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041g743 +p2855 +sg10 +VDutar +p2856 +sg12 +Vhttp://indextank.com/_static/common/demo/041g743.jpg +p2857 +ssg14 +(dp2858 +I0 +I1 +ssg16 +g2856 +sg17 +(dp2859 +g19 +VPlucked string instrument +p2860 +ssa(dp2861 +g2 +(dp2862 +g4 +Vhttp://freebase.com/view/en/player_piano +p2863 +sg6 +S'A player piano (also known as pianola or autopiano) is a self-playing piano, containing a pneumatic or electro-mechanical mechanism that operates the piano action via pre-programmed music perforated paper, or in rare instances, metallic rolls. The rise of the player piano grew with the rise of the mass-produced piano for the home in the late 19th and early 20th century. Sales peaked in 1924, as the improvement in phonograph recordings due to electrical recording methods developed in the...' +p2864 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bzl31 +p2865 +sg10 +VPlayer piano +p2866 +sg12 +Vhttp://indextank.com/_static/common/demo/02bzl31.jpg +p2867 +ssg14 +(dp2868 +I0 +I0 +ssg16 +g2866 +sg17 +(dp2869 +g19 +VPiano +p2870 +ssa(dp2871 +g2 +(dp2872 +g4 +Vhttp://freebase.com/view/en/thon_rammana +p2873 +sg6 +S'The thon-rammana (Thai: \xe0\xb9\x82\xe0\xb8\x97\xe0\xb8\x99\xe0\xb8\xa3\xe0\xb8\xb3\xe0\xb8\xa1\xe0\xb8\xb0\xe0\xb8\x99\xe0\xb8\xb2, pronounced\xc2\xa0[t\xca\xb0o\xcb\x90n ram.ma.na\xcb\x90]) are hand drums played as a pair in Central Thai classical music and Cambodian classical music. It consists of two drums: the thon (\xe0\xb9\x82\xe0\xb8\x97\xe0\xb8\x99), a goblet drum with a ceramic or wooden body) and the rammana (\xe0\xb8\xa3\xe0\xb8\xb3\xe0\xb8\xa1\xe0\xb8\xb0\xe0\xb8\x99\xe0\xb8\xb2), a small frame drum. They are used usually in the khruang sai ensemble. The thon gives a low pitch and the rammana gives a high pitch.\nEarlier in the 20th century, the thon and rammana were sometimes played separately.' +p2874 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05mhk3y +p2875 +sg10 +VThon-rammana +p2876 +sg12 +Vhttp://indextank.com/_static/common/demo/05mhk3y.jpg +p2877 +ssg14 +(dp2878 +I0 +I0 +ssg16 +g2876 +sg17 +(dp2879 +g19 +VPercussion +p2880 +ssa(dp2881 +g2 +(dp2882 +g4 +Vhttp://freebase.com/view/en/dombra +p2883 +sg6 +S'The dombura (in Turkish domb\xc4\xb1ra, in Uzbekistan and Tajikistan, also rendered dambura or danbura in northern Afghanistan, Tajikistan, and Uzbekistan, dumbura in Bashkir and Tatar, dombor in Mongolia, dombyra in Kazakhstan, Dombira or \xe5\x86\xac\xe4\xb8\x8d\xe6\x8b\x89--Dongbula in Xinjiang, China) is a long-necked lute popular in Central Asian nations. The name dombura is the Turkic rendering of Persian tanbur. The instrument shares some of its characteristics with the Central Asian komuz and dutar.\nThe instrument differs...' +p2884 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044m6d5 +p2885 +sg10 +VDombra +p2886 +sg12 +Vhttp://indextank.com/_static/common/demo/044m6d5.jpg +p2887 +ssg14 +(dp2888 +I0 +I0 +ssg16 +g2886 +sg17 +(dp2889 +g19 +VPlucked string instrument +p2890 +ssa(dp2891 +g2 +(dp2892 +g4 +Vhttp://freebase.com/view/en/torban +p2893 +sg6 +S'The torban (also teorban or Ukrainian theorbo) is a Ukrainian musical instrument that combines the features of the Baroque Lute with those of the psaltery. The \xd0\xa2orban differs from the more common European Bass lute known as the Theorbo in that it had additional short treble strings (known as prystrunky) strung along the treble side of the soundboard. It appeared ca. 1700, probably influenced by the central European Theorbo and the Angelique which Cossack mercenaries would have encountered in...' +p2894 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bxych +p2895 +sg10 +VTorban +p2896 +sg12 +Vhttp://indextank.com/_static/common/demo/02bxych.jpg +p2897 +ssg14 +(dp2898 +I0 +I0 +ssg16 +g2896 +sg17 +(dp2899 +g19 +VPlucked string instrument +p2900 +ssa(dp2901 +g2 +(dp2902 +g4 +Vhttp://freebase.com/view/en/sampler +p2903 +sg6 +S'A sampler is an electronic musical instrument similar in some respects to a synthesizer but, instead of generating sounds, it uses recordings (or "samples") of sounds that are loaded or recorded into it by the user and then played back by means of a keyboard, sequencer or other triggering device to perform or compose music. Because these samples are now usually stored in digital memory the information can be quickly accessed. A single sample may often be pitch-shifted to produce musical...' +p2904 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cgxsn +p2905 +sg10 +VSampler +p2906 +sg12 +Vhttp://indextank.com/_static/common/demo/02cgxsn.jpg +p2907 +ssg14 +(dp2908 +I0 +I105 +ssg16 +g2906 +sg17 +(dp2909 +g19 +V Electronic instruments +p2910 +ssa(dp2911 +g2 +(dp2912 +g4 +Vhttp://freebase.com/view/en/reed_organ +p2913 +sg6 +S'A reed organ, also called a parlor (or parlour) organ, pump organ, cabinet organ, cottage organ, is an organ that generates its sounds using free metal reeds. Smaller, cheaper and more portable than pipe organs, reed organs were widely used in smaller churches and in private homes in the 19th century, but their volume and tonal range are limited, and they were generally confined to one or two manuals, with pedal-boards being extremely rare.\nIn the generation of its tones, a reed organ is...' +p2914 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05lt0sf +p2915 +sg10 +VReed organ +p2916 +sg12 +Vhttp://indextank.com/_static/common/demo/05lt0sf.jpg +p2917 +ssg14 +(dp2918 +I0 +I3 +ssg16 +g2916 +sg17 +(dp2919 +g19 +VOrgan +p2920 +ssa(dp2921 +g2 +(dp2922 +g4 +Vhttp://freebase.com/view/en/shaker +p2923 +sg6 +S'The word shaker describes a large number of percussive musical instruments used for creating rhythm in music.\nThey are so called because the method of creating sound involves shaking them\xe2\x80\x94moving them back and forth rather than striking them. Most may also be struck for a greater accent on certain beats. Shakers are often used in rock and popular styles to jive the j ride pattern along with or substituting for the ride cymbal.\nA shaker may comprise a container, partially full of small loose...' +p2924 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d3lbb +p2925 +sg10 +VShaker +p2926 +sg12 +Vhttp://indextank.com/_static/common/demo/02d3lbb.jpg +p2927 +ssg14 +(dp2928 +I0 +I0 +ssg16 +g2926 +sg17 +(dp2929 +g19 +VPercussion +p2930 +ssa(dp2931 +g2 +(dp2932 +g4 +Vhttp://freebase.com/view/en/palendag +p2933 +sg6 +S"The palendag, also called Pulalu (Manabo and Mansaka), Palandag (Bagobo), Pulala (Bukidnon) and Lumundeg (Banuwaen) is a type of Philippine bamboo flute, the largest one used by the Maguindanaon, a smaller type of this instrument is called the Hulakteb (Bukidnon).. A lip-valley flute, it is considered the toughest of the three bamboo flutes (the others being the tumpong and the suling) to use because of the way one must shape one's lips against its tip to make a sound. The construction of..." +p2934 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03t2fqg +p2935 +sg10 +VPalendag +p2936 +sg12 +Vhttp://indextank.com/_static/common/demo/03t2fqg.jpg +p2937 +ssg14 +(dp2938 +I0 +I0 +ssg16 +g2936 +sg17 +(dp2939 +g19 +VFlute (transverse) +p2940 +ssa(dp2941 +g2 +(dp2942 +g4 +Vhttp://freebase.com/view/en/arpa_doppia +p2943 +sg6 +S"An Arpa Doppia is a Double Harp common throughout Europe between the 16th and 19th Centuries.\nIt was the lack of a full chromatic compass that the theorist Juan Bermudo identified as the main 'defect' of the harp in the mid-16th century. His 'remedies' included tunings with 8 or 9 notes to the octave (more than the 7 'white' notes, but less than the full 12-note chromatic scale) and tunings adapted to each mode, with different accidentals in each octave. But he noted that some players had..." +p2944 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03r8mkd +p2945 +sg10 +VArpa Doppia +p2946 +sg12 +Vhttp://indextank.com/_static/common/demo/03r8mkd.jpg +p2947 +ssg14 +(dp2948 +I0 +I0 +ssg16 +g2946 +sg17 +(dp2949 +g19 +VPlucked string instrument +p2950 +ssa(dp2951 +g2 +(dp2952 +g4 +Vhttp://freebase.com/view/en/alto_saxophone +p2953 +sg6 +S'The alto saxophone is a member of the saxophone family of woodwind instruments invented by Belgian instrument designer Adolphe Sax in 1841. It is smaller than the tenor but larger than the soprano, and is the type most used in classical compositions. The alto and tenor are the most common types of saxophone.\nThe alto saxophone is an E\xe2\x99\xad transposing instrument and reads the treble clef. A written C-natural sounds as the concert E\xe2\x99\xad a major sixth lower.\nThe range of the alto saxophone is from...' +p2954 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0291p9v +p2955 +sg10 +VAlto saxophone +p2956 +sg12 +Vhttp://indextank.com/_static/common/demo/0291p9v.jpg +p2957 +ssg14 +(dp2958 +I0 +I83 +ssg16 +g2956 +sg17 +(dp2959 +g19 +VSaxophone +p2960 +ssa(dp2961 +g2 +(dp2962 +g4 +Vhttp://freebase.com/view/en/guitar +p2963 +sg6 +S'The guitar is a plucked string instrument, usually played with fingers or a pick. The guitar consists of a body with a rigid neck to which the strings, generally six in number, are attached. Guitars are traditionally constructed of various woods and strung with animal gut or, more recently, with either nylon or steel strings. Some modern guitars are made of polycarbonate materials. Guitars are made and repaired by luthiers. There are two primary families of guitars: acoustic and...' +p2964 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/042392q +p2965 +sg10 +VGuitar +p2966 +sg12 +Vhttp://indextank.com/_static/common/demo/042392q.jpg +p2967 +ssg14 +(dp2968 +I0 +I5531 +ssg16 +g2966 +sg17 +(dp2969 +g19 +VPlucked string instrument +p2970 +ssa(dp2971 +g2 +(dp2972 +g4 +Vhttp://freebase.com/view/en/bass_clarinet +p2973 +sg6 +S'The bass clarinet is a musical instrument of the clarinet family. Like the more common soprano B\xe2\x99\xad clarinet, it is usually pitched in B\xe2\x99\xad (meaning it is a transposing instrument on which a written C sounds as B\xe2\x99\xad), but it plays notes an octave below the soprano B\xe2\x99\xad clarinet. Bass clarinets in other keys, notably C and A, also exist, but are very rare (in contrast to the regular A clarinet, which is quite common in classical music). Bass clarinets regularly perform in symphony orchestras, wind...' +p2974 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04r2w06 +p2975 +sg10 +VBass clarinet +p2976 +sg12 +Vhttp://indextank.com/_static/common/demo/04r2w06.jpg +p2977 +ssg14 +(dp2978 +I0 +I28 +ssg16 +g2976 +sg17 +(dp2979 +g19 +VClarinet +p2980 +ssa(dp2981 +g2 +(dp2982 +g4 +Vhttp://freebase.com/view/en/electric_sitar +p2983 +sg6 +S'An electric sitar is a kind of electric guitar designed to mimic the sound of the traditional Indian instrument, the sitar. Depending on the manufacturer and model, these instruments bear varying degrees of resemblance to the traditional sitar. Most resemble the electric guitar in the style of the body and headstock, though some have a body shaped to resemble that of the sitar (such as a model made by Danelectro).\nThe instrument was developed in the late 1960s at Danelectro, when many...' +p2984 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f_0sv +p2985 +sg10 +VElectric sitar +p2986 +sg12 +Vhttp://indextank.com/_static/common/demo/02f_0sv.jpg +p2987 +ssg14 +(dp2988 +I0 +I5 +ssg16 +g2986 +sg17 +(dp2989 +g19 +VGuitar +p2990 +ssa(dp2991 +g2 +(dp2992 +g4 +Vhttp://freebase.com/view/en/virginals +p2993 +sg6 +S'The virginals or virginal (the plural form does not necessarily denote more than one instrument) is a keyboard instrument of the harpsichord family. It was popular in northern Europe and Italy during the late Renaissance and early baroque periods.\nThe virginals is a smaller and simpler rectangular form of the harpsichord with only one string per note running more or less parallel to the keyboard on the long side of the case. Many, if not most, of the instruments were constructed without...' +p2994 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04s7dzx +p2995 +sg10 +VVirginals +p2996 +sg12 +Vhttp://indextank.com/_static/common/demo/04s7dzx.jpg +p2997 +ssg14 +(dp2998 +I0 +I1 +ssg16 +g2996 +sg17 +(dp2999 +g19 +VKeyboard instrument +p3000 +ssa(dp3001 +g2 +(dp3002 +g4 +Vhttp://freebase.com/view/en/tanbur +p3003 +sg6 +S'The term tanb\xc5\xabr (Persian: \xd8\xaa\xd9\x86\xd8\xa8\xd9\x88\xd8\xb1) can refer to various long-necked, fretted lutes originating in the Middle East or Central Asia. According to the New Grove Dictionary of Music and Musicians, "terminology presents a complicated situation. Nowadays the term tanbur (or tambur) is applied to a variety of distinct and related long-necked lutes used in art and folk traditions. Similar or identical instruments are also known by other terms."\nHowever one study has identified the name "tanb\xc5\xabr" as...' +p3004 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cm15x +p3005 +sg10 +VTanbur +p3006 +sg12 +Vhttp://indextank.com/_static/common/demo/02cm15x.jpg +p3007 +ssg14 +(dp3008 +I0 +I3 +ssg16 +g3006 +sg17 +(dp3009 +g19 +VPlucked string instrument +p3010 +ssa(dp3011 +g2 +(dp3012 +g4 +Vhttp://freebase.com/view/en/bandora +p3013 +sg6 +S'The Bandora or Bandore is a large long-necked plucked string-instrument that can be regarded as a bass cittern though it does not have the "re-entrant" tuning typical of the cittern. Probably first built by John Rose in England around 1560, it remained popular for over a century. A somewhat smaller version was the orpharion.\nFrequently one of the two bass instruments in a broken consort as associated with the works of Thomas Morley it is also a solo instrument in its own right. Anthony...' +p3014 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05ppmlp +p3015 +sg10 +VBandora +p3016 +sg12 +Vhttp://indextank.com/_static/common/demo/05ppmlp.jpg +p3017 +ssg14 +(dp3018 +I0 +I0 +ssg16 +g3016 +sg17 +(dp3019 +g19 +VPlucked string instrument +p3020 +ssa(dp3021 +g2 +(dp3022 +g4 +Vhttp://freebase.com/view/en/english_bagpipes +p3023 +sg6 +S'The English bagpipes are bagpipes played in England. Of these, the only continuous tradition is that of the Northumbrian smallpipes, which are used in the northeastern county of Northumberland.\nAlthough bagpipes had formerly been used in other parts of England dating back at least to the Middle Ages, all but the Northumbrian smallpipes died out. Their reconstruction is a contested issue, as several distinct types of "extinct" bagpipes have been claimed and "reconstructed" based upon...' +p3024 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cyw01 +p3025 +sg10 +VEnglish bagpipes +p3026 +sg12 +Vhttp://indextank.com/_static/common/demo/02cyw01.jpg +p3027 +ssg14 +(dp3028 +I0 +I0 +ssg16 +g3026 +sg17 +(dp3029 +g19 +VBagpipes +p3030 +ssa(dp3031 +g2 +(dp3032 +g4 +Vhttp://freebase.com/view/en/orpharion +p3033 +sg6 +S'The orpharion (pronounced /\xcb\x8c\xc9\x94\xcb\x90f\xc9\x99\xcb\x88ra\xc9\xaa\xc9\x99n/ or /\xc9\x94\xcb\x90\xcb\x88f\xc3\xa6r\xc9\xaa\xc9\x99n/) or opherion (pronounced /\xc9\x92\xcb\x88fi\xcb\x90r\xc9\xaa\xc9\x99n/) is a plucked instrument from the Renaissance. It is part of the cittern family. Its construction is similar to the larger bandora. The metal strings are tuned like a lute and are plucked with the fingers. Therefore, the orpharion can be used instead of a lute. The nut and bridge of an orpharion are typically sloped, so that the string length increases from treble to bass. Due to the extremely...' +p3034 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02gk9lz +p3035 +sg10 +VOrpharion +p3036 +sg12 +Vhttp://indextank.com/_static/common/demo/02gk9lz.jpg +p3037 +ssg14 +(dp3038 +I0 +I0 +ssg16 +g3036 +sg17 +(dp3039 +g19 +VPlucked string instrument +p3040 +ssa(dp3041 +g2 +(dp3042 +g4 +Vhttp://freebase.com/view/en/ashiko +p3043 +sg6 +S'An ashiko is a kind of drum shaped like a truncated cone and meant to be played with bare hands. The drum is played throughout sub-Saharan Africa and the Americas. The Ashiko has three primary tones, just like the Djembe. Ashiko is the male version of djembe.\nThe A\xe1\xb9\xa3\xc3\xadk\xc3\xb2 (Ashiko) drum, played by a skilled musician, can make a variety of tones, similar to the dundun or talking drums of Nigeria. It is associated with a whole genre of music of the same name and adherents of the Christian...' +p3044 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02chnjj +p3045 +sg10 +VAshiko +p3046 +sg12 +Vhttp://indextank.com/_static/common/demo/02chnjj.jpg +p3047 +ssg14 +(dp3048 +I0 +I0 +ssg16 +g3046 +sg17 +(dp3049 +g19 +VPercussion +p3050 +ssa(dp3051 +g2 +(dp3052 +g4 +Vhttp://freebase.com/view/en/lap_steel_guitar +p3053 +sg6 +S"The lap steel guitar is a type of steel guitar, an instrument derived from and similar to the guitar. The player changes pitch by pressing a metal or glass bar against the strings instead of by pressing strings against the fingerboard.\nThere are three main types of lap steel guitar:\nLap slide and resonator guitars may also be fitted with pickups, but do not depend on electrical amplification to produce sound.\nA lap steel guitar's strings are raised at both the nut and bridge ends of the..." +p3054 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029z7m_ +p3055 +sg10 +VLap steel guitar +p3056 +sg12 +Vhttp://indextank.com/_static/common/demo/029z7m_.jpg +p3057 +ssg14 +(dp3058 +I0 +I38 +ssg16 +g3056 +sg17 +(dp3059 +g19 +VGuitar +p3060 +ssa(dp3061 +g2 +(dp3062 +g4 +Vhttp://freebase.com/view/en/pocket_trumpet +p3063 +sg6 +S"The pocket trumpet is a compact size B\xe2\x99\xad trumpet, with the same playing range as the regular trumpet. The tubing is wound more tightly than that of a standard trumpet in order to reduce its size while retaining the instrument's range. It is not a standardized instrument to be found in orchestral brass sections and is generally regarded as a novelty. It is used mostly by trumpet players as a practice instrument that can be packed in a suitcase and taken to places where carrying standard..." +p3064 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c4xqc +p3065 +sg10 +VPocket trumpet +p3066 +sg12 +Vhttp://indextank.com/_static/common/demo/02c4xqc.jpg +p3067 +ssg14 +(dp3068 +I0 +I0 +ssg16 +g3066 +sg17 +(dp3069 +g19 +VTrumpet +p3070 +ssa(dp3071 +g2 +(dp3072 +g4 +Vhttp://freebase.com/view/en/bass_violin +p3073 +sg6 +S'Bass violin is the generic modern term used to denote various 16th- and 17th-century forms of bass instruments of the violin (i.e. "viola da braccio") family. They were the direct ancestor of the modern cello. Bass violins were usually somewhat larger than the modern cello, but tuned the same or sometimes just one step lower than it. Contemporary names for these instruments include "basso de viola da braccio," "basso da braccio," or the generic term "violone," which simply meant "large...' +p3074 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03rv9xn +p3075 +sg10 +VBass violin +p3076 +sg12 +Vhttp://indextank.com/_static/common/demo/03rv9xn.jpg +p3077 +ssg14 +(dp3078 +I0 +I1 +ssg16 +g3076 +sg17 +(dp3079 +g19 +VViolin +p3080 +ssa(dp3081 +g2 +(dp3082 +g4 +Vhttp://freebase.com/view/en/buccina +p3083 +sg6 +S'A buccina (Latin: buccina) or bucina (Latin: b\xc5\xabcina), anglicized buccin or bucine, is a brass instrument used in the ancient Roman army similar to the Cornu. An aeneator who blew a buccina was called a "buccinator" or "bucinator" (Latin: buccin\xc4\x81tor, b\xc5\xabcin\xc4\x81tor).\nIt was originally designed as a tube measuring some 11 to 12 feet in length, of narrow cylindrical bore, and played by means of a cup-shaped mouthpiece. The tube is bent round upon itself from the mouthpiece to the bell in the shape...' +p3084 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bbt6t +p3085 +sg10 +VBuccina +p3086 +sg12 +Vhttp://indextank.com/_static/common/demo/02bbt6t.jpg +p3087 +ssg14 +(dp3088 +I0 +I0 +ssg16 +g3086 +sg17 +(dp3089 +g19 +VBrass instrument +p3090 +ssa(dp3091 +g2 +(dp3092 +g4 +Vhttp://freebase.com/view/en/tenor_saxophone +p3093 +sg6 +S'The tenor saxophone is a medium-sized member of the saxophone family, a group of instruments invented by Adolphe Sax in the 1840s. The tenor, with the alto, are the two most common types of saxophones. The tenor is pitched in the key of B\xe2\x99\xad, and written as a transposing instrument in the treble clef, sounding an octave and a major second lower than the written pitch. Modern tenor saxophones which have a high F# key have a range from A\xe2\x99\xad2 to E5 (concert) and are therefore pitched one octave...' +p3094 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044l733 +p3095 +sg10 +VTenor saxophone +p3096 +sg12 +Vhttp://indextank.com/_static/common/demo/044l733.jpg +p3097 +ssg14 +(dp3098 +I0 +I115 +ssg16 +g3096 +sg17 +(dp3099 +g19 +VSaxophone +p3100 +ssa(dp3101 +g2 +(dp3102 +g4 +Vhttp://freebase.com/view/en/seven-string_guitar +p3103 +sg6 +S'A seven-string guitar is a guitar with seven strings instead of the usual six. Some types of seven-string guitars are specific to certain cultures (i.e. the Russian and Brazilian guitars).\nSeven-string electric guitars are particularly used in certain styles of music, and have been popular over the past few decades in the heavy metal genre. Mainstream artists such as Steve Vai, Muse, Dream Theater, Trivium, Staind, Rush, and Metallica have all experimented with seven-string guitars over the...' +p3104 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02b06zv +p3105 +sg10 +VSeven-string guitar +p3106 +sg12 +Vhttp://indextank.com/_static/common/demo/02b06zv.jpg +p3107 +ssg14 +(dp3108 +I0 +I10 +ssg16 +g3106 +sg17 +(dp3109 +g19 +VGuitar +p3110 +ssa(dp3111 +g2 +(dp3112 +g4 +Vhttp://freebase.com/view/en/trumpet +p3113 +sg6 +S'The trumpet is the musical instrument with the highest register in the brass family. Trumpets are among the oldest musical instruments, dating back to at least 1500 BCE. They are constructed of brass tubing bent twice into an oblong shape, and are played by blowing air through closed lips, producing a "buzzing" sound which starts a standing wave vibration in the air column inside the trumpet.\nThere are several types of trumpet; the most common is a transposing instrument pitched in B\xe2\x99\xad with a...' +p3114 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0291t0f +p3115 +sg10 +VTrumpet +p3116 +sg12 +Vhttp://indextank.com/_static/common/demo/0291t0f.jpg +p3117 +ssg14 +(dp3118 +I0 +I345 +ssg16 +g3116 +sg17 +(dp3119 +g19 +VBrass instrument +p3120 +ssa(dp3121 +g2 +(dp3122 +g4 +Vhttp://freebase.com/view/en/double_bell_euphonium +p3123 +sg6 +S'The double bell euphonium is an instrument based on the euphonium that has a second bell that emulates a sound such as a baritone horn or trombone that is mainly used for special effects, such as echoes.\nThe last valve on the horn (either the fourth or the fifth, depending upon the model) is used to switch the sound from the main bell to the secondary bell. Both bells cannot play at the same time because each bell usually has its own tuning slide loop, such that they can be matched...' +p3124 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dv55k +p3125 +sg10 +VDouble bell euphonium +p3126 +sg12 +Vhttp://indextank.com/_static/common/demo/02dv55k.jpg +p3127 +ssg14 +(dp3128 +I0 +I0 +ssg16 +g3126 +sg17 +(dp3129 +g19 +VEuphonium +p3130 +ssa(dp3131 +g2 +(dp3132 +g4 +Vhttp://freebase.com/view/en/bassoon +p3133 +sg6 +S'The bassoon is a woodwind instrument in the double reed family that typically plays music written in the bass and tenor registers, and occasionally higher. Appearing in its modern form in the 19th century, the bassoon figures prominently in orchestral, concert band, and chamber music literature. The bassoon is a non-transposing instrument known for its distinctive tone color, wide range, variety of character, and agility. Listeners often compare its warm, dark, reedy timbre to that of a male...' +p3134 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/042218j +p3135 +sg10 +VBassoon +p3136 +sg12 +Vhttp://indextank.com/_static/common/demo/042218j.jpg +p3137 +ssg14 +(dp3138 +I0 +I76 +ssg16 +g3136 +sg17 +(dp3139 +g19 +VWoodwind instrument +p3140 +ssa(dp3141 +g2 +(dp3142 +g4 +Vhttp://freebase.com/view/en/xiao +p3143 +sg6 +S'The xiao (simplified Chinese: \xe7\xae\xab; traditional Chinese: \xe7\xb0\xab; pinyin: xi\xc4\x81o; Wade\xe2\x80\x93Giles: hsiao) is a Chinese vertical end-blown flute. It is generally made of dark brown bamboo (called "purple bamboo" in Chinese). It is also sometimes (particularly in Taiwan) called d\xc3\xb2ngxi\xc4\x81o (simplified Chinese: \xe6\xb4\x9e\xe7\xae\xab; traditional Chinese: \xe6\xb4\x9e\xe7\xb0\xab), d\xc3\xb2ng meaning "hole." An ancient name for the xi\xc4\x81o is sh\xc3\xb9d\xc3\xad (\xe8\xb1\x8e\xe7\xac\x9b, lit. "vertical bamboo flute") but the name xi\xc4\x81o in ancient times also included the side-blown bamboo flute,...' +p3144 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02wlxpg +p3145 +sg10 +VXiao +p3146 +sg12 +Vhttp://indextank.com/_static/common/demo/02wlxpg.jpg +p3147 +ssg14 +(dp3148 +I0 +I0 +ssg16 +g3146 +sg17 +(dp3149 +g19 +VFlute (transverse) +p3150 +ssa(dp3151 +g2 +(dp3152 +g4 +Vhttp://freebase.com/view/en/viola +p3153 +sg6 +S'The viola ( /vi\xcb\x88o\xca\x8al\xc9\x99/ or /va\xc9\xaa\xcb\x88o\xca\x8al\xc9\x99/) is a bowed string instrument. It is the middle voice of the violin family, between the violin and the cello.\nThe viola is similar in material and construction to the violin but is larger in size and more variable in its proportions. A "full-size" viola\'s body is between one and four inches longer than the body of a full-size violin (i.e., between 15 and 18 inches (38 and 46 cm)), with an average length of about 16\xc2\xa0inches (41\xc2\xa0cm). Small violas made for...' +p3154 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bk5b0 +p3155 +sg10 +VViola +p3156 +sg12 +Vhttp://indextank.com/_static/common/demo/02bk5b0.jpg +p3157 +ssg14 +(dp3158 +I0 +I182 +ssg16 +g3156 +sg17 +(dp3159 +g19 +VBowed string instruments +p3160 +ssa(dp3161 +g2 +(dp3162 +g4 +Vhttp://freebase.com/view/en/pandura +p3163 +sg6 +S'The pandura is an ancient string instrument from the Mediterranian basin.\nThe ancient Greek pandoura (or pandora) (Greek: \xcf\x80\xce\xb1\xce\xbd\xce\xb4\xce\xbf\xcf\x8d\xcf\x81\xce\xb1) was a medium or long-necked lute with a small resonating chamber. It commonly had three strings: such an instrument was also known as the trichordon (McKinnon 1984:10). Its descendants still survive as Greek tambouras and bouzouki, North African Kuitras and Balkan tamburitsas. Renato Meucci (1996) suggests that the some Italian Renaissance descendants of Pandura...' +p3164 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04rxd7r +p3165 +sg10 +VPandura +p3166 +sg12 +Vhttp://indextank.com/_static/common/demo/04rxd7r.jpg +p3167 +ssg14 +(dp3168 +I0 +I0 +ssg16 +g3166 +sg17 +(dp3169 +g19 +VPlucked string instrument +p3170 +ssa(dp3171 +g2 +(dp3172 +g4 +Vhttp://freebase.com/view/en/saxhorn +p3173 +sg6 +S'The saxhorn is a valved brass instrument with a conical bore and deep cup-shaped mouthpiece. The sound has a characteristic mellow quality, and blends well with other brass.\nThe saxhorns form a family of seven instruments (although at one point ten different sizes seem to have existed). Designed for band use, they are pitched alternately in E-flat and B-flat, like the saxophone group.\nThere is much confusion as to nomenclature of the various instruments in different languages. This has been...' +p3174 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0291pc6 +p3175 +sg10 +VSaxhorn +p3176 +sg12 +Vhttp://indextank.com/_static/common/demo/0291pc6.jpg +p3177 +ssg14 +(dp3178 +I0 +I0 +ssg16 +g3176 +sg17 +(dp3179 +g19 +VBrass instrument +p3180 +ssa(dp3181 +g2 +(dp3182 +g4 +Vhttp://freebase.com/view/en/native_american_flute +p3183 +sg6 +S'The Native American flute has achieved some measure of fame for its distinctive sound, used in a variety of New Age and world music recordings. The instrument was originally very personal; its music was played without accompaniment in courtship, healing, meditation, and spiritual rituals. Now it is played solo, with other instruments or vocals, or with backing tracks. The flute has been used in Native American music, as well as other genres. There are two different types of Native American...' +p3184 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02fh3s0 +p3185 +sg10 +VNative American flute +p3186 +sg12 +Vhttp://indextank.com/_static/common/demo/02fh3s0.jpg +p3187 +ssg14 +(dp3188 +I0 +I3 +ssg16 +g3186 +sg17 +(dp3189 +g19 +VFlute (transverse) +p3190 +ssa(dp3191 +g2 +(dp3192 +g4 +Vhttp://freebase.com/view/en/gaohu +p3193 +sg6 +S'The gaohu (\xe9\xab\x98\xe8\x83\xa1; pinyin: g\xc4\x81oh\xc3\xba; Cantonese: gou1 wu4; also called yuehu \xe7\xb2\xa4\xe8\x83\xa1) is a Chinese bowed string instrument developed from the erhu in the 1920s by the musician and composer L\xc3\xbc Wencheng (1898\xe2\x80\x931981) and used in Cantonese music and Cantonese opera. It belongs to the huqin family of instruments, together with the zhonghu, erhu, banhu, jinghu, and sihu, its name means "high pitched huqin". It has two strings and its soundbox is covered on the front (playing) end with snakeskin (from a...' +p3194 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02fk9hb +p3195 +sg10 +VGaohu +p3196 +sg12 +Vhttp://indextank.com/_static/common/demo/02fk9hb.jpg +p3197 +ssg14 +(dp3198 +I0 +I0 +ssg16 +g3196 +sg17 +(dp3199 +g19 +VBowed string instruments +p3200 +ssa(dp3201 +g2 +(dp3202 +g4 +Vhttp://freebase.com/view/en/keyboard_bass +p3203 +sg6 +S"The keyboard bass is the use of a low-pitched keyboard or pedal keyboard to substitute for the bass guitar or double bass in popular music.\nThe earliest keyboard bass instrument was the 1960 Fender Rhodes piano bass, pictured above. The piano bass was an electric piano with the same pitch range as the electric bass (or the double bass), which could be used to perform basslines. It could be placed on top of a piano or organ, or mounted on a stand. As well, keyboard players such as The Doors'..." +p3204 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f8j3d +p3205 +sg10 +VKeyboard bass +p3206 +sg12 +Vhttp://indextank.com/_static/common/demo/02f8j3d.jpg +p3207 +ssg14 +(dp3208 +I0 +I4 +ssg16 +g3206 +sg17 +(dp3209 +g19 +VKeyboard instrument +p3210 +ssa(dp3211 +g2 +(dp3212 +g4 +Vhttp://freebase.com/view/en/duduk +p3213 +sg6 +S'The duduk is a traditional woodwind instrument indigenous to Armenia. Variations of it are popular in the Caucasus, the Middle East and Central Asia. The English word is often used generically for a family of ethnic instruments including the doudouk or duduk (\xd5\xa4\xd5\xb8\xd6\x82\xd5\xa4\xd5\xb8\xd6\x82\xd5\xaf), also tsiranapogh \xd5\xae\xd5\xab\xd6\x80\xd5\xa1\xd5\xb6\xd5\xa1\xd6\x83\xd5\xb8\xd5\xb2, literally "apricot horn" in Armenian), the d\xc3\xbcd\xc3\xbck or mey in Turkey, the duduki in Georgia, the balaban (or d\xc3\xbcd\xc3\xbck) in Azerbaijan, the narmeh-ney in Iran, the duduka or dudka in Russia and Ukraine. The...' +p3214 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dgq34 +p3215 +sg10 +VDuduk +p3216 +sg12 +Vhttp://indextank.com/_static/common/demo/02dgq34.jpg +p3217 +ssg14 +(dp3218 +I0 +I2 +ssg16 +g3216 +sg17 +(dp3219 +g19 +VFlute (transverse) +p3220 +ssa(dp3221 +g2 +(dp3222 +g4 +Vhttp://freebase.com/view/en/wanamaker_organ +p3223 +sg6 +S"The Wanamaker Grand Court Organ, in Philadelphia, Pennsylvania, is the largest operational pipe organ in the world, located within a spacious 7-story court at Macy's Center City (formerly Wanamaker's department store). The largest organ is the Boardwalk Hall Auditorium Organ (which is barely functional). The Wanamaker organ is played twice a day, Monday through Saturday, and more frequently during the Christmas season. The organ is also featured at several special concerts held throughout..." +p3224 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02b828b +p3225 +sg10 +VWanamaker Organ +p3226 +sg12 +Vhttp://indextank.com/_static/common/demo/02b828b.jpg +p3227 +ssg14 +(dp3228 +I0 +I0 +ssg16 +g3226 +sg17 +(dp3229 +g19 +VOrgan +p3230 +ssa(dp3231 +g2 +(dp3232 +g4 +Vhttp://freebase.com/view/en/grafton_saxophone +p3233 +sg6 +S"The Grafton saxophone was an injection moulded, cream-coloured acrylic plastic alto saxophone with metal keys, manufactured in London, England by the Grafton company, and later by 'John E. Dallas & Sons Ltd'. Only Grafton altos were ever made, due to the challenges in making larger models (i.e. the tenor) with 1950s plastic technology. Production commenced in 1950 and ended after approximately ten years. However, a few last examples were assembled from residual parts circa 1967. All tools,..." +p3234 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/07ktgsj +p3235 +sg10 +VGrafton saxophone +p3236 +sg12 +Vhttp://indextank.com/_static/common/demo/07ktgsj.jpg +p3237 +ssg14 +(dp3238 +I0 +I0 +ssg16 +g3236 +sg17 +(dp3239 +g19 +VSaxophone +p3240 +ssa(dp3241 +g2 +(dp3242 +g4 +Vhttp://freebase.com/view/en/irish_bouzouki +p3243 +sg6 +S'The Irish bouzouki is a unique development of the Greek bouzouki adapted for Irish traditional and folk music from the late 1960s onward.\nThe Greek bouzouki, in the newer tetrachordo (four course/eight string, or \xcf\x84\xce\xb5\xcf\x84\xcf\x81\xce\xac\xcf\x87\xce\xbf\xcf\x81\xce\xb4\xce\xbf) version developed in the twentieth century, was introduced into Irish Traditional Music in the late 1960s by Johnny Moynihan of the popular folk group Sweeney\xe2\x80\x99s Men, and popularized by Andy Irvine and D\xc3\xb3nal Lunny in the group Planxty. In a separate but parallel...' +p3244 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02csz05 +p3245 +sg10 +VIrish bouzouki +p3246 +sg12 +Vhttp://indextank.com/_static/common/demo/02csz05.jpg +p3247 +ssg14 +(dp3248 +I0 +I4 +ssg16 +g3246 +sg17 +(dp3249 +g19 +VPlucked string instrument +p3250 +ssa(dp3251 +g2 +(dp3252 +g4 +Vhttp://freebase.com/view/en/vibraphone +p3253 +sg6 +S'The vibraphone, sometimes called the vibraharp or simply the vibes, is a musical instrument in the mallet subfamily of the percussion family.\nIt is similar in appearance to the xylophone, marimba, and glockenspiel although the vibraphone uses aluminum bars instead of the wooden bars of the first two instruments. Each bar is paired with a resonator tube having a motor-driven butterfly valve at its upper end, mounted on a common shaft, which produces a tremolo or vibrato effect while spinning....' +p3254 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0292dq5 +p3255 +sg10 +VVibraphone +p3256 +sg12 +Vhttp://indextank.com/_static/common/demo/0292dq5.jpg +p3257 +ssg14 +(dp3258 +I0 +I45 +ssg16 +g3256 +sg17 +(dp3259 +g19 +VClassical percussion +p3260 +ssa(dp3261 +g2 +(dp3262 +g4 +Vhttp://freebase.com/view/en/tonbak +p3263 +sg6 +S"The tonbak (also tombak, donbak, dombak; in Persian: \xd8\xaa\xd9\x85\xd8\xa8\xda\xa9, or \xd8\xaa\xd9\x8f\xd9\x85\xd8\xa8\xd9\x8e\xda\xa9 ,\xd8\xaa\xd9\x86\xd8\xa8\xda\xa9 ,\xd8\xaf\xd9\x85\xd8\xa8\xda\xa9 ,\xd8\xaf\xd9\x86\xd8\xa8\xda\xa9) or zarb (\xd8\xb6\xd9\x8e\xd8\xb1\xd8\xa8 or \xd8\xb6\xd8\xb1\xd8\xa8) is a goblet drum from Persia (ancient Iran). It is considered the principal percussion instrument of Persian music. The tonbak is normally positioned diagonally across the torso while the player uses one or more fingers and/or the palm(s) of the hand(s) on the drumhead, often (for a ringing timbre) near the drumhead's edge. Sometimes tonbak players wear metal finger rings for an..." +p3264 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s6mz8 +p3265 +sg10 +VTonbak +p3266 +sg12 +Vhttp://indextank.com/_static/common/demo/03s6mz8.jpg +p3267 +ssg14 +(dp3268 +I0 +I2 +ssg16 +g3266 +sg17 +(dp3269 +g19 +VPercussion +p3270 +ssa(dp3271 +g2 +(dp3272 +g4 +Vhttp://freebase.com/view/en/saraswati_veena +p3273 +sg6 +S'The Saraswati veena (also spelled Saraswati vina) is an Indian plucked string instrument. It is named after the Hindu goddess Saraswati, who is usually depicted holding or playing the instrument.\nIt is one of the three other major types of veena popular today. The others include vichitra veena and rudra veena. Out of these the rudra and vichitra veenas are used in Hindustani music, while the Saraswati veena is used in the Carnatic music of South India. Some people play traditional music,...' +p3274 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dz35y +p3275 +sg10 +VSaraswati veena +p3276 +sg12 +Vhttp://indextank.com/_static/common/demo/02dz35y.jpg +p3277 +ssg14 +(dp3278 +I0 +I0 +ssg16 +g3276 +sg17 +(dp3279 +g19 +VPlucked string instrument +p3280 +ssa(dp3281 +g2 +(dp3282 +g4 +Vhttp://freebase.com/view/en/toy_piano +p3283 +sg6 +S"The toy piano, also known as the kinderklavier (child's keyboard), is a small piano-like musical instrument. The present form of the toy piano was invented in Philadelphia by a 17-year-old German immigrant named Albert Schoenhut. He worked as a repairman at Wanamaker's department store, repairing broken glass sounding pieces in German toy pianos damaged in shipping. Schoenhut conceived of the toy piano as it is known today in 1872, when he substituted durable steel plates for the traditional..." +p3284 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c15hd +p3285 +sg10 +VToy piano +p3286 +sg12 +Vhttp://indextank.com/_static/common/demo/02c15hd.jpg +p3287 +ssg14 +(dp3288 +I0 +I3 +ssg16 +g3286 +sg17 +(dp3289 +g19 +VPiano +p3290 +ssa(dp3291 +g2 +(dp3292 +g4 +Vhttp://freebase.com/view/en/cornett +p3293 +sg6 +S'The cornett, cornetto or zink is an early wind instrument, dating from the Medieval, Renaissance and Baroque periods. It was used in what are now called alta capellas or wind ensembles. It is not to be confused with the trumpet-like instrument cornet.\nThere are three basic types of treble cornett: curved, straight and mute. The curved (Ger. krummer Zink, schwarzer Zink; It. cornetto curvo, cornetto alto (i.e. \xe2\x80\x98loud\xe2\x80\x99), cornetto nero) is the most common type, with over 140 extant examples. It...' +p3294 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0292b3v +p3295 +sg10 +VCornett +p3296 +sg12 +Vhttp://indextank.com/_static/common/demo/0292b3v.jpg +p3297 +ssg14 +(dp3298 +I0 +I1 +ssg16 +g3296 +sg17 +(dp3299 +g19 +VWind instrument +p3300 +ssa(dp3301 +g2 +(dp3302 +g4 +Vhttp://freebase.com/view/en/gehu +p3303 +sg6 +S"The gehu (\xe9\x9d\xa9\xe8\x83\xa1; pinyin: g\xc3\xa9h\xc3\xba) is a Chinese instrument developed in the 20th century by the Chinese musician Yang Yusen (\xe6\x9d\xa8\xe9\x9b\xa8\xe6\xa3\xae, 1926-1980). It is a fusion of the Chinese huqin family and the cello. Its four strings are also tuned (from low to high) C-G-D-A, exactly like the cello's. Unlike most other musical instruments in the huqin family, the bridge does not contact the snakeskin, which faces to the side\nThere is also a contrabass gehu that functions as a Chinese double bass, known as the..." +p3304 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02gdwq6 +p3305 +sg10 +VGehu +p3306 +sg12 +Vhttp://indextank.com/_static/common/demo/02gdwq6.jpg +p3307 +ssg14 +(dp3308 +I0 +I0 +ssg16 +g3306 +sg17 +(dp3309 +g19 +VBowed string instruments +p3310 +ssa(dp3311 +g2 +(dp3312 +g4 +Vhttp://freebase.com/view/en/theatre_organ +p3313 +sg6 +S'A theatre organ (also known as a cinema organ) is a pipe organ originally designed specifically for imitation of an orchestra. New designs have tended to be around some of the sounds and blends unique to the instrument itself.\nTheatre organs took the place of the orchestra when installed in a movie theatre during the heyday of silent films. Most theatre organs were modelled after the style originally devised by Robert Hope-Jones, which he called a "unit orchestra".\nSuch instruments were...' +p3314 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dwqz6 +p3315 +sg10 +VTheatre organ +p3316 +sg12 +Vhttp://indextank.com/_static/common/demo/02dwqz6.jpg +p3317 +ssg14 +(dp3318 +I0 +I4 +ssg16 +g3316 +sg17 +(dp3319 +g19 +VPipe organ +p3320 +ssa(dp3321 +g2 +(dp3322 +g4 +Vhttp://freebase.com/view/en/zendrum +p3323 +sg6 +S'A Zendrum is a hand-crafted MIDI controller that is used as a percussion instrument. There are two Zendrum models that are well-suited for live performances, the ZX and the LT. The Zendrum ZX is worn like a guitar and consists of a triangular hardwood body with 24 touch-sensitive plastic pads which act as MIDI triggers. The Zendrum LT can also be worn with a guitar strap, but it has 25 MIDI triggers in a symmetrical layout, which provides an ambidextrous playing surface. The pads are played...' +p3324 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0dg07vb +p3325 +sg10 +VZendrum +p3326 +sg12 +Vhttp://indextank.com/_static/common/demo/0dg07vb.jpg +p3327 +ssg14 +(dp3328 +I0 +I2 +ssg16 +g3326 +sg17 +(dp3329 +g19 +VPercussion +p3330 +ssa(dp3331 +g2 +(dp3332 +g4 +Vhttp://freebase.com/view/en/sanxian +p3333 +sg6 +S'The sanxian (Chinese: \xe4\xb8\x89\xe5\xbc\xa6 (\xe7\xb5\x83?), literally "three strings") is a Chinese lute \xe2\x80\x94 a three-stringed fretless plucked musical instrument. It has a long fingerboard, and the body is traditionally made from snakeskin stretched over a rounded rectangular resonator. It is made in several sizes for different purposes and in the late 20th century a four-stringed version was also developed. The northern sanxian is generally larger, at about 122 cm in length, while southern versions of the instrument are...' +p3334 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s5n58 +p3335 +sg10 +VSanxian +p3336 +sg12 +Vhttp://indextank.com/_static/common/demo/03s5n58.jpg +p3337 +ssg14 +(dp3338 +I0 +I0 +ssg16 +g3336 +sg17 +(dp3339 +g19 +VPlucked string instrument +p3340 +ssa(dp3341 +g2 +(dp3342 +g4 +Vhttp://freebase.com/view/en/bamboo_flute +p3343 +sg6 +S'Flutes made of bamboo are found in many musical traditions.\nSome bamboo flutes include:' +p3344 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c3rxm +p3345 +sg10 +VBamboo flute +p3346 +sg12 +Vhttp://indextank.com/_static/common/demo/02c3rxm.jpg +p3347 +ssg14 +(dp3348 +I0 +I1 +ssg16 +g3346 +sg17 +(dp3349 +g19 +VFlute (transverse) +p3350 +ssa(dp3351 +g2 +(dp3352 +g4 +Vhttp://freebase.com/view/en/jouhikko +p3353 +sg6 +S"The jouhikko is a traditional, 2 or 3 stringed bowed lyre, from Finland and Karelia. Its strings are traditionally of horsehair. The playing of this instrument died out in the early 20th century but has been revived and there are now a number of musicians playing it. \nThe Jouhikko is also called jouhikannel or jouhikantele, meaning a bowed kantele. In English, the usual modern designation is 'bowed Lyre' though the earlier preferred term 'bowed Harp' is also met with. There are many..." +p3354 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/078v698 +p3355 +sg10 +VJouhikko +p3356 +sg12 +Vhttp://indextank.com/_static/common/demo/078v698.jpg +p3357 +ssg14 +(dp3358 +I0 +I0 +ssg16 +g3356 +sg17 +(dp3359 +g19 +VBowed string instruments +p3360 +ssa(dp3361 +g2 +(dp3362 +g4 +Vhttp://freebase.com/view/m/04w695 +p3363 +sg6 +S'The viola caipira (Portuguese for hillbilly guitar) is a ten-string, five-course guitar. Unlike most steel-string guitars, its strings are plucked with the fingers of the right hand similarly to the technique used for classical and flamenco guitars, rather than by the use of a plectrum.\nIt is a folk instrument commonly found in Brazil, where it is often simply called viola\nThe origins of the viola caipira are obscure, but folklorist Lu\xc3\xads da C\xc3\xa2mara Cascudo believes it to be an archaic form of...' +p3364 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05lxtkk +p3365 +sg10 +VViola caipira +p3366 +sg12 +Vhttp://indextank.com/_static/common/demo/05lxtkk.jpg +p3367 +ssg14 +(dp3368 +I0 +I1 +ssg16 +g3366 +sg17 +(dp3369 +g19 +VPlucked string instrument +p3370 +ssa(dp3371 +g2 +(dp3372 +g4 +Vhttp://freebase.com/view/en/trombone +p3373 +sg6 +S'The trombone (Ger. Posaune, Sp. tromb\xc3\xb3n) is a musical instrument in the brass family. Like all brass instruments, sound is produced when the player\xe2\x80\x99s vibrating lips (embouchure) cause the air column inside the instrument to vibrate. The trombone is usually characterised by a telescopic slide with which the player varies the length of the tube to change pitches, although the valve trombone uses three valves like those on a trumpet.\nThe word trombone derives from Italian tromba (trumpet) and...' +p3374 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bjmnh +p3375 +sg10 +VTrombone +p3376 +sg12 +Vhttp://indextank.com/_static/common/demo/02bjmnh.jpg +p3377 +ssg14 +(dp3378 +I0 +I309 +ssg16 +g3376 +sg17 +(dp3379 +g19 +VBrass instrument +p3380 +ssa(dp3381 +g2 +(dp3382 +g4 +Vhttp://freebase.com/view/en/violin +p3383 +sg6 +S'The violin is a string instrument, usually with four strings tuned in perfect fifths. It is the smallest, highest-pitched member of the violin family of string instruments, which includes the viola and cello.\nThe violin is sometimes informally called a fiddle, regardless of the type of music played on it. The word violin comes from the Middle Latin word vitula, meaning stringed instrument; this word is also believed to be the source of the Germanic "fiddle". The violin, while it has ancient...' +p3384 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bk3yy +p3385 +sg10 +VViolin +p3386 +sg12 +Vhttp://indextank.com/_static/common/demo/02bk3yy.jpg +p3387 +ssg14 +(dp3388 +I0 +I663 +ssg16 +g3386 +sg17 +(dp3389 +g19 +VBowed string instruments +p3390 +ssa(dp3391 +g2 +(dp3392 +g4 +Vhttp://freebase.com/view/en/snare_drum +p3393 +sg6 +S'The snare drum is a drum with strands of snares made of curled metal wire, metal cable, plastic cable, or gut cords stretched across the drumhead, typically the bottom. Pipe and tabor and some military snare drums often have a second set of snares on the bottom (internal) side of the top (batter) head to make a "brighter" sound. Different types can be found, like Piccolo snares, that have a smaller depth for a higher pitch, rope-tuned snares (Maracatoo snare) and the Brazilian "Tarol", that...' +p3394 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bdbkf +p3395 +sg10 +VSnare drum +p3396 +sg12 +Vhttp://indextank.com/_static/common/demo/02bdbkf.jpg +p3397 +ssg14 +(dp3398 +I0 +I3 +ssg16 +g3396 +sg17 +(dp3399 +g19 +VPercussion +p3400 +ssa(dp3401 +g2 +(dp3402 +g4 +Vhttp://freebase.com/view/en/bass_saxophone +p3403 +sg6 +S"The bass saxophone is the second largest existing member of the saxophone family (not counting the subcontrabass tubax). It is similar in design to a baritone saxophone, but it is larger, with a longer loop near the mouthpiece. Unlike the baritone, the bass saxophone is not commonly used. While some composers did write parts for the instrument through the early twentieth century (such as Percy Grainger in Lincolnshire Posy), the bass sax part in today's wind bands is usually handled by the..." +p3404 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f_0qz +p3405 +sg10 +VBass saxophone +p3406 +sg12 +Vhttp://indextank.com/_static/common/demo/02f_0qz.jpg +p3407 +ssg14 +(dp3408 +I0 +I3 +ssg16 +g3406 +sg17 +(dp3409 +g19 +VSaxophone +p3410 +ssa(dp3411 +g2 +(dp3412 +g4 +Vhttp://freebase.com/view/en/splash_cymbal +p3413 +sg6 +S'A splash cymbal is a small cymbal used for an accent in a drum kit. Splash cymbals and china cymbals are the main types of effects cymbals. The cymbal is also known as a multi-crash cymbal or crescent cymbal.\nMost splash cymbals range in size from 6" to 12" in diameter, though some splash cymbals go as low as 4". Some makers have produced cymbals described as splash cymbals up to 22" in diameter but these would be better described as medium thin crash cymbals. The most common size is 10",...' +p3414 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0290zn9 +p3415 +sg10 +VSplash cymbal +p3416 +sg12 +Vhttp://indextank.com/_static/common/demo/0290zn9.jpg +p3417 +ssg14 +(dp3418 +I0 +I0 +ssg16 +g3416 +sg17 +(dp3419 +g19 +VCymbal +p3420 +ssa(dp3421 +g2 +(dp3422 +g4 +Vhttp://freebase.com/view/en/timpani +p3423 +sg6 +S"Timpani, or kettledrums, are musical instruments in the percussion family. A type of drum, they consist of a skin called a head stretched over a large bowl traditionally made of copper. They are played by striking the head with a specialized drum stick called a timpani stick or timpani mallet. Unlike most drums, they are capable of producing an actual pitch when struck, and can be tuned, often with the use of a pedal mechanism to control each drum's range of notes. Timpani evolved from..." +p3424 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02byz95 +p3425 +sg10 +VTimpani +p3426 +sg12 +Vhttp://indextank.com/_static/common/demo/02byz95.jpg +p3427 +ssg14 +(dp3428 +I0 +I6 +ssg16 +g3426 +sg17 +(dp3429 +g19 +VPercussion +p3430 +ssa(dp3431 +g2 +(dp3432 +g4 +Vhttp://freebase.com/view/en/lyre +p3433 +sg6 +S'The lyre is a stringed musical instrument well known for its use in Greek classical antiquity and later. The word comes from the Greek "\xce\xbb\xcf\x8d\xcf\x81\xce\xb1" (lyra) and the earliest reference to the word is the Mycenaean Greek ru-ra-ta-e, meaning "lyrists", written in Linear B syllabic script. The earliest picture of a lyre with seven strings appears in the famous sarcophagus of Hagia Triada (a Minoan settlement in Crete). The sarcophagus was used during the Mycenaean occupation of Crete (1400 BC). The...' +p3434 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dvvxt +p3435 +sg10 +VLyre +p3436 +sg12 +Vhttp://indextank.com/_static/common/demo/02dvvxt.jpg +p3437 +ssg14 +(dp3438 +I0 +I1 +ssg16 +g3436 +sg17 +(dp3439 +g19 +VPlucked string instrument +p3440 +ssa(dp3441 +g2 +(dp3442 +g4 +Vhttp://freebase.com/view/en/theremin +p3443 +sg6 +S"The theremin (/\xcb\x88\xce\xb8\xc9\x9br\xc9\x99m\xc9\xaan/), originally known as the aetherphone/etherophone, thereminophone or termenvox/thereminvox is an early electronic musical instrument controlled without discernible physical contact from the player. It is named after its Russian inventor, Professor L\xc3\xa9on Theremin, who patented the device in 1928. The controlling section usually consists of two metal antennas which sense the position of the player's hands and control oscillators for frequency with one hand, and..." +p3444 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bgfft +p3445 +sg10 +VTheremin +p3446 +sg12 +Vhttp://indextank.com/_static/common/demo/02bgfft.jpg +p3447 +ssg14 +(dp3448 +I0 +I24 +ssg16 +g3446 +sg17 +(dp3449 +g19 +VElectronic keyboard +p3450 +ssa(dp3451 +g2 +(dp3452 +g4 +Vhttp://freebase.com/view/en/harpsichord +p3453 +sg6 +S'A harpsichord is a musical instrument played by means of a keyboard. It produces sound by plucking a string when a key is pressed.\nIn the narrow sense, "harpsichord" designates only the large wing-shaped instruments in which the strings are perpendicular to the keyboard. In a broader sense, "harpsichord" designates the whole family of similar plucked keyboard instruments, including the smaller virginals, muselar, and spinet.\nThe harpsichord was widely used in Renaissance and Baroque music....' +p3454 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05khvdd +p3455 +sg10 +VHarpsichord +p3456 +sg12 +Vhttp://indextank.com/_static/common/demo/05khvdd.jpg +p3457 +ssg14 +(dp3458 +I0 +I106 +ssg16 +g3456 +sg17 +(dp3459 +g19 +VPlucked string instrument +p3460 +ssa(dp3461 +g2 +(dp3462 +g4 +Vhttp://freebase.com/view/en/pedal_harp +p3463 +sg6 +S'The pedal harp (also known as the concert harp) is a large and technically modern harp, designed primarily for classical music and played either solo, as part of chamber ensembles, as soloist with or as a section or member in an orchestra. The pedal harp is a descendant of ancient harps.\nA pedal harp typically has six and a half octaves (46 or 47 strings), weighs about 80\xc2\xa0lb (36\xc2\xa0kg), is approximately 6\xc2\xa0ft (1.8 m) high, has a depth of 4\xc2\xa0ft (1.2 m), and is 21.5 in (55\xc2\xa0cm) wide at the bass end...' +p3464 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03t9bsc +p3465 +sg10 +VPedal harp +p3466 +sg12 +Vhttp://indextank.com/_static/common/demo/03t9bsc.jpg +p3467 +ssg14 +(dp3468 +I0 +I0 +ssg16 +g3466 +sg17 +(dp3469 +g19 +VHarp +p3470 +ssa(dp3471 +g2 +(dp3472 +g4 +Vhttp://freebase.com/view/en/piccolo_trumpet +p3473 +sg6 +S'The smallest of the trumpet family is the piccolo trumpet, pitched one octave higher than the standard B\xe2\x99\xad trumpet. Most piccolo trumpets are built to play in either B\xe2\x99\xad or A, using a separate leadpipe for each key. The tubing in the B\xe2\x99\xad piccolo trumpet is one-half the length of that in a standard B\xe2\x99\xad trumpet. Piccolo trumpets in G, F, and even high C are also manufactured, but are rarer.\nThe soprano trumpet in D is also known as the Bach trumpet and was invented in about 1890 by the Belgian...' +p3474 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03trfhw +p3475 +sg10 +VPiccolo trumpet +p3476 +sg12 +Vhttp://indextank.com/_static/common/demo/03trfhw.jpg +p3477 +ssg14 +(dp3478 +I0 +I2 +ssg16 +g3476 +sg17 +(dp3479 +g19 +VTrumpet +p3480 +ssa(dp3481 +g2 +(dp3482 +g4 +Vhttp://freebase.com/view/en/octave_mandolin +p3483 +sg6 +S'The octave mandolin is a fretted string instrument with four pairs of strings tuned in 5ths, G, D, A, E (low to high), an octave below a mandolin. It has a 20 to 23 inch scale length and its construction is similar to other instruments in the mandolin family. Usually the courses are all unison pairs but the lower two may sometimes be strung as octave pairs with the higher pitched octave string on top so that it is hit before the thicker lower pitched string.\nThe names of the mandolin family...' +p3484 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03sywnm +p3485 +sg10 +VOctave mandolin +p3486 +sg12 +Vhttp://indextank.com/_static/common/demo/03sywnm.jpg +p3487 +ssg14 +(dp3488 +I0 +I0 +ssg16 +g3486 +sg17 +(dp3489 +g19 +VPlucked string instrument +p3490 +ssa(dp3491 +g2 +(dp3492 +g4 +Vhttp://freebase.com/view/en/valiha +p3493 +sg6 +S'The valiha is a tube zither from Madagascar made from a species of local bamboo. It is played by plucking the strings, which may be made of metal or (originally) the bamboo skin which is pried up in long strands and propped up by small bridges made of pieces of dried gourd. The valiha is considered the national instrument of Madagascar.\nThe strings of the modern valiha are generally made of bicycle brake cable. The cables are unstrung into individual strands and each string of the instrument...' +p3494 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0dhf4b7 +p3495 +sg10 +VValiha +p3496 +sg12 +Vhttp://indextank.com/_static/common/demo/0dhf4b7.jpg +p3497 +ssg14 +(dp3498 +I0 +I1 +ssg16 +g3496 +sg17 +(dp3499 +g19 +VPlucked string instrument +p3500 +ssa(dp3501 +g2 +(dp3502 +g4 +Vhttp://freebase.com/view/en/bajo_sexto +p3503 +sg6 +S'A bajo sexto (Spanish: "sixth bass") is a musical instrument with 12 strings in 6 double courses, used in Mexican music. It is used primarily in norte\xc3\xb1o music of northern Mexico and across the border in the music of south Texas known as "Tex-Mex," "conjunto," or "m\xc3\xbasica mexicana-tejana".\nA similar instrument with five courses is the bajo quinto. The manufacture of bajo quinto achieved high quality in the 19th century, in the states of Aguascalientes, Morelos, Puebla, Oaxaca, Tlaxcala and...' +p3504 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05lclxv +p3505 +sg10 +VBajo sexto +p3506 +sg12 +Vhttp://indextank.com/_static/common/demo/05lclxv.jpg +p3507 +ssg14 +(dp3508 +I0 +I0 +ssg16 +g3506 +sg17 +(dp3509 +g19 +VPlucked string instrument +p3510 +ssa(dp3511 +g2 +(dp3512 +g4 +Vhttp://freebase.com/view/en/gaita_transmontana +p3513 +sg6 +S'The gaita transmontana (or gaita-de-fole transmontana, gaita mirandesa) is a type of bagpipe native to the Tr\xc3\xa1s-os-Montes region of Portugal.' +p3514 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0636_zv +p3515 +sg10 +VGaita transmontana +p3516 +sg12 +Vhttp://indextank.com/_static/common/demo/0636_zv.jpg +p3517 +ssg14 +(dp3518 +I0 +I0 +ssg16 +g3516 +sg17 +(dp3519 +g19 +VBagpipes +p3520 +ssa(dp3521 +g2 +(dp3522 +g4 +Vhttp://freebase.com/view/en/baroque_trumpet +p3523 +sg6 +S'The baroque trumpet is a musical instrument in the brass family. It is most associated with music from the 16th to 18th centuries. Often synonymous with \'natural trumpet\', the term \'baroque trumpet\' is also often used to differentiate a modern replica that has added vent holes, with an original or replica natural trumpet which does not.\nSee natural trumpet.\nNowadays, the term "baroque trumpet" has come to mean a reproduction or copy of an original natural trumpet. These are the instruments...' +p3524 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03qxzkp +p3525 +sg10 +VBaroque trumpet +p3526 +sg12 +Vhttp://indextank.com/_static/common/demo/03qxzkp.jpg +p3527 +ssg14 +(dp3528 +I0 +I0 +ssg16 +g3526 +sg17 +(dp3529 +g19 +VTrumpet +p3530 +ssa(dp3531 +g2 +(dp3532 +g4 +Vhttp://freebase.com/view/en/begena +p3533 +sg6 +S"The begena (or b\xc3\xa8gu\xc3\xa8na, as in French) is an Ethiopian and Eritrean string instrument that resembles a large lyre. According to Ethiopian tradition, Menelik I brought the instrument to Ethiopia from Israel, where David had used the begena to soothe King Saul's nerves and heal him of insomnia. Its actual origin remains in doubt, though Ethiopian manuscripts depict the instrument at the beginning of the 15th century (Kimberlin 1978: 13).\nKnown as the instrument of noblemen, monks, and the upper..." +p3534 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d06m7 +p3535 +sg10 +VBegena +p3536 +sg12 +Vhttp://indextank.com/_static/common/demo/02d06m7.jpg +p3537 +ssg14 +(dp3538 +I0 +I0 +ssg16 +g3536 +sg17 +(dp3539 +g19 +VPlucked string instrument +p3540 +ssa(dp3541 +g2 +(dp3542 +g4 +Vhttp://freebase.com/view/en/cimbasso +p3543 +sg6 +S'The Cimbasso is a brass instrument in the trombone family, with a sound ranging from warm and mellow to bright and menacing. It has three to five piston or rotary valves, a highly cylindrical bore, and is usually pitched in F or Bb. It is in the same range as a tuba or a contrabass trombone.\nThe modern instrument can be played by a tubist or a bass trombonist.\nThe modern cimbasso is most commonly used in opera scores by Verdi from Oberto to Aida and Puccini in Le Villi only, though the word...' +p3544 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bj9wv +p3545 +sg10 +VCimbasso +p3546 +sg12 +Vhttp://indextank.com/_static/common/demo/02bj9wv.jpg +p3547 +ssg14 +(dp3548 +I0 +I0 +ssg16 +g3546 +sg17 +(dp3549 +g19 +VTrombone +p3550 +ssa(dp3551 +g2 +(dp3552 +g4 +Vhttp://freebase.com/view/en/sarrusophone +p3553 +sg6 +S'The sarrusophone is a family of transposing musical instruments patented and placed into production by Pierre-Louis Gautrot in 1856. It was named after the French bandmaster Pierre-Auguste Sarrus (1813\xe2\x80\x931876) who is credited with the concept of the instrument (it is not clear if Sarrus benefited financially from this association). The instrument was intended to serve as a replacement in wind bands for the oboe and bassoon which, at that time, lacked the carrying power required for outdoor...' +p3554 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029947_ +p3555 +sg10 +VSarrusophone +p3556 +sg12 +Vhttp://indextank.com/_static/common/demo/029947_.jpg +p3557 +ssg14 +(dp3558 +I0 +I0 +ssg16 +g3556 +sg17 +(dp3559 +g19 +VWoodwind instrument +p3560 +ssa(dp3561 +g2 +(dp3562 +g4 +Vhttp://freebase.com/view/en/rhodes_piano +p3563 +sg6 +S'The Rhodes piano is an electro-mechanical piano, invented by Harold Rhodes during the fifties and later manufactured in a number of models, first in collaboration with Fender and after 1965 by CBS.\nAs a member of the electrophone sub-group of percussion instruments, it employs a piano-like keyboard with hammers that hit small metal tines, amplified by electromagnetic pickups. A 2001 New York Times article described the instrument as "a pianistic counterpart to the electric guitar" having a...' +p3564 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02923_q +p3565 +sg10 +VRhodes piano +p3566 +sg12 +Vhttp://indextank.com/_static/common/demo/02923_q.jpg +p3567 +ssg14 +(dp3568 +I0 +I13 +ssg16 +g3566 +sg17 +(dp3569 +g19 +VElectric piano +p3570 +ssa(dp3571 +g2 +(dp3572 +g4 +Vhttp://freebase.com/view/en/woodwind_instrument +p3573 +sg6 +S"A woodwind instrument is a musical instrument which produces sound when the player blows air against a sharp edge or through a reed, causing the air within its resonator (usually a column of air) to vibrate. Most of these instruments are made of wood but can be made of other materials, such as metals or plastics.\nWoodwind instruments can further be divided into 2 groups: flutes and reed instruments.\nThe modern symphony orchestra's woodwinds section typically includes: 3 flutes, 1 piccolo, 3..." +p3574 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bp_1j +p3575 +sg10 +VWoodwind instrument +p3576 +sg12 +Vhttp://indextank.com/_static/common/demo/02bp_1j.jpg +p3577 +ssg14 +(dp3578 +I0 +I7 +ssg16 +g3576 +sg17 +(dp3579 +g19 +VWind instrument +p3580 +ssa(dp3581 +g2 +(dp3582 +g4 +Vhttp://freebase.com/view/en/drum +p3583 +sg6 +S'The drum is a member of the percussion group of musical instruments, technically classified as the membranous. Drums consist of at least one membrane, called a drumhead or drum skin, that is stretched over a shell and struck, either directly with the player\'s hands, or with a drumstick, to produce sound. There is usually a "resonance head" on the underside of the drum. Other techniques have been used to cause drums to make sound, such as the thumb roll. Drums are the world\'s oldest and most...' +p3584 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s3c52 +p3585 +sg10 +VDrum +p3586 +sg12 +Vhttp://indextank.com/_static/common/demo/03s3c52.jpg +p3587 +ssg14 +(dp3588 +I0 +I1217 +ssg16 +g3586 +sg17 +(dp3589 +g19 +VPercussion +p3590 +ssa(dp3591 +g2 +(dp3592 +g4 +Vhttp://freebase.com/view/en/tro +p3593 +sg6 +S'Tro is the generic name for traditional bowed string instruments in Cambodia.\nInstruments in this family include the two-stringed tro u, tro sau toch, tro sau thom, and tro che, as well as the three-stringed tro Khmer spike fiddle.' +p3594 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041m7qn +p3595 +sg10 +VTro +p3596 +sg12 +Vhttp://indextank.com/_static/common/demo/041m7qn.jpg +p3597 +ssg14 +(dp3598 +I0 +I0 +ssg16 +g3596 +sg17 +(dp3599 +g19 +VBowed string instruments +p3600 +ssa(dp3601 +g2 +(dp3602 +g4 +Vhttp://freebase.com/view/en/kit_violin +p3603 +sg6 +S"The kit violin, or kit (Tanzmeistergeige in German), is a stringed musical instrument. It is essentially a very small violin, designed to fit in a pocket \xe2\x80\x94 hence its other common name, the pochette. It was used by dance masters in royal courts and other places of nobility, as well as by street musicians up until around the 18th century. Occasionally, the rebec was used in the same way. Several are called for (as violini piccoli alla francese - small French violins) in Monteverdi's 1607..." +p3604 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029mhmx +p3605 +sg10 +VKit violin +p3606 +sg12 +Vhttp://indextank.com/_static/common/demo/029mhmx.jpg +p3607 +ssg14 +(dp3608 +I0 +I0 +ssg16 +g3606 +sg17 +(dp3609 +g19 +VViolin +p3610 +ssa(dp3611 +g2 +(dp3612 +g4 +Vhttp://freebase.com/view/en/volynka +p3613 +sg6 +S'The volynka (Ukrainian: \xd0\xb2\xd0\xbe\xd0\xbb\xd0\xb8\xd0\xbd\xd0\xba\xd0\xb0, Russian: \xd0\xb2\xd0\xbe\xd0\xbb\xd1\x8b\xd0\xbd\xd0\xba\xd0\xb0, Crimean Tatar: tulup zurna \xe2\x80\x93 see also duda, koza, and kobza) is a Slavic bagpipe. Its etymology comes from the region Volyn, Ukraine, where it was borrowed from Romania.\nThe volynka is constructed around a goat skin air reservoir into which air is blown through a pipe with a valve to stop air escaping. (Modern concert instruments often have a reservoir made from a basketball bladder}. A number of playing pipes [two to four] extend from the...' +p3614 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03rlj3v +p3615 +sg10 +VVolynka +p3616 +sg12 +Vhttp://indextank.com/_static/common/demo/03rlj3v.jpg +p3617 +ssg14 +(dp3618 +I0 +I0 +ssg16 +g3616 +sg17 +(dp3619 +g19 +VBagpipes +p3620 +ssa(dp3621 +g2 +(dp3622 +g4 +Vhttp://freebase.com/view/en/berimbau +p3623 +sg6 +S"The berimbau (English pronounced /b\xc9\x99r\xc9\xaam\xcb\x88ba\xca\x8a/, Brazilian Portuguese [be\xc9\xbe\xc4\xa9\xcb\x88baw]) is a single-string percussion instrument, a musical bow, from Brazil. The berimbau's origins are not entirely clear, but there is not much doubt on its African origin, as no Indigenous Brazilian or European people use musical bows, and very similar instruments are played in the southern parts of Africa. The berimbau was eventually incorporated into the practice of the Afro-Brazilian martial art capoeira, where it..." +p3624 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029wcz0 +p3625 +sg10 +VBerimbau +p3626 +sg12 +Vhttp://indextank.com/_static/common/demo/029wcz0.jpg +p3627 +ssg14 +(dp3628 +I0 +I1 +ssg16 +g3626 +sg17 +(dp3629 +g19 +VStruck string instruments +p3630 +ssa(dp3631 +g2 +(dp3632 +g4 +Vhttp://freebase.com/view/en/domra +p3633 +sg6 +S'The domra (Russian: \xd0\xb4\xd0\xbe\xd0\xbc\xd1\x80\xd0\xb0) is a long-necked Russian string instrument of the lute family with a round body and three or four metal strings.\nIn 1896, a student of Vassily Vassilievich Andreyev found a broken instrument in a stable in rural Russia. It was thought that this instrument may have been an example of a domra, although no illustrations or examples of the traditional domra were known to exist in Russian chronicles. A three-stringed version of this instrument was later redesigned in...' +p3634 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bs20q +p3635 +sg10 +VDomra +p3636 +sg12 +Vhttp://indextank.com/_static/common/demo/02bs20q.jpg +p3637 +ssg14 +(dp3638 +I0 +I1 +ssg16 +g3636 +sg17 +(dp3639 +g19 +VPlucked string instrument +p3640 +ssa(dp3641 +g2 +(dp3642 +g4 +Vhttp://freebase.com/view/en/contrabass_saxophone +p3643 +sg6 +S'The contrabass saxophone is the lowest-pitched extant member of the saxophone family proper. It is extremely large (twice the length of tubing of the baritone saxophone, with a bore twice as wide, standing 1.9 meters tall, or 6 feet four inches) and heavy (approximately 20 kilograms, or 45 pounds), and is pitched in the key of E\xe2\x99\xad, one octave below the baritone.\nThe contrabass saxophone was part of the original saxophone family as conceived by Adolphe Sax, and is included in his saxophone...' +p3644 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02fx9nl +p3645 +sg10 +VContrabass saxophone +p3646 +sg12 +Vhttp://indextank.com/_static/common/demo/02fx9nl.jpg +p3647 +ssg14 +(dp3648 +I0 +I0 +ssg16 +g3646 +sg17 +(dp3649 +g19 +VSaxophone +p3650 +ssa(dp3651 +g2 +(dp3652 +g4 +Vhttp://freebase.com/view/en/nyckelharpa +p3653 +sg6 +S'A nyckelharpa (literally "key harp", plural nyckelharpor or sometimes keyed fiddle) is a traditional Swedish musical instrument. It is a string instrument or chordophone. Its keys are attached to tangents which, when a key is depressed, serve as frets to change the pitch of the string.\nThe nyckelharpa is similar in appearance to a fiddle or the bowed Byzantine lira. Structurally, it is more closely related to the hurdy gurdy, both employing key-actuated tangents to change the pitch.\nA...' +p3654 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029qc_t +p3655 +sg10 +VNyckelharpa +p3656 +sg12 +Vhttp://indextank.com/_static/common/demo/029qc_t.jpg +p3657 +ssg14 +(dp3658 +I0 +I5 +ssg16 +g3656 +sg17 +(dp3659 +g19 +VBowed string instruments +p3660 +ssa(dp3661 +g2 +(dp3662 +g4 +Vhttp://freebase.com/view/en/bandurria +p3663 +sg6 +S'The bandurria is a plectrum plucked chordophone from Spain, similar to the cittern and the mandolin, primarily used in Spanish folk music.\nPrior to the 18th century, the bandurria had with a round back, similar or related to the mandore. It had become a flat-backed instrument by the 18th century, with five double courses of strings, tuned in fourths. The original bandurrias of the Medieval period had three strings. During the Renaissance they gained a fourth string. During the Baroque period...' +p3664 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04pm0zr +p3665 +sg10 +VBandurria +p3666 +sg12 +Vhttp://indextank.com/_static/common/demo/04pm0zr.jpg +p3667 +ssg14 +(dp3668 +I0 +I0 +ssg16 +g3666 +sg17 +(dp3669 +g19 +VPlucked string instrument +p3670 +ssa(dp3671 +g2 +(dp3672 +g4 +Vhttp://freebase.com/view/en/steelpan +p3673 +sg6 +S'Steelpans (also known as steel drums or pans, and sometimes, collectively with musicians, as a steel band) is a musical instrument originating from Trinidad and Tobago. Steel pan musicians are called pannists.\nThe pan is a chromatically pitched percussion instrument (although some toy or novelty steelpans are tuned diatonically), made from 55 gallon drums that usually store oil. In fact, drum refers to the steel drum containers from which the pans are made; the steeldrum is correctly called...' +p3674 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03r1dm4 +p3675 +sg10 +VSteelpan +p3676 +sg12 +Vhttp://indextank.com/_static/common/demo/03r1dm4.jpg +p3677 +ssg14 +(dp3678 +I0 +I3 +ssg16 +g3676 +sg17 +(dp3679 +g19 +VPercussion +p3680 +ssa(dp3681 +g2 +(dp3682 +g4 +Vhttp://freebase.com/view/en/zither +p3683 +sg6 +S'The zither is a musical string instrument, most commonly found in Slovenia, Austria, Hungary citera, northwestern Croatia, the southern regions of Germany, alpine Europe and East Asian cultures, including China. The term "citre" is also used more broadly, to describe the entire family of stringed instruments in which the strings do not extend beyond the sounding box, including the hammered dulcimer, psaltery, Appalachian dulcimer, guqin, guzheng (Chinese zither), koto, gusli, kantele,...' +p3684 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c4cl9 +p3685 +sg10 +VZither +p3686 +sg12 +Vhttp://indextank.com/_static/common/demo/02c4cl9.jpg +p3687 +ssg14 +(dp3688 +I0 +I6 +ssg16 +g3686 +sg17 +(dp3689 +g19 +VPlucked string instrument +p3690 +ssa(dp3691 +g2 +(dp3692 +g4 +Vhttp://freebase.com/view/en/basset_clarinet +p3693 +sg6 +S'The basset clarinet is a clarinet, similar to the usual soprano clarinet but longer and with additional keys to enable playing several additional lower notes. Typically a basset clarinet has keywork going to a low (written) C, as opposed to the standard clarinet\'s E or E\xe2\x99\xad (both written), and is most commonly a transposing instrument in A, although basset clarinets in C and B\xe2\x99\xad also exist, and Stephen Fox makes a "G basset clarinet/basset horn". The similarly named basset horn is also a...' +p3694 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029tknz +p3695 +sg10 +VBasset clarinet +p3696 +sg12 +Vhttp://indextank.com/_static/common/demo/029tknz.jpg +p3697 +ssg14 +(dp3698 +I0 +I0 +ssg16 +g3696 +sg17 +(dp3699 +g19 +VClarinet +p3700 +ssa(dp3701 +g2 +(dp3702 +g4 +Vhttp://freebase.com/view/en/rattlesnake +p3703 +sg6 +S'Rattlesnakes are a group of venomous snakes, genera Crotalus and Sistrurus. They belong to the subfamily of venomous snakes known as Crotalinae (pit vipers).\nThere are approximately thirty species of rattlesnake, with numerous subspecies. They receive their name for the rattle located at the end of their tails. The rattle is used as a warning device when they are threatened, taking the place of a loud hiss as with other snakes. The scientific name Crotalus derives from the Greek, \xce\xba\xcf\x81\xcf\x8c\xcf\x84\xce\xb1\xce\xbb\xce\xbf\xce\xbd,...' +p3704 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02cttmf +p3705 +sg10 +VRattlesnake +p3706 +sg12 +Vhttp://indextank.com/_static/common/demo/02cttmf.jpg +p3707 +ssg14 +(dp3708 +I0 +I0 +ssg16 +g3706 +sg17 +(dp3709 +g19 +VRattle +p3710 +ssa(dp3711 +g2 +(dp3712 +g4 +Vhttp://freebase.com/view/en/sackbut +p3713 +sg6 +S'The sackbut (var. "sacbutt"; "sackbutt"; "sagbutt") is a trombone from the Renaissance and Baroque eras, i.e., a musical instrument in the brass family similar to the trumpet except characterised by a telescopic slide with which the player varies the length of the tube to change pitches, thus allowing them to obtain chromaticism, as well as easy and accurate doubling of voices. More delicately constructed than their modern counterparts, and featuring a softer, more flexible sound, they...' +p3714 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0291pbj +p3715 +sg10 +VSackbut +p3716 +sg12 +Vhttp://indextank.com/_static/common/demo/0291pbj.jpg +p3717 +ssg14 +(dp3718 +I0 +I0 +ssg16 +g3716 +sg17 +(dp3719 +g19 +VTrombone +p3720 +ssa(dp3721 +g2 +(dp3722 +g4 +Vhttp://freebase.com/view/en/twelve_string_guitar +p3723 +sg6 +S'The twelve-string guitar is an acoustic or electric guitar with 12 strings in 6 courses, which produces a richer, more ringing tone than a standard six-string guitar. Essentially, it is a type of guitar with a natural chorus effect due to the subtle differences in the frequencies produced by each of the two strings on each course.\nThe strings are placed in courses of two strings each that are usually played together. The two strings in each bass course are normally tuned an octave apart,...' +p3724 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03shp7_ +p3725 +sg10 +VTwelve string guitar +p3726 +sg12 +Vhttp://indextank.com/_static/common/demo/03shp7_.jpg +p3727 +ssg14 +(dp3728 +I0 +I20 +ssg16 +g3726 +sg17 +(dp3729 +g19 +VPlucked string instrument +p3730 +ssa(dp3731 +g2 +(dp3732 +g4 +Vhttp://freebase.com/view/en/clash_cymbals +p3733 +sg6 +S'Clash cymbals or hand cymbals are cymbals played in identical pairs by holding one cymbal in each hand and striking the two together.\nThe technical term clash cymbal is rarely used. In musical scores, clash cymbals are normally indicated as cymbals, crash cymbals, or sometimes simply C.C. If another type of cymbal, for example a suspended cymbal, is required in an orchestral score, then for historical reasons this is often indicated cymbals. Some composers and arrangers use the plural...' +p3734 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04pj4dz +p3735 +sg10 +VClash cymbals +p3736 +sg12 +Vhttp://indextank.com/_static/common/demo/04pj4dz.jpg +p3737 +ssg14 +(dp3738 +I0 +I0 +ssg16 +g3736 +sg17 +(dp3739 +g19 +VCymbal +p3740 +ssa(dp3741 +g2 +(dp3742 +g4 +Vhttp://freebase.com/view/en/liuqin +p3743 +sg6 +S'The liuqin (\xe6\x9f\xb3\xe7\x90\xb4; pinyin: li\xc7\x94q\xc3\xadn) is a four-stringed Chinese mandolin with a pear-shaped body. It is small in size, almost a miniature copy of another Chinese plucked musical instrument, the pipa. The range of its voice is much higher than the pipa, and it has its own special place in Chinese music, whether in orchestral music or in solo pieces. This has been the result of a modernization in its usage in recent years, leading to a gradual elevation in status of the liuqin from an accompaniment...' +p3744 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02d91l2 +p3745 +sg10 +VLiuqin +p3746 +sg12 +Vhttp://indextank.com/_static/common/demo/02d91l2.jpg +p3747 +ssg14 +(dp3748 +I0 +I0 +ssg16 +g3746 +sg17 +(dp3749 +g19 +VPlucked string instrument +p3750 +ssa(dp3751 +g2 +(dp3752 +g4 +Vhttp://freebase.com/view/en/tambourine +p3753 +sg6 +S'The tambourine or marine (commonly called tambo) is a musical instrument of the percussion family consisting of a frame, often of wood or plastic, with pairs of small metal jingles, called "zils". Classically the term tambourine denotes an instrument with a drumhead, though some variants may not have a head at all. Tambourines are often used with regular percussion sets. They can be mounted, but position is largely down to preference.\nTambourines come in many different shapes with the most...' +p3754 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0291sfp +p3755 +sg10 +VTambourine +p3756 +sg12 +Vhttp://indextank.com/_static/common/demo/0291sfp.jpg +p3757 +ssg14 +(dp3758 +I0 +I33 +ssg16 +g3756 +sg17 +(dp3759 +g19 +VPercussion +p3760 +ssa(dp3761 +g2 +(dp3762 +g4 +Vhttp://freebase.com/view/en/baglama +p3763 +sg6 +S'The ba\xc4\x9flama (Turkish: ba\xc4\x9flama, from ba\xc4\x9flamak, "to tie", pronounced\xc2\xa0[ba\xcb\x90\xc9\xaba\xcb\x88ma]) is a stringed musical instrument shared by various cultures in the Eastern Mediterranean, Near East, and Central Asia.\nIt is sometimes referred to as the saz (from the Persian \xd8\xb3\xd8\xa7\xd8\xb2\xe2\x80\x8e, meaning a kit or set), although the term "saz" actually refers to a family of plucked string instruments, long-necked lutes used in Ottoman classical music, Turkish folk music, Azeri music, Kurdish music, Persian music, Assyrian music,...' +p3764 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c5x6v +p3765 +sg10 +VBa\u011flama +p3766 +sg12 +Vhttp://indextank.com/_static/common/demo/02c5x6v.jpg +p3767 +ssg14 +(dp3768 +I0 +I7 +ssg16 +g3766 +sg17 +(dp3769 +g19 +VPlucked string instrument +p3770 +ssa(dp3771 +g2 +(dp3772 +g4 +Vhttp://freebase.com/view/en/soprano_saxophone +p3773 +sg6 +S'The soprano saxophone is a variety of the saxophone, a woodwind instrument, invented in 1840. The soprano is the third smallest member of the saxophone family, which consists (from smallest to largest) of the soprillo, sopranino, soprano, alto, tenor, baritone, bass, contrabass and tubax.\nA transposing instrument pitched in the key of B\xe2\x99\xad, modern soprano saxophones with a high F# key have a range from A\xe2\x99\xad3 to E6 and are therefore pitched one octave above the tenor saxophone. Some saxophones...' +p3774 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dl_1s +p3775 +sg10 +VSoprano saxophone +p3776 +sg12 +Vhttp://indextank.com/_static/common/demo/02dl_1s.jpg +p3777 +ssg14 +(dp3778 +I0 +I51 +ssg16 +g3776 +sg17 +(dp3779 +g19 +VSaxophone +p3780 +ssa(dp3781 +g2 +(dp3782 +g4 +Vhttp://freebase.com/view/en/sarod +p3783 +sg6 +S'The sarod is a stringed musical instrument, used mainly in Indian classical music. Along with the sitar, it is the most popular and prominent instrument in the classical music of Hindustan (northern India, Bangladesh and Pakistan). The sarod is known for a deep, weighty, introspective sound, in contrast with the sweet, overtone-rich texture of the sitar, with sympathetic strings that give it a resonant, reverberant quality. It is a fretless instrument able to produce the continuous slides...' +p3784 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044r6x1 +p3785 +sg10 +VSarod +p3786 +sg12 +Vhttp://indextank.com/_static/common/demo/044r6x1.jpg +p3787 +ssg14 +(dp3788 +I0 +I13 +ssg16 +g3786 +sg17 +(dp3789 +g19 +VPlucked string instrument +p3790 +ssa(dp3791 +g2 +(dp3792 +g4 +Vhttp://freebase.com/view/en/tabor +p3793 +sg6 +S'Tabor, or tabret, ("Tabwrdd" = Welsh) refers to a portable snare drum played with one hand. The word "tabor" is simply an English variant of a Latin-derived word meaning "drum" - cf. tambour (Fr.), tamburo (It.). It has been used in the military as a marching instrument, and has been used as accompaniment in parades and processions.\nA tabor has a cylindrical wood shell, two skin heads tightened by rope tension, a leather strap, and an adjustable gut snare. Each tabor has a pitch range of...' +p3794 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02g57g5 +p3795 +sg10 +VTabor +p3796 +sg12 +Vhttp://indextank.com/_static/common/demo/02g57g5.jpg +p3797 +ssg14 +(dp3798 +I0 +I0 +ssg16 +g3796 +sg17 +(dp3799 +g19 +VSnare drum +p3800 +ssa(dp3801 +g2 +(dp3802 +g4 +Vhttp://freebase.com/view/en/koto +p3803 +sg6 +S'The koto (\xe7\xae\x8f) is a traditional Japanese stringed musical instrument, similar to the Chinese guzheng. The koto is the national instrument of Japan. Koto are about 180\xc2\xa0centimetres (71\xc2\xa0in) width, and made from kiri wood (Paulownia tomentosa). They have 13 strings that are strung over 13 movable bridges along the width of the instrument. Players can adjust the string pitches by moving these bridges before playing, and use three finger picks (on thumb, index finger, and middle finger) to pluck the...' +p3804 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bg10g +p3805 +sg10 +VKoto +p3806 +sg12 +Vhttp://indextank.com/_static/common/demo/02bg10g.jpg +p3807 +ssg14 +(dp3808 +I0 +I5 +ssg16 +g3806 +sg17 +(dp3809 +g19 +VPlucked string instrument +p3810 +ssa(dp3811 +g2 +(dp3812 +g4 +Vhttp://freebase.com/view/en/haegeum +p3813 +sg6 +S'The haegeum is a traditional Korean string instrument, resembling a fiddle. It has a rodlike neck, a hollow wooden soundbox, and two silk strings, and is held vertically on the knee of the performer and played with a bow.\nThe haegeum is related to similar Chinese instruments in the huqin family of instruments, such as the erhu. Of these, it is most closely related to the ancient xiqin, as well as the erxian used in nanguan and Cantonese music.\nThe sohaegeum (\xec\x86\x8c\xed\x95\xb4\xea\xb8\x88) is a modernized fiddle with...' +p3814 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03sql3x +p3815 +sg10 +VHaegeum +p3816 +sg12 +Vhttp://indextank.com/_static/common/demo/03sql3x.jpg +p3817 +ssg14 +(dp3818 +I0 +I0 +ssg16 +g3816 +sg17 +(dp3819 +g19 +VBowed string instruments +p3820 +ssa(dp3821 +g2 +(dp3822 +g4 +Vhttp://freebase.com/view/en/violone +p3823 +sg6 +S'The term violone (literally "large viol" in Italian, "-one" being the augmentative suffix) can refer to several distinct large, bowed musical instruments which belong to either the viol or violin family. The violone is sometimes a fretted instrument, and may have six, five, four, or even only three strings. The violone is also not always a contrabass instrument. In modern parlance, one usually tries to clarify the \'type\' of violone by adding a qualifier based on the tuning (such as "G...' +p3824 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02g1bpy +p3825 +sg10 +VViolone +p3826 +sg12 +Vhttp://indextank.com/_static/common/demo/02g1bpy.jpg +p3827 +ssg14 +(dp3828 +I0 +I1 +ssg16 +g3826 +sg17 +(dp3829 +g19 +VViol +p3830 +ssa(dp3831 +g2 +(dp3832 +g4 +Vhttp://freebase.com/view/en/biniou +p3833 +sg6 +S'Binio\xc3\xb9 means bagpipe in the Breton language.\nThere are two bagpipes called binio\xc3\xb9 in Brittany: the traditional binio\xc3\xb9 kozh (kozh means "old" in Breton) and the binio\xc3\xb9 bras (bras means "big"), which was brought into Brittany from Scotland in the late 19th century. The oldest native bagpipe in Brittany is the veuze, from which the binio\xc3\xb9 kozh is thought to be derived.\nThe binio\xc3\xb9 bras is essentially the same as the Scottish Great Highland Bagpipe; sets are manufactured by Breton makers or...' +p3834 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05m2syg +p3835 +sg10 +VBiniou +p3836 +sg12 +Vhttp://indextank.com/_static/common/demo/05m2syg.jpg +p3837 +ssg14 +(dp3838 +I0 +I0 +ssg16 +g3836 +sg17 +(dp3839 +g19 +VBagpipes +p3840 +ssa(dp3841 +g2 +(dp3842 +g4 +Vhttp://freebase.com/view/en/mexican_vihuela +p3843 +sg6 +S'Vihuela is the name of two different guitar-like string instruments: the historical vihuela (proper) of 16th century Spain, usually with 12 paired strings, and the Mexican vihuela from 19th century Mexico with five strings and typically played in mariachi groups.\nWhile the Mexican vihuela shares the same name as the historic Spanish plucked string instrument, the two have little to do with each other, and they are not closely related. The Mexican vihuela has more in common with the Timple...' +p3844 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/09h1b5x +p3845 +sg10 +VMexican vihuela +p3846 +sg12 +Vhttp://indextank.com/_static/common/demo/09h1b5x.jpg +p3847 +ssg14 +(dp3848 +I0 +I0 +ssg16 +g3846 +sg17 +(dp3849 +g19 +VVihuela +p3850 +ssa(dp3851 +g2 +(dp3852 +g4 +Vhttp://freebase.com/view/en/acoustic_bass_guitar +p3853 +sg6 +S'The acoustic bass guitar (also called ABG or acoustic bass) is a bass instrument with a hollow wooden body similar to, though usually somewhat larger than a steel-string acoustic guitar. Like the traditional electric bass guitar and the double bass, the acoustic bass guitar commonly has four strings, which are normally tuned E-A-D-G, an octave below the lowest four strings of the 6-string guitar, which is the same tuning pitch as an electric bass guitar.\nBecause it can be difficult to hear...' +p3854 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dxvg3 +p3855 +sg10 +VAcoustic bass guitar +p3856 +sg12 +Vhttp://indextank.com/_static/common/demo/02dxvg3.jpg +p3857 +ssg14 +(dp3858 +I0 +I3 +ssg16 +g3856 +sg17 +(dp3859 +g19 +VPlucked string instrument +p3860 +ssa(dp3861 +g2 +(dp3862 +g4 +Vhttp://freebase.com/view/en/tom-tom_drum +p3863 +sg6 +S'A tom-tom drum (not to be confused with a tam-tam) is a cylindrical drum with no snare.\nAlthough "tom-tom" is the British term for a child\'s toy drum, the name came originally from the Anglo-Indian and Sinhala; the tom-tom itself comes from Asian or Native American cultures. The tom-tom drum is also a traditional means of communication. The tom-tom drum was added to the drum kit in the early part of the 20th century.\nThe first drum kit tom-toms had no rims; the heads were tacked to the...' +p3864 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bdbkf +p3865 +sg10 +VTom-tom drum +p3866 +sg12 +Vhttp://indextank.com/_static/common/demo/02bdbkf.jpg +p3867 +ssg14 +(dp3868 +I0 +I1 +ssg16 +g3866 +sg17 +(dp3869 +g19 +VPercussion +p3870 +ssa(dp3871 +g2 +(dp3872 +g4 +Vhttp://freebase.com/view/en/ibanez_jem +p3873 +sg6 +S"Ibanez JEM is an electric guitar manufactured by Ibanez and first produced in 1987. The guitar's most notable user is its co-designer, Steve Vai. As of 2010, there have been five sub-models of the JEM: the JEM7, JEM77, JEM777, JEM555 and the JEM333. Although the Ibanez JEM series is a signature series guitar, Ibanez mass-produces several of the guitar's sub-models.\nThe Ibanez JEM series is heavily influenced by the superstrat to model name or bodyshape is called a soloist concept, a more..." +p3874 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bdlbh +p3875 +sg10 +VIbanez JEM +p3876 +sg12 +Vhttp://indextank.com/_static/common/demo/02bdlbh.jpg +p3877 +ssg14 +(dp3878 +I0 +I1 +ssg16 +g3876 +sg17 +(dp3879 +g19 +VElectric guitar +p3880 +ssa(dp3881 +g2 +(dp3882 +g4 +Vhttp://freebase.com/view/en/gibson_les_paul +p3883 +sg6 +S'The Gibson Les Paul is a solid body electric guitar that was first sold in 1952. The Les Paul was designed by Ted McCarty in collaboration with popular guitarist Les Paul, whom Gibson enlisted to endorse the new model. It is one of the most well-known electric guitar types in the world, along with the Fender Stratocaster and Telecaster.\nThe Les Paul model was the result of a design collaboration between Gibson Guitar Corporation and the late jazz guitarist and electronics inventor Les Paul....' +p3884 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029xg0x +p3885 +sg10 +VGibson Les Paul +p3886 +sg12 +Vhttp://indextank.com/_static/common/demo/029xg0x.jpg +p3887 +ssg14 +(dp3888 +I0 +I1 +ssg16 +g3886 +sg17 +(dp3889 +g19 +VElectric guitar +p3890 +ssa(dp3891 +g2 +(dp3892 +g4 +Vhttp://freebase.com/view/m/09_4nd +p3893 +sg6 +S'The tulum (guda (\xe1\x83\x92\xe1\x83\xa3\xe1\x83\x93\xe1\x83\x90) in Laz) is a musical instrument, a form of bagpipe from Turkey. It is droneless with two parallel chanters, usually played by the Laz, Hamsheni people, and Pontic Greeks (particularly Chaldians). It is a prominent instrument in the music of Pazar, Hem\xc5\x9fin, \xc3\x87aml\xc4\xb1hem\xc5\x9fin, Arde\xc5\x9fen, F\xc4\xb1nd\xc4\xb1kl\xc4\xb1, Arhavi, Hopa, partly in other districts of Artvin and in the villages of the Tatos range (the watershed between the provinces of Rize and Trabzon) of \xc4\xb0spir. Tulum is the instrument of...' +p3894 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02fkf23 +p3895 +sg10 +VTulum +p3896 +sg12 +Vhttp://indextank.com/_static/common/demo/02fkf23.jpg +p3897 +ssg14 +(dp3898 +I0 +I0 +ssg16 +g3896 +sg17 +(dp3899 +g19 +VBagpipes +p3900 +ssa(dp3901 +g2 +(dp3902 +g4 +Vhttp://freebase.com/view/en/ukulele +p3903 +sg6 +S'The ukulele, ( /\xcb\x8cju\xcb\x90k\xc9\x99\xcb\x88le\xc9\xaali\xcb\x90/ EW-k\xc9\x99-LAY-lee; from Hawaiian: \xca\xbbukulele [\xcb\x88\xca\x94uku\xcb\x88l\xc9\x9bl\xc9\x9b]; variantly spelled ukelele in the UK), sometimes abbreviated to uke, is a chordophone classified as a plucked lute; it is a subset of the guitar family of instruments, generally with four nylon or gut strings or four courses of strings.\nThe ukulele originated in the 19th century as a Hawaiian interpretation of the cavaquinho or braguinha and the raj\xc3\xa3o, small guitar-like instruments taken to Hawai\xca\xbbi by...' +p3904 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bk34k +p3905 +sg10 +VUkulele +p3906 +sg12 +Vhttp://indextank.com/_static/common/demo/02bk34k.jpg +p3907 +ssg14 +(dp3908 +I0 +I95 +ssg16 +g3906 +sg17 +(dp3909 +g19 +VPlucked string instrument +p3910 +ssa(dp3911 +g2 +(dp3912 +g4 +Vhttp://freebase.com/view/en/messiah_stradivarius +p3913 +sg6 +S'The Messiah-Salabue Stradivarius of 1716 is a violin made by Italian luthier Antonio Stradivari of Cremona.\nThe Messiah, sobriquet Le Messie, remained in the Stradivarius workshop until his death in 1737. It was then sold by his son Paolo to Count Cozio di Salabue in 1775, and for a time, the violin bore the name Salabue. The instrument was then purchased by Luigi Tarisio in 1827. Upon Tarisio\xe2\x80\x99s death, in 1854, French luthier Jean Baptiste Vuillaume of Paris purchased The Messiah along with...' +p3914 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05m3gjg +p3915 +sg10 +VMessiah Stradivarius +p3916 +sg12 +Vhttp://indextank.com/_static/common/demo/05m3gjg.jpg +p3917 +ssg14 +(dp3918 +I0 +I0 +ssg16 +g3916 +sg17 +(dp3919 +g19 +VViolin +p3920 +ssa(dp3921 +g2 +(dp3922 +g4 +Vhttp://freebase.com/view/en/brass_instrument +p3923 +sg6 +S'A brass instrument is a musical instrument whose sound is produced by sympathetic vibration of air in a tubular resonator in sympathy with the vibration of the player\'s lips. Brass instruments are also called labrosones, literally meaning "lip-vibrated instruments".\nThere are several factors involved in producing different pitches on a brass instrument: One is alteration of the player\'s lip tension (or "embouchure"), and another is air flow. Also, slides (or valves) are used to change the...' +p3924 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02ds_pg +p3925 +sg10 +VBrass instrument +p3926 +sg12 +Vhttp://indextank.com/_static/common/demo/02ds_pg.jpg +p3927 +ssg14 +(dp3928 +I0 +I0 +ssg16 +g3926 +sg17 +(dp3929 +g19 +VWind instrument +p3930 +ssa(dp3931 +g2 +(dp3932 +g4 +Vhttp://freebase.com/view/en/bowed_psaltery +p3933 +sg6 +S'A bowed psaltery is a psaltery that is played with a bow.\nIn 1925 a German patent was issued to the Clemens Neuber Company for a bowed psaltery which also included a set of strings arranged in chords, so that one could play the melody on the bowed psaltery strings, and strum accompaniment with the other hand. These are usually called violin zithers.\nSimilar instruments were being produced by American companies of the same time period, often with Hawaiian-inspired names, such as Hawaiian Art...' +p3934 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02g96xy +p3935 +sg10 +VBowed psaltery +p3936 +sg12 +Vhttp://indextank.com/_static/common/demo/02g96xy.jpg +p3937 +ssg14 +(dp3938 +I0 +I1 +ssg16 +g3936 +sg17 +(dp3939 +g19 +VBowed string instruments +p3940 +ssa(dp3941 +g2 +(dp3942 +g4 +Vhttp://freebase.com/view/en/ocarina +p3943 +sg6 +S'The ocarina ( /\xc9\x92k\xc9\x99\xcb\x88ri\xcb\x90n\xc9\x99/) is an ancient flute-like wind instrument. Variations do exist, but a typical ocarina is an enclosed space with four to twelve finger holes and a mouthpiece that projects from the body. It is often ceramic, but other materials may also be used, such as plastic, wood, glass, clay, and metal.\nThe ocarina belongs to a very old family of instruments, believed to date back to over 12,000 years. Ocarina-type instruments have been of particular importance in Chinese and...' +p3944 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bzf1c +p3945 +sg10 +VOcarina +p3946 +sg12 +Vhttp://indextank.com/_static/common/demo/02bzf1c.jpg +p3947 +ssg14 +(dp3948 +I0 +I2 +ssg16 +g3946 +sg17 +(dp3949 +g19 +VDuct flutes +p3950 +ssa(dp3951 +g2 +(dp3952 +g4 +Vhttp://freebase.com/view/en/orchestrion +p3953 +sg6 +S'An orchestrion is a generic name for a machine that plays music and is designed to sound like an orchestra or band. Orchestrions may be operated by means of a large pinned cylinder or by a music roll and less commonly book music. The sound is usually produced by pipes, though they will be voiced differently to those found in a pipe organ, as well as percussion instruments. Many orchestrions contain a piano as well.\nThe fist known automatic playing orchestrion was the panharmonicon a musical...' +p3954 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02fkp50 +p3955 +sg10 +VOrchestrion +p3956 +sg12 +Vhttp://indextank.com/_static/common/demo/02fkp50.jpg +p3957 +ssg14 +(dp3958 +I0 +I0 +ssg16 +g3956 +sg17 +(dp3959 +g19 +VMechanical organ +p3960 +ssa(dp3961 +g2 +(dp3962 +g4 +Vhttp://freebase.com/view/en/didgeridoo +p3963 +sg6 +S'The didgeridoo (also known as a didjeridu or didge) is a wind instrument developed by Indigenous Australians of northern Australia at least 1,500 years ago and is still in widespread usage today both in Australia and around the world. It is sometimes described as a natural wooden trumpet or "drone pipe". Musicologists classify it as a brass aerophone.\nThere are no reliable sources stating the didgeridoo\'s exact age. Archaeological studies of rock art in Northern Australia suggest that the...' +p3964 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02912jy +p3965 +sg10 +VDidgeridoo +p3966 +sg12 +Vhttp://indextank.com/_static/common/demo/02912jy.jpg +p3967 +ssg14 +(dp3968 +I0 +I34 +ssg16 +g3966 +sg17 +(dp3969 +g19 +VNatural brass instruments +p3970 +ssa(dp3971 +g2 +(dp3972 +g4 +Vhttp://freebase.com/view/en/rackett +p3973 +sg6 +S'The rackett is a Renaissance-era double reed wind instrument related to the bassoon.\nThere are several sizes of rackett, in a family ranging from soprano to great bass. Relative to their pitch, racketts are quite small (the tenor rackett is only 4\xc2\xbd inches long, yet its lowest note is F, two octaves below middle C). This is achieved through its ingenious construction. The body consists of a wooden chamber into which nine parallel cylinders are drilled. These are connected alternately at the...' +p3974 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03r57_w +p3975 +sg10 +VRackett +p3976 +sg12 +Vhttp://indextank.com/_static/common/demo/03r57_w.jpg +p3977 +ssg14 +(dp3978 +I0 +I0 +ssg16 +g3976 +sg17 +(dp3979 +g19 +VWind instrument +p3980 +ssa(dp3981 +g2 +(dp3982 +g4 +Vhttp://freebase.com/view/en/soprano_cornet +p3983 +sg6 +S'The soprano cornet is a brass instrument that is very similar to the standard B\xe2\x99\xad cornet. It is a transposing instrument in E\xe2\x99\xad, pitched higher than the standard B\xe2\x99\xad cornet.\nOne soprano cornet is usually seen in brass bands and silver bands and can often be found playing lead or descant parts in ensembles.' +p3984 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05lxf0b +p3985 +sg10 +VSoprano cornet +p3986 +sg12 +Vhttp://indextank.com/_static/common/demo/05lxf0b.jpg +p3987 +ssg14 +(dp3988 +I0 +I0 +ssg16 +g3986 +sg17 +(dp3989 +g19 +VCornet +p3990 +ssa(dp3991 +g2 +(dp3992 +g4 +Vhttp://freebase.com/view/en/mezzo-soprano_saxophone +p3993 +sg6 +S'The mezzo-soprano saxophone, sometimes called the F alto saxophone, is an instrument in the saxophone family. It is in the key of F, pitched a whole step above the alto saxophone. Its size and the sound are similar to the E\xe2\x99\xad alto, although the upper register sounds more like a B\xe2\x99\xad soprano. Very few mezzo-sopranos exist \xe2\x80\x94 they were only produced in 1928 and 1929 by the C. G. Conn company. They were not popular and did not sell widely, as their production coincided with the Wall Street Crash of...' +p3994 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02gsfgs +p3995 +sg10 +VMezzo-soprano saxophone +p3996 +sg12 +Vhttp://indextank.com/_static/common/demo/02gsfgs.jpg +p3997 +ssg14 +(dp3998 +I0 +I0 +ssg16 +g3996 +sg17 +(dp3999 +g19 +VSaxophone +p4000 +ssa(dp4001 +g2 +(dp4002 +g4 +Vhttp://freebase.com/view/en/tumpong +p4003 +sg6 +S'The tumpong (also inci by Maranao) is a type of Philippine bamboo flute used by the Maguindanaon, half the size of the largest bamboo flute, the palendag. A lip-valley flute like the palendag, the tumpong makes a sound when players blow through \xc4\xb0NC\xc4\xb0 GELD\xc4\xb0 a bamboo reed placed on top of the instrument and the air stream produced is passed over an airhole atop the instrument. This masculine instrument is usually played during family gatherings in the evening and is presently the most common...' +p4004 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03t2fx_ +p4005 +sg10 +VTumpong +p4006 +sg12 +Vhttp://indextank.com/_static/common/demo/03t2fx_.jpg +p4007 +ssg14 +(dp4008 +I0 +I0 +ssg16 +g4006 +sg17 +(dp4009 +g19 +VFlute (transverse) +p4010 +ssa(dp4011 +g2 +(dp4012 +g4 +Vhttp://freebase.com/view/en/zhonghu +p4013 +sg6 +S'The zhonghu (\xe4\xb8\xad\xe8\x83\xa1, pinyin: zh\xc5\x8dngh\xc3\xba) is a low-pitched Chinese bowed string instrument. Together with the erhu and gaohu, it is a member of the huqin family, and was developed in the 20th century as the alto member of the huqin family (similar to how the European viola is used in traditional Chinese orchestras).\nThe zhonghu is analogous with the erhu, but is slightly larger and lower pitched. Its body is covered on the playing end with snakeskin. The instrument has two strings, which are...' +p4014 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05mnrh7 +p4015 +sg10 +VZhonghu +p4016 +sg12 +Vhttp://indextank.com/_static/common/demo/05mnrh7.jpg +p4017 +ssg14 +(dp4018 +I0 +I0 +ssg16 +g4016 +sg17 +(dp4019 +g19 +VBowed string instruments +p4020 +ssa(dp4021 +g2 +(dp4022 +g4 +Vhttp://freebase.com/view/en/tahitian_ukulele +p4023 +sg6 +S'The Tahitian ukulele (also known as the Tahitian banjo) is a short-necked fretted lute with eight nylon strings in four doubled courses, native to Tahiti. This variant of the older Hawaiian Ukulele is noted by a higher and thinner sound and is often strummed much faster.\nThe Tahitian ukulele is significantly different from other ukuleles in that it does not have a hollow soundbox. The body (including the head and neck) is usually carved from a single piece of wood, with a wide conical hole...' +p4024 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/04rxjcm +p4025 +sg10 +VTahitian ukulele +p4026 +sg12 +Vhttp://indextank.com/_static/common/demo/04rxjcm.jpg +p4027 +ssg14 +(dp4028 +I0 +I0 +ssg16 +g4026 +sg17 +(dp4029 +g19 +VPlucked string instrument +p4030 +ssa(dp4031 +g2 +(dp4032 +g4 +Vhttp://freebase.com/view/en/fender_stratocaster +p4033 +sg6 +S'The Fender Stratocaster, often referred to as "Strat", is a model of electric guitar designed by Leo Fender, George Fullerton, and Freddie Tavares in 1954, and manufactured continuously by the Fender Musical Instruments Corporation to the present. It is a double-cutaway guitar, with an extended top horn for balance while standing. The Stratocaster has been used by many leading guitarists and can be heard on many historic recordings. Along with the Gibson Les Paul, the Gibson SG and the...' +p4034 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044rrlv +p4035 +sg10 +VFender Stratocaster +p4036 +sg12 +Vhttp://indextank.com/_static/common/demo/044rrlv.jpg +p4037 +ssg14 +(dp4038 +I0 +I40 +ssg16 +g4036 +sg17 +(dp4039 +g19 +VGuitar +p4040 +ssa(dp4041 +g2 +(dp4042 +g4 +Vhttp://freebase.com/view/en/crotales +p4043 +sg6 +S'Crotales (pronounced "kro-tah\'-les", IPA: [kro\xcb\x88t\xce\xb1les]), sometimes called antique cymbals, are percussion instruments consisting of small, tuned bronze or brass disks. Each is about 4\xc2\xa0inches in diameter with a flat top surface and a nipple on the base. They are commonly played by being struck with hard mallets. However, they may also be played by striking two disks together in the same manner as finger cymbals, or by bowing. Their sound is rather like a small tuned bell, only with a much...' +p4044 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029y2l0 +p4045 +sg10 +VCrotales +p4046 +sg12 +Vhttp://indextank.com/_static/common/demo/029y2l0.jpg +p4047 +ssg14 +(dp4048 +I0 +I0 +ssg16 +g4046 +sg17 +(dp4049 +g19 +VCymbal +p4050 +ssa(dp4051 +g2 +(dp4052 +g4 +Vhttp://freebase.com/view/en/drumitar +p4053 +sg6 +S'The Synthaxe Drumitar is an instrument created by Roy "Future Man" Wooten of B\xc3\xa9la Fleck and the Flecktones. The Drumitar comprises piezo elements mounted in a guitar shaped body, connected by cable to assorted MIDI devices including samplers and drum machines. The instrument is one of a kind, and it was originally modified from a SynthAxe previously owned by jazz musician Lee Ritenour.\nThe Zendrum is a similar instrument that is commercially available.' +p4054 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s6xrn +p4055 +sg10 +VDrumitar +p4056 +sg12 +Vhttp://indextank.com/_static/common/demo/03s6xrn.jpg +p4057 +ssg14 +(dp4058 +I0 +I1 +ssg16 +g4056 +sg17 +(dp4059 +g19 +VZendrum +p4060 +ssa(dp4061 +g2 +(dp4062 +g4 +Vhttp://freebase.com/view/en/shawm +p4063 +sg6 +S'The shawm was a medieval and Renaissance musical instrument of the woodwind family made in Europe from the 12th century (at the latest) until the 17th century. It was developed from the oriental zurna and is the predecessor of the modern oboe. The body of the shawm was usually turned from a single piece of wood, and terminated in a flared bell somewhat like that of a trumpet. Beginning in the 16th century, shawms were made in several sizes, from sopranino to great bass, and four and...' +p4064 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c9mfg +p4065 +sg10 +VShawm +p4066 +sg12 +Vhttp://indextank.com/_static/common/demo/02c9mfg.jpg +p4067 +ssg14 +(dp4068 +I0 +I0 +ssg16 +g4066 +sg17 +(dp4069 +g19 +VWoodwind instrument +p4070 +ssa(dp4071 +g2 +(dp4072 +g4 +Vhttp://freebase.com/view/en/duda +p4073 +sg6 +S'The Magyar duda\xe2\x80\x94Hungarian duda\xe2\x80\x94(also known as t\xc3\xb6ml\xc3\xb6s\xc3\xadp and b\xc3\xb6rduda) is the traditional bagpipe of Hungary. It is an example of a group of bagpipes called Medio-Carparthian bagpipes. In common with most bagpipes in the area east of an imaginary line running from the Baltics to the Istrian Coast, the duda\xe2\x80\x99s chanters use single reeds much like Western drone reeds.\nDudmaisis or duda are made of sheep, ox, goat or dogskin or of sheep\xe2\x80\x99s stomach. A blowing tube is attached to the top. On one side...' +p4074 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02g521y +p4075 +sg10 +VDuda +p4076 +sg12 +Vhttp://indextank.com/_static/common/demo/02g521y.jpg +p4077 +ssg14 +(dp4078 +I0 +I0 +ssg16 +g4076 +sg17 +(dp4079 +g19 +VBagpipes +p4080 +ssa(dp4081 +g2 +(dp4082 +g4 +Vhttp://freebase.com/view/en/lute +p4083 +sg6 +S'Lute can refer generally to any plucked string instrument with a neck (either fretted or unfretted) and a deep round back, or more specifically to an instrument from the family of European lutes.\nThe European lute and the modern Near-Eastern oud both descend from a common ancestor via diverging evolutionary paths. The lute is used in a great variety of instrumental music from the early Renaissance to the late Baroque eras. It is also an accompanying instrument, especially in vocal works,...' +p4084 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s3ynh +p4085 +sg10 +VLute +p4086 +sg12 +Vhttp://indextank.com/_static/common/demo/03s3ynh.jpg +p4087 +ssg14 +(dp4088 +I0 +I23 +ssg16 +g4086 +sg17 +(dp4089 +g19 +VPlucked string instrument +p4090 +ssa(dp4091 +g2 +(dp4092 +g4 +Vhttp://freebase.com/view/en/c_melody_saxophone +p4093 +sg6 +S'The C melody saxophone is a saxophone pitched in the key of C, one whole step above the tenor saxophone. In the UK it is sometimes referred to as a "C tenor", and in France as a "tenor en ut". The C melody was part of the series of saxophones pitched in C and F, intended by the instrument\'s inventor, Adolphe Sax, for orchestral use. Since 1930, only saxophones in the key of B\xe2\x99\xad and E\xe2\x99\xad (originally intended by Sax for use in military bands and wind ensembles) have been produced on a large...' +p4094 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bvy35 +p4095 +sg10 +VC melody saxophone +p4096 +sg12 +Vhttp://indextank.com/_static/common/demo/02bvy35.jpg +p4097 +ssg14 +(dp4098 +I0 +I1 +ssg16 +g4096 +sg17 +(dp4099 +g19 +VSaxophone +p4100 +ssa(dp4101 +g2 +(dp4102 +g4 +Vhttp://freebase.com/view/en/degerpipes +p4103 +sg6 +S"The electronic bagpipes are an electronic instrument emulating the tone and/or playing style of the bagpipes. Most electronic bagpipe emulators feature a simulated chanter, which is used to play the melody. Some models also produce a harmonizing drone(s). Some variants employ a simulated bag, wherein the player's pressure on the bag activates a switch maintaining a constant tone.\nElectronic bagpipes are produced to replicate various types of bagpipes from around the world, including the..." +p4104 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03thxc1 +p4105 +sg10 +VElectronic bagpipes +p4106 +sg12 +Vhttp://indextank.com/_static/common/demo/03thxc1.jpg +p4107 +ssg14 +(dp4108 +I0 +I0 +ssg16 +g4106 +sg17 +(dp4109 +g19 +VBagpipes +p4110 +ssa(dp4111 +g2 +(dp4112 +g4 +Vhttp://freebase.com/view/m/042v_gx +p4113 +sg6 +S'An acoustic guitar is a guitar that uses only acoustic methods to project the sound produced by its strings. The term is a retronym, coined after the advent of electric guitars, which rely on electronic amplification to make their sound audible.\nIn all types of guitars the sound is produced by the vibration of the strings. However, because the string can only displace a small amount of air, the volume of the sound needs to be increased in order to be heard. In an acoustic guitar, this is...' +p4114 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0290vfk +p4115 +sg10 +VAcoustic guitar +p4116 +sg12 +Vhttp://indextank.com/_static/common/demo/0290vfk.jpg +p4117 +ssg14 +(dp4118 +I0 +I276 +ssg16 +g4116 +sg17 +(dp4119 +g19 +VPlucked string instrument +p4120 +ssa(dp4121 +g2 +(dp4122 +g4 +Vhttp://freebase.com/view/en/concertina +p4123 +sg6 +S'A concertina is a free-reed musical instrument, like the various accordions and the harmonica. It has a bellows and buttons typically on both ends of it. When pressed, the buttons travel in the same direction as the bellows, unlike accordion buttons which travel perpendicularly to it. Also, each button produces one note, while accordions typically can produce chords with a single button.\nThe concertina was developed in England and Germany, most likely independently. The English version was...' +p4124 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0292mbm +p4125 +sg10 +VConcertina +p4126 +sg12 +Vhttp://indextank.com/_static/common/demo/0292mbm.jpg +p4127 +ssg14 +(dp4128 +I0 +I10 +ssg16 +g4126 +sg17 +(dp4129 +g19 +VAccordion +p4130 +ssa(dp4131 +g2 +(dp4132 +g4 +Vhttp://freebase.com/view/en/an_tranh +p4133 +sg6 +S'The \xc4\x91\xc3\xa0n tranh (\xe5\xbd\x88\xe7\xae\x8f) is a plucked zither of Vietnam. It has a wooden body and steel strings, each of which is supported by a bridge in the shape of an inverted "V."\nThe \xc4\x91\xc3\xa0n tranh can be used either as a solo instrument, or as one of many to accompany singer/s. The \xc4\x91\xc3\xa0n tranh originally had 16 strings but it was renovated by Master Nguy\xe1\xbb\x85n V\xc4\xa9nh B\xe1\xba\xa3o (b. 1918) of South Vietnam in the mid 1950s. Since then, the 17-stringed \xc4\x91\xc3\xa0n tranh has gained massive popularity and become the most preferred form of...' +p4134 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03srw1w +p4135 +sg10 +V\u0110àn tranh +p4136 +sg12 +Vhttp://indextank.com/_static/common/demo/03srw1w.jpg +p4137 +ssg14 +(dp4138 +I0 +I0 +ssg16 +g4136 +sg17 +(dp4139 +g19 +VPlucked string instrument +p4140 +ssa(dp4141 +g2 +(dp4142 +g4 +Vhttp://freebase.com/view/en/boha +p4143 +sg6 +S'The boha (prounouced bou-heu, also known as the Cornemuse Landaise or bohaossac) is a type of bagpipe native to the Landes and Gascony regions of southwestern France.\nThis bagpipe is notable in that it bears a greater resemblance to Eastern European bagpipes, particularly the contra-chanter bagpipes of the Pannonian Plain (e.g., the Hungarian duda), than to other Western European pipes. It features both a chanter and a drone bored into a common rectangular body. Both chanter and drone...' +p4144 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/078qd7p +p4145 +sg10 +VBoha +p4146 +sg12 +Vhttp://indextank.com/_static/common/demo/078qd7p.jpg +p4147 +ssg14 +(dp4148 +I0 +I0 +ssg16 +g4146 +sg17 +(dp4149 +g19 +VBagpipes +p4150 +ssa(dp4151 +g2 +(dp4152 +g4 +Vhttp://freebase.com/view/en/balalaika +p4153 +sg6 +S'The balalaika (Russian: \xd0\xb1\xd0\xb0\xd0\xbb\xd0\xb0\xd0\xbb\xd0\xb0\xcc\x81\xd0\xb9\xd0\xba\xd0\xb0, Russian pronunciation:\xc2\xa0[b\xc9\x90l\xc9\x90\xcb\x88lajk\xc9\x99]) is a stringed musical instrument of Russian origin, with a characteristic triangular body and three strings.\nThe balalaika family of instruments includes, from the highest-pitched to the lowest, the prima balalaika, secunda balalaika, alto balalaika, bass balalaika and contrabass balalaika. All have three-sided bodies, spruce or fir tops, backs made of 3-9 wooden sections, and usually three strings. The prima balalaika...' +p4154 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bcg1_ +p4155 +sg10 +VBalalaika +p4156 +sg12 +Vhttp://indextank.com/_static/common/demo/02bcg1_.jpg +p4157 +ssg14 +(dp4158 +I0 +I1 +ssg16 +g4156 +sg17 +(dp4159 +g19 +VPlucked string instrument +p4160 +ssa(dp4161 +g2 +(dp4162 +g4 +Vhttp://freebase.com/view/en/bell +p4163 +sg6 +S'A bell is a simple sound-making device. The bell is a percussion instrument and an idiophone. Its form is usually a hollow, cup-shaped object, which resonates upon being struck. The striking implement can be a tongue suspended within the bell, known as a clapper, a small, free sphere enclosed within the body of the bell or a separate mallet or hammer.\nBells are usually made of cast metal, but small bells can also be made from ceramic or glass. Bells can be of all sizes: from tiny dress...' +p4164 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044wp6c +p4165 +sg10 +VBell +p4166 +sg12 +Vhttp://indextank.com/_static/common/demo/044wp6c.jpg +p4167 +ssg14 +(dp4168 +I0 +I3 +ssg16 +g4166 +sg17 +(dp4169 +g19 +VPercussion +p4170 +ssa(dp4171 +g2 +(dp4172 +g4 +Vhttp://freebase.com/view/en/piccolo +p4173 +sg6 +S'The piccolo (Italian for small) is a half-size flute, and a member of the woodwind family of musical instruments. The piccolo has the same fingerings as its larger sibling, the standard transverse flute, but the sound it produces is an octave higher than written. This gave rise to the name "ottavino," the name by which the instrument is referred to in the scores of Italian composers.\nPiccolos are now only manufactured in the key of C; however, they were once also available in D\xe2\x99\xad. It was for...' +p4174 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bt06d +p4175 +sg10 +VPiccolo +p4176 +sg12 +Vhttp://indextank.com/_static/common/demo/02bt06d.jpg +p4177 +ssg14 +(dp4178 +I0 +I8 +ssg16 +g4176 +sg17 +(dp4179 +g19 +VFlute (transverse) +p4180 +ssa(dp4181 +g2 +(dp4182 +g4 +Vhttp://freebase.com/view/en/geomungo +p4183 +sg6 +S'The geomungo (also spelled komungo or k\xc5\x8fmun\'go) or hyeongeum (literally "black zither", also spelled hyongum or hy\xc5\x8fn\'g\xc5\xadm) is a traditional Korean stringed musical instrument of the zither family of instruments with both bridges and frets. Scholars believe that the name refers to Goguryeo and translates to "Goguryeo zither" or that it refers to the colour and translates to "black crane zither".\nThe instrument originated circa the fourth century (see Anak Tomb No.3 infra) through the 7th...' +p4184 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041hxhx +p4185 +sg10 +VGeomungo +p4186 +sg12 +Vhttp://indextank.com/_static/common/demo/041hxhx.jpg +p4187 +ssg14 +(dp4188 +I0 +I0 +ssg16 +g4186 +sg17 +(dp4189 +g19 +VPlucked string instrument +p4190 +ssa(dp4191 +g2 +(dp4192 +g4 +Vhttp://freebase.com/view/en/euphonium +p4193 +sg6 +S'The euphonium is a conical-bore, tenor-voiced brass instrument. It derives its name from the Greek word euphonos, meaning "well-sounding" or "sweet-voiced" (eu means "well" or "good" and phonos means "of sound", so "of good sound"). The euphonium is a valved instrument; nearly all current models are piston valved, though rotary valved models do exist.\nA person who plays the euphonium is sometimes called a euphoniumist, euphophonist, or a euphonist, while British players often colloquially...' +p4194 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02bdncm +p4195 +sg10 +VEuphonium +p4196 +sg12 +Vhttp://indextank.com/_static/common/demo/02bdncm.jpg +p4197 +ssg14 +(dp4198 +I0 +I7 +ssg16 +g4196 +sg17 +(dp4199 +g19 +VBrass instrument +p4200 +ssa(dp4201 +g2 +(dp4202 +g4 +Vhttp://freebase.com/view/en/fender_jazz_bass +p4203 +sg6 +S'The Jazz Bass (or J Bass) was the second model of electric bass created by Leo Fender. The bass is distinct from the Precision Bass in that its tone is brighter and richer in the midrange and treble with less emphasis on the fundamental harmonic. Because of this, many bass players who want to be more "forward" in the mix (including smaller bands such as power trios) prefer the Jazz Bass. The sound of the Fender Jazz Bass has been fundamental in the development of signature sounds in certain...' +p4204 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f6r8s +p4205 +sg10 +VFender Jazz Bass +p4206 +sg12 +Vhttp://indextank.com/_static/common/demo/02f6r8s.jpg +p4207 +ssg14 +(dp4208 +I0 +I5 +ssg16 +g4206 +sg17 +(dp4209 +g19 +VBass guitar +p4210 +ssa(dp4211 +g2 +(dp4212 +g4 +Vhttp://freebase.com/view/en/resonator_guitar +p4213 +sg6 +S'A resonator guitar or resophonic guitar is an acoustic guitar whose sound is produced by one or more spun metal cones (resonators) instead of the wooden sound board (guitar top/face). Resonator guitars were originally designed to be louder than conventional acoustic guitars which were overwhelmed by horns and percussion instruments in dance orchestras. They became prized for their distinctive sound, however, and found life with several musical styles (most notably bluegrass and also blues)...' +p4214 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02f27b3 +p4215 +sg10 +VResonator guitar +p4216 +sg12 +Vhttp://indextank.com/_static/common/demo/02f27b3.jpg +p4217 +ssg14 +(dp4218 +I0 +I8 +ssg16 +g4216 +sg17 +(dp4219 +g19 +VGuitar +p4220 +ssa(dp4221 +g2 +(dp4222 +g4 +Vhttp://freebase.com/view/m/0dkc489 +p4223 +sg6 +S'Asaph Music has a vision to see the name of the Lord exalted through worship and people encouraged to trust in Christ. Its motto is "O God, my heart is fixed; I will sing and give praise, even with my glory," founded upon Psalm 108:1. It is our earnest desired to see Christians inspired in their Christian walk through our musical presentations. Asaph Music works in conjunction with Kingdom Builders Productions in producing musical works.' +p4224 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0dkc4d5 +p4225 +sg10 +VAsaph Music +p4226 +sg12 +Vhttp://indextank.com/_static/common/demo/0dkc4d5.jpg +p4227 +ssg14 +(dp4228 +I0 +I0 +ssg16 +g4226 +sg17 +(dp4229 +g19 +VAcoustic guitar +p4230 +ssa(dp4231 +g2 +(dp4232 +g4 +Vhttp://freebase.com/view/en/ems_vcs_3 +p4233 +sg6 +S"The VCS 3 (an initialism for Voltage Controlled Studio with 3 oscillators) is a portable analog synthesiser with a flexible semi-modular voice architecture.\nIt was created in 1969 by Peter Zinovieff's EMS company. The electronics were largely designed by David Cockerell and the machine's distinctive visual appearance was the work of electronic composer Tristram Cary. The VCS 3 was more or less the first portable commercially available synthesiser\xe2\x80\x94portable in the sense that the VCS 3 was..." +p4234 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03t04ld +p4235 +sg10 +VEMS VCS 3 +p4236 +sg12 +Vhttp://indextank.com/_static/common/demo/03t04ld.jpg +p4237 +ssg14 +(dp4238 +I0 +I21 +ssg16 +g4236 +sg17 +(dp4239 +g19 +VSynthesizer +p4240 +ssa(dp4241 +g2 +(dp4242 +g4 +Vhttp://freebase.com/view/en/mandolin +p4243 +sg6 +S"A mandolin (Italian: mandolino) is a musical instrument in the lute family (plucked, or strummed). It descends from the mandore, a soprano member of the lute family. The mandolin soundboard (the top) comes in many shapes\xe2\x80\x94but generally round or teardrop-shaped, sometimes with scrolls or other projections. A mandolin may have f-holes, or a single round or oval sound hole. A round or oval sound hole may be bordered with decorative rosettes or purfling, but usually doesn't feature an intricately..." +p4244 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03s49h2 +p4245 +sg10 +VMandolin +p4246 +sg12 +Vhttp://indextank.com/_static/common/demo/03s49h2.jpg +p4247 +ssg14 +(dp4248 +I0 +I266 +ssg16 +g4246 +sg17 +(dp4249 +g19 +VPlucked string instrument +p4250 +ssa(dp4251 +g2 +(dp4252 +g4 +Vhttp://freebase.com/view/en/pedal_steel_guitar +p4253 +sg6 +S'The pedal steel guitar is a type of electric guitar that uses a metal bar to "fret" or shorten the length of the strings, rather than fingers on strings as with a conventional guitar. Unlike other types of steel guitar, it also uses foot pedals and knee levers to affect the pitch, hence the name "pedal" steel guitar. The word "steel" in the name comes from the metal tone bar, which is called a "steel", and which acts as a moveable fret, shortening the effective length of the string or...' +p4254 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/029z7lx +p4255 +sg10 +VPedal steel guitar +p4256 +sg12 +Vhttp://indextank.com/_static/common/demo/029z7lx.jpg +p4257 +ssg14 +(dp4258 +I0 +I39 +ssg16 +g4256 +sg17 +(dp4259 +g19 +VElectric guitar +p4260 +ssa(dp4261 +g2 +(dp4262 +g4 +Vhttp://freebase.com/view/en/ruan +p4263 +sg6 +S'The ruan (\xe9\x98\xae, pinyin: ru\xc7\x8en) is a Chinese plucked string instrument. It is a lute with a fretted neck, a circular body, and four strings. Its strings were formerly made of silk but since the 20th century they have been made of steel (flatwound for the lower strings). The modern ruan has 24 frets with 12 semitones on each string, which has greatly expanded its range from a previous 13 frets. The frets are commonly made of ivory. Or in recent times, metal mounted on wood. The metal frets produce...' +p4264 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044vyw1 +p4265 +sg10 +VRuan +p4266 +sg12 +Vhttp://indextank.com/_static/common/demo/044vyw1.jpg +p4267 +ssg14 +(dp4268 +I0 +I0 +ssg16 +g4266 +sg17 +(dp4269 +g19 +VPlucked string instrument +p4270 +ssa(dp4271 +g2 +(dp4272 +g4 +Vhttp://freebase.com/view/en/banjo +p4273 +sg6 +S'The banjo is a stringed instrument with, typically, four or five strings, which vibrate a membrane of plastic material or animal hide stretched over a circular frame. Simpler forms of the instrument were fashioned by enslaved Africans in Colonial America, adapted from several African instruments of the same basic design.\nThe banjo is usually associated with country, folk, classical music, Irish traditional music and bluegrass music. Historically, the banjo occupied a central place in African...' +p4274 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/0290x6g +p4275 +sg10 +VBanjo +p4276 +sg12 +Vhttp://indextank.com/_static/common/demo/0290x6g.jpg +p4277 +ssg14 +(dp4278 +I0 +I357 +ssg16 +g4276 +sg17 +(dp4279 +g19 +VPlucked string instrument +p4280 +ssa(dp4281 +g2 +(dp4282 +g4 +Vhttp://freebase.com/view/m/0cc8zh +p4283 +sg6 +S'The bombard, also known as talabard or ar vombard in the Breton language or bombarde in French, is a popular contemporary conical bore double reed instrument widely used to play traditional Breton music. The bombard is a woodwind instrument; the reed is held between the lips. The bombard is a member of the oboe family. Describing it as an oboe, however, can be misleading since it has a broader and very powerful sound, vaguely resembling a trumpet. It is played as other oboes are played, with...' +p4284 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05l29l2 +p4285 +sg10 +VBombard +p4286 +sg12 +Vhttp://indextank.com/_static/common/demo/05l29l2.jpg +p4287 +ssg14 +(dp4288 +I0 +I0 +ssg16 +g4286 +sg17 +(dp4289 +g19 +VWoodwind instrument +p4290 +ssa(dp4291 +g2 +(dp4292 +g4 +Vhttp://freebase.com/view/en/charango +p4293 +sg6 +S'The charango is a small South American stringed instrument of the lute family, about 66 cm long, traditionally made with the shell of the back of an armadillo. Many contemporary charangos are now made with different types of wood. It typically has 10 strings in five courses of 2 strings each, though other variations exist.\nThe instrument was invented in the early 18th century in the Viceroyalty of Peru (nowadays Per\xc3\xba and Bolivia).\nWhen the Spanish conquistadores came to South America, they...' +p4294 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02btn26 +p4295 +sg10 +VCharango +p4296 +sg12 +Vhttp://indextank.com/_static/common/demo/02btn26.jpg +p4297 +ssg14 +(dp4298 +I0 +I8 +ssg16 +g4296 +sg17 +(dp4299 +g19 +VPlucked string instrument +p4300 +ssa(dp4301 +g2 +(dp4302 +g4 +Vhttp://freebase.com/view/en/wurlitzer_electric_piano +p4303 +sg6 +S'The Wurlitzer electric piano was one of a series of electromechanical stringless pianos manufactured and marketed by the Rudolph Wurlitzer Company, Corinth, Mississippi, U.S. and Tonawanda, New York. The Wurlitzer company itself never called the instrument an "electric piano", instead inventing the phrase "Electronic Piano" and using this as a trademark throughout the production of the instrument. See however electronic piano, the generally accepted term for a completely different type of...' +p4304 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/041w6pq +p4305 +sg10 +VWurlitzer electric piano +p4306 +sg12 +Vhttp://indextank.com/_static/common/demo/041w6pq.jpg +p4307 +ssg14 +(dp4308 +I0 +I2 +ssg16 +g4306 +sg17 +(dp4309 +g19 +VElectric piano +p4310 +ssa(dp4311 +g2 +(dp4312 +g4 +Vhttp://freebase.com/view/en/jazz_guitar +p4313 +sg6 +S'The term jazz guitar may refer to either a type of guitar or to the variety of guitar playing styles used in the various genres which are commonly termed "jazz". The jazz-type guitar was born as a result of using electric amplification to increase the volume of conventional acoustic guitars.\nConceived in the early 1930s, the electric guitar became a necessity as jazz musicians sought to amplify their sound. Arguably, no other musical instrument had greater influence on how music evolved...' +p4314 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/05l99wd +p4315 +sg10 +VJazz guitar +p4316 +sg12 +Vhttp://indextank.com/_static/common/demo/05l99wd.jpg +p4317 +ssg14 +(dp4318 +I0 +I2 +ssg16 +g4316 +sg17 +(dp4319 +g19 +VGuitar +p4320 +ssa(dp4321 +g2 +(dp4322 +g4 +Vhttp://freebase.com/view/en/wagner_tuba +p4323 +sg6 +S'The Wagner tuba is a comparatively rare brass instrument that combines elements of both the French horn and the tuba. Also referred to as the "Bayreuth Tuba", it was originally created for Richard Wagner\'s operatic cycle Der Ring des Nibelungen. Since then, other composers have written for it, most notably Anton Bruckner, in whose Symphony No. 7 a quartet of them is first heard in the slow movement in memory of Wagner. The euphonium is sometimes used as a substitute when a Wagner tuba cannot...' +p4324 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02c8561 +p4325 +sg10 +VWagner tuba +p4326 +sg12 +Vhttp://indextank.com/_static/common/demo/02c8561.jpg +p4327 +ssg14 +(dp4328 +I0 +I1 +ssg16 +g4326 +sg17 +(dp4329 +g19 +VTuba +p4330 +ssa(dp4331 +g2 +(dp4332 +g4 +Vhttp://freebase.com/view/en/sopranino_saxophone +p4333 +sg6 +S'The sopranino saxophone is one of the smallest members of the saxophone family. A sopranino saxophone is tuned in the key of E\xe2\x99\xad, and sounds an octave above the alto saxophone. This saxophone has a sweet sound and although the sopranino is one of the least common of the saxophones in regular use today, it is still being produced by several of the major musical manufacturing companies. Due to their small size, sopraninos are not usually curved like other saxes. Orsi, however, does make curved...' +p4334 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02dl_1s +p4335 +sg10 +VSopranino saxophone +p4336 +sg12 +Vhttp://indextank.com/_static/common/demo/02dl_1s.jpg +p4337 +ssg14 +(dp4338 +I0 +I1 +ssg16 +g4336 +sg17 +(dp4339 +g19 +VSaxophone +p4340 +ssa(dp4341 +g2 +(dp4342 +g4 +Vhttp://freebase.com/view/en/baroque_guitar +p4343 +sg6 +S'The Baroque guitar is a guitar from the baroque era (c. 1600\xe2\x80\x931750), an ancestor of the modern classical guitar. The term is also used for modern instruments made in the same style.\nThe instrument was smaller than a modern guitar, of lighter construction, and had gut strings. The frets were also usually made of gut, and tied around the neck. A typical instrument had five courses, each consisting of two separate strings although the first (highest sounding) course was often a single string,...' +p4344 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03t4sks +p4345 +sg10 +VBaroque guitar +p4346 +sg12 +Vhttp://indextank.com/_static/common/demo/03t4sks.jpg +p4347 +ssg14 +(dp4348 +I0 +I1 +ssg16 +g4346 +sg17 +(dp4349 +g19 +VPlucked string instrument +p4350 +ssa(dp4351 +g2 +(dp4352 +g4 +Vhttp://freebase.com/view/en/contra-alto_clarinet +p4353 +sg6 +S'The contra-alto clarinet is a large, low-sounding musical instrument of the clarinet family. The modern contra-alto clarinet is pitched in the key of EEb and is sometimes incorrectly referred to as the EEb contrabass clarinet. The unhyphenated form "contra alto clarinet" is also sometimes used, as is "contralto clarinet", but the latter is confusing since the instrument\'s range is much lower than the contralto vocal range; the more correct term "contra-alto" is meant to convey, by analogy...' +p4354 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02b9drl +p4355 +sg10 +VContra-alto clarinet +p4356 +sg12 +Vhttp://indextank.com/_static/common/demo/02b9drl.jpg +p4357 +ssg14 +(dp4358 +I0 +I1 +ssg16 +g4356 +sg17 +(dp4359 +g19 +VClarinet +p4360 +ssa(dp4361 +g2 +(dp4362 +g4 +Vhttp://freebase.com/view/en/vihuela +p4363 +sg6 +S'Vihuela is a name given to two different guitar-like string instruments: one from 15th and 16th century Spain, usually with 12 paired strings, and the other, the Mexican vihuela, from the 19th century Mexico with five strings and typically played in Mariachi bands.\nThe vihuela, as it was known in Spain, was called the viola da mano in Italy and Portugal. The two names are functionally synonymous and interchangeable. In its most developed form, the vihuela was a guitar-like instrument with...' +p4364 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/02csx2d +p4365 +sg10 +VVihuela +p4366 +sg12 +Vhttp://indextank.com/_static/common/demo/02csx2d.jpg +p4367 +ssg14 +(dp4368 +I0 +I7 +ssg16 +g4366 +sg17 +(dp4369 +g19 +VPlucked string instrument +p4370 +ssa(dp4371 +g2 +(dp4372 +g4 +Vhttp://freebase.com/view/en/baritone_saxophone +p4373 +sg6 +S'The baritone saxophone, often called "bari sax" (to avoid confusion with the baritone horn, which is often referred to simply as "baritone"), is one of the larger and lower pitched members of the saxophone family. It was invented by Adolphe Sax. The baritone is distinguished from smaller sizes of saxophone by the extra loop near its mouthpiece; this helps to keep the instrument at a practical height (the rarer bass saxophone has a similar, but larger loop). It is the lowest pitched saxophone...' +p4374 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/044mtf9 +p4375 +sg10 +VBaritone saxophone +p4376 +sg12 +Vhttp://indextank.com/_static/common/demo/044mtf9.jpg +p4377 +ssg14 +(dp4378 +I0 +I29 +ssg16 +g4376 +sg17 +(dp4379 +g19 +VSaxophone +p4380 +ssa(dp4381 +g2 +(dp4382 +g4 +Vhttp://freebase.com/view/en/portative_organ +p4383 +sg6 +S'A portative organ (portatif organ, portativ organ, or simply portative, portatif, or portativ) (from the Latin verb portare, "to carry") is a small pipe organ that consists of one rank of flue pipes and played while strapped to the performer at a right angle. The performer manipulates the bellows with one hand and fingers the keys with the other. The portative organ lacks a reservoir to retain a supply of wind, thus it will only produce sound while the bellows are being operated. The...' +p4384 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/042tms2 +p4385 +sg10 +VPortative organ +p4386 +sg12 +Vhttp://indextank.com/_static/common/demo/042tms2.jpg +p4387 +ssg14 +(dp4388 +I0 +I0 +ssg16 +g4386 +sg17 +(dp4389 +g19 +VPipe organ +p4390 +ssa(dp4391 +g2 +(dp4392 +g4 +Vhttp://freebase.com/view/en/electric_mandolin +p4393 +sg6 +S'The electric mandolin is an instrument tuned and played as the mandolin and amplified in similar fashion to an electric guitar. As with electric guitars, electric mandolins take many forms:\nElectric mandolins were built in the United States as early as the late 1920s. Among the first companies to produce them were Stromberg-Voisinet, Electro (which later became Rickenbacker), ViViTone, and National Reso-Phonic. Gibson and Vega introduced their electric mandolins in 1936.\nIn the United...' +p4394 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03sgsdn +p4395 +sg10 +VElectric mandolin +p4396 +sg12 +Vhttp://indextank.com/_static/common/demo/03sgsdn.jpg +p4397 +ssg14 +(dp4398 +I0 +I1 +ssg16 +g4396 +sg17 +(dp4399 +g19 +VMandolin +p4400 +ssa(dp4401 +g2 +(dp4402 +g4 +Vhttp://freebase.com/view/en/yazheng +p4403 +sg6 +S'The yazheng (simplified: \xe8\xbd\xa7\xe7\xad\x9d; traditional: \xe8\xbb\x8b\xe7\xae\x8f; pinyin: y\xc3\xa0zh\xc4\x93ng; also spelled ya zheng or ya cheng) is a Chinese string instrument. It is a long zither similar to the guzheng but bowed by scraping with a sorghum stem dusted with resin, a bamboo stick, or a piece of forsythia wood. The musical instrument was popular in the Tang Dynasty, but is today little used except in the folk music of some parts of northern China, where it is called yaqin (simplified: \xe8\xbd\xa7\xe7\x90\xb4; traditional: \xe8\xbb\x8b\xe7\x90\xb4).\nThe Korean ajaeng...' +p4404 +sg8 +Vhttp://img.freebase.com/api/trans/raw/m/03spb3n +p4405 +sg10 +VYazheng +p4406 +sg12 +Vhttp://indextank.com/_static/common/demo/03spb3n.jpg +p4407 +ssg14 +(dp4408 +I0 +I0 +ssg16 +g4406 +sg17 +(dp4409 +g19 +VBowed string instruments +p4410 +ssa. \ No newline at end of file diff --git a/nebu/.project b/nebu/.project new file mode 100644 index 0000000..2ba6302 --- /dev/null +++ b/nebu/.project @@ -0,0 +1,18 @@ + + + nebu + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + org.python.pydev.django.djangoNature + + diff --git a/nebu/.pydevproject b/nebu/.pydevproject new file mode 100644 index 0000000..1695c95 --- /dev/null +++ b/nebu/.pydevproject @@ -0,0 +1,17 @@ + + + + +Default +python 2.6 + +DJANGO_MANAGE_LOCATION +nebu/manage.py + + +/nebu + + +../ + + diff --git a/nebu/__init__.py b/nebu/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nebu/amazon_credential.py b/nebu/amazon_credential.py new file mode 100644 index 0000000..c2910f5 --- /dev/null +++ b/nebu/amazon_credential.py @@ -0,0 +1,2 @@ +AMAZON_USER = "" +AMAZON_PASSWORD = "" diff --git a/nebu/api_linked_models.py b/nebu/api_linked_models.py new file mode 100644 index 0000000..0908f60 --- /dev/null +++ b/nebu/api_linked_models.py @@ -0,0 +1,938 @@ +import hashlib +import random +import binascii + +from lib.indextank.client import ApiClient, IndexAlreadyExists +from lib.authorizenet import AuthorizeNet, BillingException + +from django.db import models +from django.contrib.auth.models import User +from django.utils import simplejson as json +from django.db import IntegrityError +from django.db.models.aggregates import Sum, Count + +from lib import encoder, flaptor_logging + +from django.conf import settings +from datetime import datetime + +logger = flaptor_logging.get_logger('Models') + +# idea taken from https://www.grc.com/passwords.htm +def generate_apikey(id): + key = "2A1A8AE7CAEFAC47D6F74920CE4B0CE46430CDA6CF03D254C1C29402D727E570" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:14] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return unique_part + '-' + random_part + +def generate_onetimepass(id): + key = "CAEFAC47D6F7D727E57024920CE4B0CE46430CDA6CF03D254C1C29402A1A8AE7" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:5] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return unique_part + random_part + +def generate_forgotpass(id): + key = "E57024920CE4B0CE4643CAEFAC47D6F7D7270CDA6CF03D254C1C29402A1A8AE7" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:6] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return random_part + unique_part + + +# StoreFront models +class Account(models.Model): + apikey = models.CharField(max_length=22, unique=True) + creation_time = models.DateTimeField() + package = models.ForeignKey('Package', null=True) + status = models.CharField(max_length=30, null=False) + provisioner = models.ForeignKey('Provisioner', null=True) + + configuration = models.ForeignKey('IndexConfiguration', null=True) + default_analyzer = models.ForeignKey('Analyzer', null=True, related_name="accounts") + + class Statuses: + operational = 'OPERATIONAL' + creating = 'CREATING' + closed = 'CLOSED' + + def __repr__(self): + return 'Account (%s):\n\tuser_email: %s\n\tapikey: %s\n\tcreation_time: %s\n\tstatus: %s\n\tpackage: %s\n\tconfiguration: %s\n' % (self.id, PFUser.objects.filter(account=self)[0].email, str(self.apikey), str(self.creation_time), str(self.status), self.package.name, self.configuration.description) + + def __str__(self): + return '(apikey: %s; creation_time: %s; status: %s)' % (str(self.apikey), str(self.creation_time), str(self.status)) + + def count_indexes(self): + return self.indexes.aggregate(cnt=Count('id'))['cnt'] + + def count_documents(self): + return self.indexes.aggregate(cnt=Sum('current_docs_number'))['cnt'] + + def is_operational(self): + return self.status == Account.Statuses.operational + + def is_heroku(self): + # HACK UNTIL HEROKU IS A PROVISIONER + return self.package.code.startswith('HEROKU_') + #return self.provisioner and self.provisioner.name == 'heroku' + + @classmethod + def create_account(cls, dt, email=None, password=None): + account = Account() + + account.creation_time = datetime.now() + account.status = Account.Statuses.creating + account.save() + + account.apikey = generate_apikey(account.id) + account.save() + + unique_part, random_part = account.apikey.split('-', 1) + if email is None: + email = '%s@indextank.com' % unique_part + + if password is None: + password = random_part + + try: + user = User.objects.create_user(email, '', password) + except IntegrityError, e: + account.delete() + raise e + + try: + pfu = PFUser() + pfu.account = account + pfu.user = user + pfu.email = email + + pfu.save() + except IntegrityError, e: + account.delete() + user.delete() + raise e + + return account, pfu + + def create_index(self, index_name, public_search=None): + index = Index() + + # basic index data + index.populate_for_account(self) + index.name = index_name + index.creation_time = datetime.now() + index.language_code = 'en' + index.status = Index.States.new + if not public_search is None: + index.public_api = public_search + + # test for name uniqueness + # raises IntegrityError if the index name already exists + index.save() + + # define the default function + function = ScoreFunction() + function.index = index + function.name = '0' + function.definition = '-age' + function.save() + + # deduce code from id + index.code = encoder.to_key(index.id) + index.save() + + return index + + def create_demo_index(self): + try: + dataset = DataSet.objects.get(code='DEMO') + except DataSet.DoesNotExist: + logger.exception('DemoIndex dataset not present in database. Aborting demo index creation') + return + + index = self.create_index('DemoIndex') + + index.public_api = True + index.save() + + population = IndexPopulation() + population.index = index + population.status = IndexPopulation.Statuses.created + population.dataset = dataset + population.time = datetime.now() + population.populated_size = 0 + + population.save() + + def close(self): + # Dropping an account implies: + + # - removing the payment information from the account + # - removing the subscriptions from authorize.net + for info in self.payment_informations.all(): + auth = AuthorizeNet() + for subscription in info.subscriptions.all(): + auth.subscription_cancel(subscription.reference_id, subscription.subscription_id) + subscription.delete() + info.delete() + + + # - changing the status to CLOSED + self.status = Account.Statuses.closed + + # - removing and stopping the indexes for the account + for index in self.indexes.all(): + self.drop_index(index) + + # - notify + # send_notification(//close account) + + # - FIXME: handle authorize net errors! + + + self.save() + + def drop_index(self, index): + client = ApiClient(self.get_private_apiurl()) + client.delete_index(index.name) + + def apply_package(self, package): + self.package = package + + self.configuration = package.configuration + + def update_apikey(self): + self.apikey = generate_apikey(self.id) + + def get_private_apikey(self): + return self.apikey.split('-', 1)[1] + + def get_public_apikey(self): + return self.apikey.split('-', 1)[0] + + def get_private_apiurl(self): + return 'http://:%s@%s.api.indextank.com' % (self.get_private_apikey(), self.get_public_apikey()) + + def get_public_apiurl(self): + return 'http://%s.api.indextank.com' % self.get_public_apikey() + + class Meta: + db_table = 'storefront_account' + +class AccountPayingInformation(models.Model): + account = models.ForeignKey('Account', related_name='payment_informations') + + first_name = models.CharField(max_length=50, null=True) + last_name = models.CharField(max_length=50, null=True) + address = models.CharField(max_length=60, null=True) + city = models.CharField(max_length=60, null=True) + state = models.CharField(max_length=2, null=True) + zip_code = models.CharField(max_length=60, null=True) + country = models.CharField(max_length=2, null=True) + + company = models.CharField(max_length=50, null=True) + + credit_card_last_digits = models.CharField(max_length=4, null=True) + contact_email = models.EmailField(max_length=255, null=True) + + #custom subscription + monthly_amount = models.DecimalField(max_digits=8, decimal_places=2, null=True) + subscription_status = models.CharField(max_length=30, null=True) + subscription_type = models.CharField(max_length=30, null=True) + + + class Meta: + db_table = 'storefront_accountpayinginformation' + + +class PaymentSubscription(models.Model): + account = models.ForeignKey('AccountPayingInformation', related_name='subscriptions') + + # authorizenet id + subscription_id = models.CharField(max_length=20, null=False, blank=False) + # indextank id + reference_id = models.CharField(max_length=13, null=False, blank=False) + + amount = models.DecimalField(max_digits=8, decimal_places=2, null=False) + + # Frequency + start_date = models.DateTimeField() + frequency_length = models.IntegerField(null=False) + frequency_unit = models.CharField(max_length=10, null=False, blank=False) + + class Meta: + db_table = 'storefront_paymentsubscription' + + +class EffectivePayment(models.Model): + account = models.ForeignKey('AccountPayingInformation', related_name='payments') + + transaction_date = models.DateTimeField() + + # authorizenet data + transaction_id = models.CharField(max_length=12, null=False, blank=False) + customer_id = models.CharField(max_length=8, null=False, blank=False) + transaction_message = models.CharField(max_length=300, null=True) + subscription_id = models.CharField(max_length=20, null=False, blank=False) + subscription_payment_number = models.IntegerField(null=False) + first_name = models.CharField(max_length=50, null=False, blank=False) + last_name = models.CharField(max_length=50, null=False, blank=False) + address = models.CharField(max_length=60, null=True) + city = models.CharField(max_length=60, null=True) + state = models.CharField(max_length=2, null=True) + zip_code = models.CharField(max_length=60, null=True) + country = models.CharField(max_length=2, null=True) + company = models.CharField(max_length=50, null=True) + + # Inherited data (from account information + credit_card_last_digits = models.CharField(max_length=4, null=False, blank=False) + contact_email = models.EmailField(max_length=255) + + amount = models.DecimalField(max_digits=8, decimal_places=2, null=False) + + class Meta: + db_table = 'storefront_effectivepayment' + +class DataSet(models.Model): + name = models.CharField(null=True, max_length=50, unique=True) + code = models.CharField(null=True, max_length=15, unique=True) + filename = models.CharField(null=True, max_length=100, unique=True) + size = models.IntegerField(default=0) + + class Meta: + db_table = 'storefront_dataset' + +class IndexPopulation(models.Model): + index = models.ForeignKey('Index', related_name='datasets') + dataset = models.ForeignKey('DataSet', related_name='indexes') + time = models.DateTimeField() + populated_size = models.IntegerField(default=0) + + status = models.CharField(max_length=50,null=True) + + class Statuses: + created = 'CREATED' + populating = 'POPULATING' + finished = 'FINISHED' + + class Meta: + db_table = 'storefront_indexpopulation' + + +class Index(models.Model): + account = models.ForeignKey('Account', related_name='indexes') + code = models.CharField(null=True, max_length=22, unique=True) + name = models.CharField(max_length=50) + language_code = models.CharField(max_length=2) + creation_time = models.DateTimeField() + + analyzer_config = models.TextField(null=True) + configuration = models.ForeignKey('IndexConfiguration', null=True) + public_api = models.BooleanField(default=False, null=False) + + status = models.CharField(max_length=50) + + deleted = models.BooleanField(default=False, null=False) + + class States: + new = 'NEW' + live = 'LIVE' + hibernate_requested = 'HIBERNATE_REQUESTED' + hibernated = 'HIBERNATED' + waking_up = 'WAKING_UP' + + def get_json_for_analyzer(self): + if self.analyzer_config is None: + return None + configuration = json.loads(self.analyzer_config) + final_configuration = {} + + if configuration.has_key('per_field'): + per_field_final = {} + per_field = configuration.get('per_field') + for field in per_field.keys(): + per_field_final[field] = Index.get_analyzer(per_field[field]) + final_configuration['perField'] = per_field_final + final_configuration['default'] = Index.get_analyzer(per_field.get('default')) + else: + final_configuration = Index.get_analyzer(configuration) + + return final_configuration + + @classmethod + def get_analyzer(cls, configuration): + analyzer_map = {} + code = configuration.get('code') + if code is None: + raise ValueError('Analyzer configuration has no "code" key') + + try: + analyzer = AnalyzerComponent.objects.get(code=code) + except AnalyzerComponent.DoesNotExist: + raise ValueError('Analyzer configuration "code" key doesn\'t match any analyzers') + + analyzer_map['factory'] = analyzer.factory + analyzer_map['configuration'] = json.loads(analyzer.config) + + if configuration.has_key('filters'): + filters_list = [] + for filter in configuration.get('filters'): + filters_list.append(Index.get_analyzer(filter)) + analyzer_map['configuration']['filters'] = filters_list + + return analyzer_map + +# allows_adds = models.BooleanField(null=False,default=True) +# allows_queries = models.BooleanField(null=False,default=True) + + # index creation data +# allows_snippets = models.BooleanField() +# +# allows_autocomplete = models.BooleanField(default=True) +# autocomplete_type = models.models.CharField(max_length=10, null=True) # NEW +# +# allows_faceting = models.BooleanField() +# facets_bits = models.IntegerField(null=True) # NEW +# +# max_variables = models.IntegerField(null=False) # NEW +# +# max_memory_mb = models.IntegerField(null=False) # NEW +# rti_documents_number = models.IntegerField(null=False) # NEW + + # statistics + current_size = models.FloatField(default=0) + current_docs_number = models.IntegerField(default=0) + queries_per_day = models.FloatField(default=0) + + #demo + base_port = models.IntegerField(null=True) + + def __repr__(self): + return 'Index (%s):\n\tname: %s\n\tcode: %s\n\tcreation_time: %s\n\tconfiguration: %s\n\taccount\'s package: %s\ncurrent deploys: %r' % (self.id, self.name, self.code, self.creation_time, self.configuration.description, self.account.package.name, self.deploys.all()) + + def is_populating(self): + for population in self.datasets.all(): + if not population.status == IndexPopulation.Statuses.finished: + return True + return False + + def is_demo(self): + return self.name == 'DemoIndex' and self.datasets.count() > 0 + + + def is_ready(self): + ''' + Returns True if the end-user can use the index. + (this means for read and write, and it's meant to + be shown in the storefront page). Internally, this + means that at least one deployment for this index + is readable, and at least one is writable. + ''' + return self.is_writable() and self.is_readable() + + def is_hibernated(self): + return self.status in (Index.States.hibernated, Index.States.waking_up) + + def is_writable(self): + ''' + Returns true if there's at least one index that can be written. + ''' + for deploy in self.deploys.all(): + if deploy.is_writable(): + return True + + def is_readable(self): + ''' + Returns true if there's at least one index that can be read. + ''' + for deploy in self.deploys.all(): + if deploy.is_readable(): + return True + + def populate_for_account(self, account): + self.account = account + self.configuration = account.configuration + if account.default_analyzer is not None: + self.analyzer_config = account.default_analyzer.configuration + + def searchable_deploy(self): + '''Returns a single deploy that can be used to search. If no deploy is searcheable + it returns None. Note that if more than one deploy is searcheable, there are no warranties + of wich one will be returned.''' + # TODO : should change once deploy roles are implemented + ds = self.deploys.all() + ds = [d for d in ds if d.is_readable()] + return ds[0] if ds else None + + def indexable_deploys(self): + '''Returns the list of all deploys that should be updated (adds/updates/deletes/etc)''' + # TODO : should change once deploy roles are implemented + ds = self.deploys.all() + ds = [d for d in ds if d.is_writable()] + return ds + + def get_functions_dict(self): + return dict((str(f.name), f.definition) for f in self.scorefunctions.all()) + + def get_debug_info(self): + info = 'Index: %s [%s]\n' % (self.name, self.code) +\ + 'Account: %s\n' % self.account.user.email +\ + 'Deploys:\n' + for d in self.deploys.all(): + info += ' [deploy:%d] %s on [worker:%s] %s:%s' % (d.id, d.status, d.worker.id, d.worker.wan_dns, d.base_port) + return info + + def update_status(self, new_status): + print 'updating status %s for %r' % (new_status, self) + logger.debug('Updating status to %s for idnex %r', new_status, self) + Index.objects.filter(id=self.id).update(status=new_status) + + def mark_deleted(self): + Index.objects.filter(id=self.id).update(deleted=True) + + class AutocompleTypes: + created = 'DOCUMENTS' + initializing = 'QUERIES' + + class Meta: + unique_together = (('code','account'),('name','account')) + db_table = 'storefront_index' + +class Insight(models.Model): + index = models.ForeignKey(Index, related_name='insights') + code = models.CharField(max_length=30, null=False) + data = models.TextField(null=False) + last_update = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = ('index', 'code') + db_table = 'storefront_insight' + +class IndexConfiguration(models.Model): + description = models.TextField(null=False) + creation_date = models.DateField() + json_configuration = models.TextField(null=False) + + def __repr__(self): + j_map = json.loads(self.json_configuration) + mapStr = '{\n' + for m in j_map: + mapStr += '\t\t%s -> %s\n' % (m, j_map[m]) + mapStr += '\t}\n' + return 'IndexConfiguration (%s):\n\tdescription: %s\n\tcreation_date: %s\n\tjson_configuration: %s\n' % (self.id, self.description, str(self.creation_date), mapStr) + + def __str__(self): + return '(description: %s; creation_date: %s; json_configuration: %s)' % (self.description, str(self.creation_date), self.json_configuration) + + def get_data(self): + map = json.loads(self.json_configuration) + data = {} + for k,v in map.items(): + data[str(k)] = v + data['ram'] = data.get('xmx',0) + data.get('bdb_cache',0) + return data + def set_data(self, data): + self.json_configuration = json.dumps(data) + + class Meta: + db_table = 'storefront_indexconfiguration' + +class Analyzer(models.Model): + account = models.ForeignKey('Account', related_name='analyzers') + code = models.CharField(max_length=64) + configuration = models.TextField() + + class Meta: + db_table = 'storefront_analyzer' + +class AnalyzerComponent(models.Model): + code = models.CharField(max_length=15, unique=True) + name = models.CharField(max_length=200) + description = models.CharField(max_length=1000) + config = models.TextField(null=False,blank=False) + factory = models.CharField(max_length=200) + type = models.CharField(max_length=20) + enabled = models.BooleanField() + + class Types: + tokenizer = 'TOKENIZER' + filter = 'FILTER' + + class Meta: + db_table = 'storefront_analyzercomponent' + +def create_analyzer(code, name, config, factory, type, enabled): + analyzer = None + try: + analyzer = AnalyzerComponent.objects.get(code=code) + + analyzer.name = name + analyzer.config = config + analyzer.factory = factory + analyzer.type = type + analyzer.enabled = enabled + + analyzer.save() + except AnalyzerComponent.DoesNotExist: + analyzer = AnalyzerComponent(code=code, name=name, config=config, type=type, enabled=enabled) + analyzer.save() + +class Package(models.Model): + ''' + Packages define what a user have the right to when creating an Account and how does the indexes in that Account + behave. + There are two sections for what the Package configures. A fixed section with the control and limits information + that is used by nebu, storefront or api (base_price, index_max_size, searches_per_day, max_indexes). A dynamic + section that is handled by the IndexConfiguration object. The information of that section is passed to the IndexEngine + as it is and handled by it. + ''' + name = models.CharField(max_length=50) + code = models.CharField(max_length=30) + base_price = models.FloatField() + index_max_size = models.IntegerField() + searches_per_day = models.IntegerField() + max_indexes = models.IntegerField() + + configuration = models.ForeignKey('IndexConfiguration', null=True) + + def __repr__(self): + return 'Package (%s):\n\tname: %s\n\tcode: %s\n\tbase_price: %.2f\n\tindex_max_size: %i\n\tsearches_per_day: %i\n\tmax_indexes: %i\n' % (self.id, self.name, self.code, self.base_price, self.index_max_size, self.searches_per_day, self.max_indexes) + + def __str__(self): + return '(name: %s; code: %s; base_price: %.2f; index_max_size: %i; searches_per_day: %i; max_indexes: %i)' % (self.name, self.code, self.base_price, self.index_max_size, self.searches_per_day, self.max_indexes) + + def max_size_mb(self): + return self.index_max_size * settings.INDEX_SIZE_RATIO + class Meta: + db_table = 'storefront_package' + +class ScoreFunction(models.Model): + index = models.ForeignKey(Index, related_name='scorefunctions') + name = models.IntegerField(null=False) # TODO the java API expects an int. But a String may be nicer for name. + definition = models.CharField(max_length=255, blank=False, null=True) + + class Meta: + db_table = 'storefront_scorefunction' + unique_together = (('index','name')) + + +def create_configuration(description, data, creation_date=None): + configuration = IndexConfiguration() + configuration.description = description + configuration.creation_date = creation_date or datetime.now() + configuration.json_configuration = json.dumps(data) + + configuration.save() + return configuration + +def create_package(code, name, base_price, index_max_size, searches_per_day, max_indexes, configuration_map): +# The configuration_map will only be considered if the package if new or if it didn't already have a configuration + + package = None + try: + package = Package.objects.get(code=code) + + package.name = name + package.base_price = base_price + package.index_max_size = index_max_size + package.searches_per_day = searches_per_day + package.max_indexes = max_indexes + + if not package.configuration: + package.configuration = create_configuration('package:' + code, configuration_map) + + package.save() + except Package.DoesNotExist: + configuration = create_configuration('package:' + code, configuration_map) + package = Package(code=code, base_price=base_price, index_max_size=index_max_size, searches_per_day=searches_per_day, max_indexes=max_indexes, configuration=configuration) + package.save() + +def create_provisioner(name, token, email, plans): + provisioner = None + try: + provisioner = Provisioner.objects.get(name=name) + except Provisioner.DoesNotExist: + provisioner = Provisioner() + provisioner.name = name + provisioner.token = token + provisioner.email = email + provisioner.save() + + provisioner.plans.all().delete() + for plan, code in plans.items(): + pp = ProvisionerPlan() + pp.plan = plan + pp.provisioner = provisioner + pp.package = Package.objects.get(code=code) + pp.save() + + +class AccountMovement(models.Model): + account = models.ForeignKey('Account', related_name='movements') + class Meta: + db_table = 'storefront_accountmovement' + +class ActionLog(models.Model): + account = models.ForeignKey('Account', related_name='actions') + class Meta: + db_table = 'storefront_actionlog' + +class PFUser(models.Model): + user = models.ForeignKey(User, unique=True) + account = models.OneToOneField('Account', related_name='user') + email = models.EmailField(unique=True, max_length=255) + change_password = models.BooleanField(default=False, null=False) + class Meta: + db_table = 'storefront_pfuser' + + + +MAX_USABLE_RAM_PERCENTAGE = 0.9 +# Nebulyzer stuff +class Worker(models.Model): + ''' + Describes an amazon ec2 instance. + ''' + instance_name = models.CharField(max_length=50,null=False,blank=False) + lan_dns = models.CharField(max_length=100,null=False,blank=False) + wan_dns = models.CharField(max_length=100,null=False,blank=False) + status = models.CharField(max_length=30) + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) + #physical memory in MegaBytes + ram = models.IntegerField() + + class States: + created = 'CREATED' + initializing = 'INITIALIZING' + updating = 'UPDATING' + controllable = 'CONTROLLABLE' + decommissioning = 'DECOMMISSIONING' + dying = 'DYING' + dead = 'DEAD' + + class Meta: + db_table = 'storefront_worker' + + def get_usable_ram(self): + '''Return the amount of ram that can be used in this machine for + indexengines. It's calculated as a fixed percentage of the physical + ram. Value returned in MegaBytes''' + return MAX_USABLE_RAM_PERCENTAGE * self.ram + + def get_used_ram(self): + xmx = self.deploys.aggregate(xmx=Sum('effective_xmx'))['xmx'] + bdb = self.deploys.aggregate(bdb=Sum('effective_bdb'))['bdb'] + if xmx == None: + xmx = 0 + if bdb == None: + bdb = 0 + return xmx + bdb + + def is_assignable(self): + return self.status != Worker.States.decommissioning + + def is_ready(self): + return self.status in [Worker.States.controllable, Worker.States.decommissioning] + + def __repr__(self): + return 'Worker (%s):\n\tinstance_name: %s\n\tlan_dns: %s\n\twan_dns: %s\n\tstatus: %s\n\ttimestamp: %s\n\tram: %s\n' %(self.pk, self.instance_name, self.lan_dns, self.wan_dns, self.status, self.timestamp, self.ram) + +class Service(models.Model): + name = models.CharField(max_length=50,null=False,blank=False) + type = models.CharField(max_length=50,null=True,blank=True ) + host = models.CharField(max_length=100,null=False,blank=False) + port = models.IntegerField() + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) + + class Meta: + db_table = 'storefront_service' + + def __repr__(self): + return 'Service (%s):\n\tname: %s\n\ttype: %s\n\thost: %s\n\tport: %s\n\ttimestamp: %s\n' % (self.pk, self.name, self.type, self.host, self.port, self.timestamp) + + +# CPU Stats +class WorkerMountInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="disk_infos") + timestamp = models.DateTimeField() + + mount = models.CharField(max_length=100,null=False,blank=False) + available = models.IntegerField() + used = models.IntegerField() + + class Meta: + db_table = 'storefront_workermountinfo' + + +class WorkerLoadInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="load_infos") + timestamp = models.DateTimeField() + + load_average = models.FloatField() + + class Meta: + db_table = 'storefront_workerloadinfo' + +class WorkerIndexInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="indexes_infos") + timestamp = models.DateTimeField() + + deploy = models.ForeignKey('Deploy', related_name="index_infos") + used_disk = models.IntegerField() + used_mem = models.IntegerField() + + class Meta: + db_table = 'storefront_workerindexinfo' + + +class Deploy(models.Model): + ''' + Describes a deploy of an index on a worker, and it's status. + The idea is that an index can be moving from one worker to another, + so queries and indexing requests have to be mapped to one or more + index engines. + ''' + index = models.ForeignKey(Index, related_name="deploys") + worker = models.ForeignKey(Worker, related_name="deploys") + base_port = models.IntegerField() + status = models.CharField(max_length=30) + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) # Last time we updated this deploy. + parent = models.ForeignKey('self', related_name='children', null=True) # For moving deploys. + effective_xmx = models.IntegerField() + effective_bdb = models.IntegerField() + dying = models.BooleanField(default=False, null=False) + + # TODO add role fields + #searching_role = models.BooleanField() + #indexing_role = models.BooleanField() + + def __repr__(self): + return 'Deploy (%s):\n\tparent deploy: %s\n\tindex code: %s\n\tstatus: %s\n\tworker ip: %s\n\tport: %d\n\teffective_xmx: %d\n\teffective_bdb: %d\n' % (self.id, self.parent_id, self.index.code, self.status, self.worker.lan_dns, self.base_port, self.effective_xmx, self.effective_bdb) + + def __unicode__(self): + return "Deploy: %s on %s:%d" % (self.status, self.worker.lan_dns, self.base_port) + + def is_readable(self): + '''Returns true if a search can be performed on this deployment, and + the returned data is up to date''' + return self.status == Deploy.States.controllable or \ + self.status == Deploy.States.move_requested or \ + self.status == Deploy.States.moving or \ + (self.status == Deploy.States.recovering and not self.parent) + + def is_writable(self): + '''Returns True if new data has to be written to this deployment.''' + return self.status == Deploy.States.controllable or \ + self.status == Deploy.States.recovering or \ + self.status == Deploy.States.move_requested or \ + self.status == Deploy.States.moving + + def total_ram(self): + return self.effective_xmx + self.effective_bdb + + def update_status(self, new_status): + print 'updating status %s for %r' % (new_status, self) + logger.debug('Updating status to %s for deploy %r', new_status, self) + Deploy.objects.filter(id=self.id).update(status=new_status, timestamp=datetime.now()) + + def update_parent(self, new_parent): + logger.debug('Updating parent to %s for deploy %r', new_parent, self) + Deploy.objects.filter(id=self.id).update(parent=new_parent) + + class States: + created = 'CREATED' + initializing = 'INITIALIZING' + recovering = 'RECOVERING' + resurrecting = 'RESURRECTING' + controllable = 'CONTROLLABLE' + move_requested = 'MOVE_REQUESTED' + moving = 'MOVING' + decommissioning = 'DECOMMISSIONING' + + class Meta: + db_table = 'storefront_deploy' + +class BetaTestRequest(models.Model): + email = models.EmailField(unique=True, max_length=255) + site_url = models.CharField(max_length=200,null=False,blank=False) + summary = models.TextField(null=False,blank=False) + + request_date = models.DateTimeField(default=datetime.now) + status = models.CharField(max_length=50,null=True) + + class Meta: + db_table = 'storefront_betatestrequest' + +class BetaInvitation(models.Model): + password = models.CharField(max_length=20, null=True) + account = models.ForeignKey('Account', null=True) + assigned_customer = models.CharField(max_length=50, null=True) + beta_requester = models.ForeignKey('BetaTestRequest', null=True, related_name="invitation") + + invitation_date = models.DateTimeField(default=datetime.now) + forced_package = models.ForeignKey('Package', null=False) + + class Meta: + db_table = 'storefront_signupotp' + +class ContactInfo(models.Model): + name = models.CharField(max_length=64) + email = models.EmailField(unique=True, max_length=255) + request_date = models.DateTimeField(default=datetime.now) + source = models.CharField(max_length=64, null=True) + + class Meta: + db_table = 'storefront_contactinfo' + + + +class Provisioner(models.Model): + name = models.CharField(max_length=64) + token = models.CharField(max_length=64, null=False, blank=False) + email = models.EmailField(max_length=255) # contact info for the provisioner + + class Meta: + db_table = "storefront_provisioner" + +class ProvisionerPlan(models.Model): + plan = models.CharField(max_length=50) + provisioner = models.ForeignKey('Provisioner', related_name='plans') + package = models.ForeignKey('Package') + + class Meta: + db_table = "storefront_provisionerplan" + +class BlogPostInfo(models.Model): + title = models.CharField(max_length=200) + url = models.CharField(max_length=1024) + date = models.DateTimeField() + author = models.CharField(max_length=64) + + class Meta: + db_table = 'storefront_blogpost' + diff --git a/nebu/api_linked_rpc.py b/nebu/api_linked_rpc.py new file mode 100644 index 0000000..2fd7649 --- /dev/null +++ b/nebu/api_linked_rpc.py @@ -0,0 +1,169 @@ +from flaptor.indextank.rpc import Indexer, Searcher, Suggestor, Storage, LogWriter, WorkerManager,\ + DeployManager, Controller, FrontendManager + +from flaptor.indextank.rpc.ttypes import NebuException, IndextankException + +''' =========================== + THRIFT STUFF + =========================== ''' +from thrift.transport import TSocket, TTransport +from thrift.protocol import TBinaryProtocol +from lib import flaptor_logging, exceptions +from thrift.transport.TTransport import TTransportException +from socket import socket +from socket import error as SocketError + + +logger = flaptor_logging.get_logger('RPC') + +# Missing a way to close transport +def getThriftControllerClient(host, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host,19010, timeout_ms) + client = Controller.Client(protocol) + transport.open() + return client + +# Missing a way to close transport +def getThriftIndexerClient(host, base_port, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host, base_port + 1, timeout_ms) + client = Indexer.Client(protocol) + transport.open() + return client + +def getThriftSearcherClient(host, base_port, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host, base_port + 2, timeout_ms) + client = Searcher.Client(protocol) + transport.open() + return client + +def getThriftSuggestorClient(host, base_port): + protocol, transport = __getThriftProtocolTransport(host, base_port + 3) + client = Suggestor.Client(protocol) + transport.open() + return client + +storage_port = 10000 +def getThriftStorageClient(): + protocol, transport = __getThriftProtocolTransport('storage',storage_port) + client = Storage.Client(protocol) + transport.open() + return client + +def getThriftLogWriterClient(host, port, timeout_ms=500): + protocol, transport = __getThriftProtocolTransport(host,port,timeout_ms) + client = LogWriter.Client(protocol) + transport.open() + return client + +def getThriftLogReaderClient(host, port, timeout_ms=None): + protocol, transport = __getThriftProtocolTransport(host,port,timeout_ms) + client = LogWriter.Client(protocol) + transport.open() + return client + +class ReconnectingClient: + def __init__(self, factory): + self.factory = factory + self.delegate = None #factory() + + def __getattr__(self, name): + import types + if self.delegate is None: + self.delegate = self.factory() + att = getattr(self.delegate, name) + if type(att) is types.MethodType: + def wrap(*args, **kwargs): + try: + return att(*args, **kwargs) + except (NebuException, IndextankException): + logger.warn('raising catcheable exception') + raise + except (TTransportException, IOError, SocketError): + logger.warn('failed to run %s, reconnecting once', name) + self.delegate = self.factory() + att2 = getattr(self.delegate, name) + return att2(*args, **kwargs) + except Exception: + logger.exception('Unexpected failure to run %s, reconnecting once', name) + self.delegate = self.factory() + att2 = getattr(self.delegate, name) + return att2(*args, **kwargs) + + return wrap + else: + return att + +def getReconnectingStorageClient(): + return ReconnectingClient(getThriftStorageClient) + +def getReconnectingLogWriterClient(host, port): + return ReconnectingClient(lambda: getThriftLogWriterClient(host, port)) + +worker_manager_port = 8799 +def getThriftWorkerManagerClient(host): + protocol, transport = __getThriftProtocolTransport(host,worker_manager_port) + client = WorkerManager.Client(protocol) + transport.open() + return client + +deploymanager_port = 8899 +def get_deploy_manager(): + protocol, transport = __getThriftProtocolTransport('deploymanager',deploymanager_port) + client = DeployManager.Client(protocol) + transport.open() + return client + + +def __getThriftProtocolTransport(host, port=0, timeout_ms=None): + ''' returns protocol,transport''' + # Make socket + transport = TSocket.TSocket(host, port) + + if timeout_ms is not None: + transport.setTimeout(timeout_ms) + + # Buffering is critical. Raw sockets are very slow + transport = TTransport.TBufferedTransport(transport) + + # Wrap in a protocol + protocol = TBinaryProtocol.TBinaryProtocol(transport) + return protocol, transport + + +def get_searcher_client(index, timeout_ms=None): + ''' + This method returns a single searcherclient, or None + ''' + deploy = index.searchable_deploy() + if deploy: + return getThriftSearcherClient(deploy.worker.lan_dns, int(deploy.base_port), timeout_ms) + else: + return None + +def get_worker_controller(worker, timeout_ms=None): + return getThriftControllerClient(worker. lan_dns) + +def get_suggestor_client(index): + ''' + This method returns a single suggestorclient, or None + ''' + deploy = index.searchable_deploy() + if deploy: + return getThriftSuggestorClient(deploy.worker.lan_dns, int(deploy.base_port)) + else: + return None + +def get_indexer_clients(index, timeout_ms=1000): + ''' + This method returns the list of all indexerclients that should be updated + on add,delete,update, and category updates. + @raise exceptions.NoIndexerException if this index has no writable deploy. + ''' + deploys = index.indexable_deploys() + retval = [] + for d in deploys: + retval.append(getThriftIndexerClient(d.worker.lan_dns, int(d.base_port), timeout_ms)) + if retval: + return retval + else: + raise exceptions.NoIndexerException() diff --git a/nebu/checkPort.sh b/nebu/checkPort.sh new file mode 100755 index 0000000..1f7cb24 --- /dev/null +++ b/nebu/checkPort.sh @@ -0,0 +1,6 @@ +#!/bin/bash +port=`echo $1 | rev | cut -c2- | rev` +(netstat -ln |egrep -m 1 "$port[0-9]") +VAL=$? +echo $VAL +exit $VAL diff --git a/nebu/controller.py b/nebu/controller.py new file mode 100644 index 0000000..5b5f022 --- /dev/null +++ b/nebu/controller.py @@ -0,0 +1,145 @@ +#!/usr/bin/python +import subprocess +import shutil +import os +import signal +import systemutils +import subprocess + +import flaptor.indextank.rpc.Controller as TController +from flaptor.indextank.rpc.ttypes import WorkerMountStats, WorkerLoadStats, IndexStats + + +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol +from thrift.server import TServer +import sys +import commands +from lib import flaptor_logging +import simplejson as json + +logger = flaptor_logging.get_logger('Controller') + +def _get_working_directory(index_code,base_port): + return "/data/indexes/%s-%d"% ( index_code, base_port) # XXX /data? + +class Controller: + """ Controls a Host""" + + def __init__(self): + pass + + def __create_indexengine_environment(self,working_directory): + shutil.copytree('indextank_lib', working_directory + "/lib") + shutil.copy('startIndex.py', working_directory) + shutil.copy('log4j.properties', working_directory) + + + def _check_port_available(self, port): + '''Returns true if the port is not being used''' + o = subprocess.call(['./checkPort.sh', str(port)]) + if o == 0: + return False + else: + return True + + + + def start_engine(self, json_configuration = '{}'): + + config = json.loads(json_configuration) + index_code = config['index_code'] + base_port = int(config['base_port']) + + if not self._check_port_available(base_port): + logger.warn("I was requested to start an index engine with code %s, but the port (%i) is not free." % (index_code, base_port)) + return False + working_directory = _get_working_directory(index_code, base_port) + + self.__create_indexengine_environment(working_directory) + + config_file_name = 'indexengine_config' + config_file = open(os.path.join(working_directory, config_file_name), 'w') + config_file.write(json_configuration) + config_file.close() + + logger.info("starting %s on port %d", index_code, base_port) + cmd = ['python','startIndex.py', config_file_name] + + logger.debug("executing %r", cmd) + subprocess.call(cmd, cwd=working_directory, close_fds=True) + return True + + + def get_worker_mount_stats(self): + return WorkerMountStats(systemutils.get_available_sizes()) + + def get_worker_load_stats(self): + loads = systemutils.get_load_averages() + return WorkerLoadStats(loads[0], loads[1], loads[2]) + + def get_worker_index_stats(self, index_code, port): + index_stats = systemutils.get_index_stats(index_code, port) + return IndexStats(used_disk=index_stats['disk'], used_mem=index_stats.get('mem') or 0) + + def kill_engine(self,index_code, base_port): + """ Kills the IndexEngine running on base_port. + This is a safeguard. The correct way to stop an IndexEngine is + asking it to do so through it's thrift api + """ + + try: + f = open("%s/pid"% _get_working_directory(index_code, base_port)) + pid = int(f.read().strip()) + os.kill(pid,signal.SIGKILL) # signal 9 + f.close() + #shutil.rmtree(_get_working_directory(index_code, base_port)) + return 0 + except Exception, e: + logger.error(e) + return 1 + + def stats(self): + """ Provides stats about the host it is running on. May need to parse top, free, uptime and such.""" + logger.debug("host stats") + pass + + def update_worker(self, host): + logger.info("UPDATING WORKER FROM %s", host) + retcode = subprocess.call(['bash', '-c', 'rsync -avL -e "ssh -o StrictHostKeyChecking=no -i /home/indextank/.ssh/id_rsa_worker -l indextank" %s:/home/indextank/nebu/ /home/indextank/nebu' % (host)]) + return retcode + + def restart_controller(self): + logger.info('Attempting restart') + subprocess.call(['bash', '-c', 'sudo /etc/init.d/indexengine-nebu-controller restart']) + sys.exit(0) + logger.error('Survived a restart?') + + def tail(self, file, lines, index_code, base_port): + if index_code: + file = '/data/indexes/%s-%d/%s' % (index_code, base_port, file) + return commands.getoutput("tail -n %d %s" % (lines, file)) + + def head(self, file, lines, index_code, base_port): + if index_code: + file = '/data/indexes/%s-%d/%s' % (index_code, base_port, file) + return commands.getoutput("head -n %d %s" % (lines, file)) + + def ps_info(self, pidfile, index_code, base_port): + if index_code: + pidfile = '/data/indexes/%s-%d/pid' % (index_code, base_port) + return commands.getoutput("ps u -p `%s`" % (pidfile)) + + +if __name__ == '__main__': + handler = Controller() + processor = TController.Processor(handler) + transport = TSocket.TServerSocket(19010) + tfactory = TTransport.TBufferedTransportFactory() + pfactory = TBinaryProtocol.TBinaryProtocolFactory() + + server = TServer.TThreadPoolServer(processor, transport, tfactory, pfactory) + logger.info('Starting the controller server...') + server.serve() + diff --git a/nebu/deploy_manager.py b/nebu/deploy_manager.py new file mode 100644 index 0000000..425f34a --- /dev/null +++ b/nebu/deploy_manager.py @@ -0,0 +1,514 @@ +#!/usr/bin/python + +from flaptor.indextank.rpc import DeployManager as TDeployManager +from flaptor.indextank.rpc.ttypes import IndexerStatus, NebuException +from nebu.models import Worker, Deploy, Index, IndexConfiguration, Service + +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol +from thrift.server import TServer + +from lib import flaptor_logging, mail +import simplejson as json +import rpc +import sys +import random +from traceback import format_tb +from django.db import transaction +from datetime import datetime, timedelta + +logger = flaptor_logging.get_logger('DeployMgr') + +INITIAL_XMX = 100 +INITIAL_XMX_THRESHOLD = 1000 +MAXIMUM_PARALLEL_MOVES = 10 +timeout_ms = 1000 + +def logerrors(func): + def decorated(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception, e: + logger.exception("Failed while executing %s", func.__name__) + raise e + return decorated + + +class DeployManager: + ''' + Move index from A to B: + A.Flush { + switch + make hardlink copy + } + Rsync A/Flushed -> B + B.Start_Recovery + Index to A and B + Wait until B.Finished_Recovery + Direct index ad search to B + Kill and delete A + ''' + + @logerrors + def __find_best_worker(self, required_ram): + ''' + finds the 'best' worker to use for a deploy requiring 'required_ram' + + constraints: + * the worker can accomodate 'required_ram' + * the worker is not in 'decomissioning' state + + for all those workers that comply to the above constraint, pick one considering: + * workers in 'controllable' state are better than other workers + * to choose between 2 workers, having less ram available is better + + if there is NO worker matching the constraints, this method will try to create a new worker + ''' + + + # make sure at least 1 worker matches the constraints + workers = Worker.objects.exclude(status=Worker.States.decommissioning).exclude(status=Worker.States.dying).exclude(status=Worker.States.dead) + + should_create_worker = True + for worker in workers: + used = self.__get_total_ram(worker) + available = worker.get_usable_ram() - used + if available > required_ram: + # found one that could accomodate 'required_ram' .. no need to look further. + should_create_worker = False + break + + if should_create_worker: + # create a new worker. it will be available on next loop iteration + wm_client = rpc.getThriftWorkerManagerClient('workermanager') + wm_client.add_worker('m2.xlarge') + + + # OK, either there were workers that could accomodate 'required_ram', or we just created one. + # find the best + + # first try only controllable workers + # this way the deploy will be available FASTER + workers = Worker.objects.filter(status=Worker.States.controllable) + + for i in range(2): + best_worker = None + best_ram = 1000000 # we should bump this number when we get workers with ram > 1TB + for worker in workers: + used = self.__get_total_ram(worker) + available = worker.get_usable_ram() - used + if available > required_ram and available < best_ram: + best_ram = available + best_worker = worker + + if best_worker is not None: + return best_worker + + # need to try again. let workers be everything but decomissioning, dying or dead ones. + workers = Worker.objects.exclude(status=Worker.States.decommissioning).exclude(status=Worker.States.dying).exclude(status=Worker.States.dead) + + # tried only controllable, and everything .. still not found + raise Exception('Second iteration and no suitable worker. Something is wrong...') + + def __get_total_ram(self, worker): + return sum(d.total_ram() for d in worker.deploys.all()) + + + """ Return values for start_index """ + INDEX_ALREADY_RUNNING = 0 + WORKER_NOT_READY_YET = 1 + INDEX_INITIALIZING = 2 + INDEX_RECOVERING = 4 + INDEX_CONTROLLABLE = 3 + INDEX_MOVE_REQUESTED = 5 + INDEX_MOVING = 6 + INDEX_MOVED = 7 + + def service_deploys(self): + error_list = [] + + for index in Index.objects.filter(deploys__isnull=True).exclude(status=Index.States.hibernated).exclude(status=Index.States.hibernate_requested).exclude(deleted=True): + try: + self._create_new_deploy(index) + except Exception: + logger.exception('Failed to service index %s [%s]', index.name, index.code) + exc_type, exc_value, exc_traceback = sys.exc_info() + error_list.append('Failed to service index\n\n%s\n\nEXCEPTION: %s : %s\ntraceback:\n%s' % (index.get_debug_info(), exc_type, exc_value, ''.join(format_tb(exc_traceback)))) + + for index in Index.objects.filter(deploys__isnull=True, deleted=True): + index.delete() + + for index in Index.objects.filter(deleted=True): + index.deploys.all().update(status=Deploy.States.decommissioning) + + for index in Index.objects.filter(status=Index.States.hibernate_requested): + self._hibernate_index(index) + + + for worker in Worker.objects.filter(status=Worker.States.dying): + deploys = worker.deploys.all() + + if deploys: + for deploy in deploys: + deploy.dying = True + deploy.save() + else: + worker.status = Worker.States.dead + worker.save() + + deploys = Deploy.objects.select_related('parent', 'index', 'index__configuration', 'index__account__package', 'worker').all() + + for deploy in deploys: + try: + if deploy.dying: + self._service_dying_deploy(deploy) + else: + self._service_deploy(deploy) + + except Exception: + logger.exception('Failed to service deploy %s of index %s [%s]', deploy.id, deploy.index.name, deploy.index.code) + exc_type, exc_value, exc_traceback = sys.exc_info() + error_list.append('Failed to service deploy %s for index\n\n%s\n\nEXCEPTION: %s : %s\ntraceback:\n%s' % (deploy.id, deploy.index.get_debug_info(), exc_type, exc_value, ''.join(format_tb(exc_traceback)))) + + exc_type, exc_value, exc_traceback = sys.exc_info() + error_list.append('Failed to service deploy %s for index\n\n%s\n\nEXCEPTION: %s : %s\ntraceback:\n%s' % (deploy.id, deploy.index.get_debug_info(), exc_type, exc_value, ''.join(format_tb(exc_traceback)))) + + if error_list: + raise NebuException('Deploy manager failed to service some indexes:\n---\n' + '\n---\n'.join(error_list)) + + def _get_xmx(self, index): + if index.current_docs_number < INITIAL_XMX_THRESHOLD: + config = index.configuration.get_data() + return config.get('initial_xmx', INITIAL_XMX) + else: + return index.configuration.get_data()['xmx'] + + def _get_bdb(self, index): + return index.configuration.get_data().get('bdb_cache',0) + + def _create_new_deploy(self, index, parent=None): + ''' + Creates a deploy, but doesn't start it + ''' + if parent: + assert(parent.index == index) + logger.debug('Creating new deploy for Index "%s" (%s).', index.name, index.code) + + xmx = self._get_xmx(index) + bdb = self._get_bdb(index) + + worker = self.__find_best_worker(xmx) + + deploy = Deploy() + deploy.parent = parent + deploy.base_port = 0 + deploy.worker = worker + deploy.index = index + deploy.status = Deploy.States.created + deploy.effective_xmx = xmx + deploy.effective_bdb = bdb + if Index.objects.filter(id=index.id).count() > 0: + deploy.save() + logger.info('Deploy %s:%d created for index "%s" (%s).', deploy.worker.instance_name, deploy.base_port, index.name, index.code) + else: + logger.error('Deploy %s:%d was NOT created for index "%s" (%s) since the index was deleted.', deploy.worker.instance_name, deploy.base_port, index.name, index.code) + + + def _service_deploy(self, deploy): + # Handle all decommissioning deploys including those for deleted indexes + if deploy.status == Deploy.States.decommissioning: + self._handle_decommissioning(deploy) + # Only handle states if the index is not deleted + elif not deploy.index.deleted: + if deploy.status == Deploy.States.moving: + self._handle_moving(deploy) + elif deploy.status == Deploy.States.move_requested: + self._handle_move_requested(deploy) + elif deploy.status == Deploy.States.controllable: + self._handle_controllable(deploy) + elif deploy.status == Deploy.States.recovering: + self._handle_recovering(deploy) + elif deploy.status == Deploy.States.initializing: + self._handle_initializing(deploy) + elif deploy.status == Deploy.States.created: + self._handle_created(deploy) + else: + logger.error('Unknown deploy state %s for deploy %s. Will do nothing.' % (deploy.status, deploy)) + + + def _service_dying_deploy(self, deploy): + deploy = Deploy.objects.get(id=deploy.id) + if deploy.status in [Deploy.States.decommissioning, Deploy.States.initializing, Deploy.States.created, Deploy.States.moving] : + self._handle_dying_killable(deploy) + elif deploy.status in [Deploy.States.move_requested, Deploy.States.controllable]: + self._handle_dying_controllable(deploy) + elif deploy.status == Deploy.States.recovering: + self._handle_dying_recovering(deploy) + else: + logger.error('Unknown deploy state %s for deploy %s. Will do nothing.' % (deploy.status, deploy)) + + def _handle_move_requested(self, deploy): + if Deploy.objects.filter(status=Deploy.States.moving).count() < MAXIMUM_PARALLEL_MOVES: + self._create_new_deploy(deploy.index, deploy) + deploy.update_status(Deploy.States.moving) + else: + logger.warn("Too many parallel moves. Waiting to move deploy %s for index %s", deploy, deploy.index.code) + + def _handle_controllable(self, deploy): + should_have_xmx = self._get_xmx(deploy.index) + if deploy.effective_xmx < should_have_xmx: + logger.info("Requesting move. Effective XMX (%dM) was smaller than needed (%dM) for deploy %s for index %s", deploy.effective_xmx, self._get_xmx(deploy.index), deploy, deploy.index.code) + deploy.update_status(Deploy.States.move_requested) + mail.report_automatic_redeploy(deploy, deploy.effective_xmx, should_have_xmx) + + def _handle_dying_controllable(self, deploy): + # Create a new deploy for the index with no parent + if deploy.index.status != Index.States.hibernate_requested: + self._create_new_deploy(deploy.index) + + # Remove the deploy + self._delete_deploy(deploy) + + def _handle_dying_recovering(self, deploy): + logger.info('deleting recovering deploy %s due to dying worker' % (deploy)) + + if deploy.parent: + # Re request the move for the original + deploy.parent.update_status(Deploy.States.move_requested) + else: + # Create a new deploy for the index with no parent + if deploy.index.status != Index.States.hibernate_requested: + self._create_new_deploy(deploy.index) + + # Remove the deploy + self._delete_deploy(deploy) + + def _handle_decommissioning(self, deploy): + td = datetime.now() - deploy.timestamp + min_delay = timedelta(seconds=10) + if td > min_delay: + logger.info('deleting deploy %s' % (deploy)) + self._delete_deploy(deploy) + + def _handle_dying_killable(self, deploy): + logger.info('deleting deploy %s due to dying worker' % (deploy)) + + deploy.children.update(parent=None) + + if deploy.parent: + parent = deploy.parent + parent.status=Deploy.States.move_requested + parent.save() + + # Remove the parent relationship for all children + # That way, those deploys are considered "readable" + + # Remove the deploy entirely. RIP. + self._delete_deploy(deploy) + + def _handle_moving(self, deploy): + child = deploy.children.all()[0] #Assumes there's only 1 child + if child.status == Deploy.States.controllable or child.status == Deploy.States.move_requested or child.status == Deploy.States.moving: + logger.info('deploy %s:%d of Index "%s" (%s), replaces deploy %s:%d', child.worker.instance_name, child.base_port, deploy.index.name, deploy.index.code, deploy.worker.instance_name, deploy.base_port) + child.update_parent(None) + deploy.update_status(Deploy.States.decommissioning) + else: + logger.info('deploy %s:%d of Index "%s" (%s), moving to deploy %s:%d', deploy.worker.instance_name, deploy.base_port, deploy.index.name, deploy.index.code, child.worker.instance_name, child.base_port) + + + def _handle_recovering(self, deploy): + logger.debug('Contacting %s (%s) on %d of %s to check if it finished recovering', deploy.index.name, deploy.index.code, deploy.base_port, deploy.worker.wan_dns) + try: + indexer = rpc.getThriftIndexerClient(deploy.worker.lan_dns, int(deploy.base_port), timeout_ms) + indexer_status = indexer.getStatus() + if indexer_status == IndexerStatus.started: + logger.info('Requesting full recovery for deploy for %s (%s) on %d.', deploy.index.name, deploy.index.code, deploy.base_port) + indexer.startFullRecovery() + return DeployManager.INDEX_RECOVERING + elif indexer_status == IndexerStatus.recovering: + logger.info("Index %s is in state %s. Waiting untill it's ready", deploy.index.code, indexer_status) + return DeployManager.INDEX_RECOVERING + elif indexer_status == IndexerStatus.ready: + deploy.update_status(Deploy.States.controllable) + if deploy.index.status == Index.States.waking_up: + deploy.index.update_status(Index.States.live) + mail.report_new_deploy(deploy) + + logger.info('Deploy for %s (%s) on %d of %s reports it has finished recovering. New state moved to %s.', deploy.index.name, deploy.index.code, deploy.base_port, deploy.worker.wan_dns, Deploy.States.controllable) + + # The following is a HACK to restore twitvid's promotes after its index was moved + # because we don't record promotes and they are lost after each move. + # Luckily we know what twitvid promotes, so we can reproduce it here. This will break if they + # change their code and start promoting something else. GitHub issue #41 calls for a proper + # implementaion or to remove the feature altogether. Twitvid could now do this by using the + # caret operator like this: "name:(q)^100 OR author:(q)^100 OR (q)". + + if deploy.index.code == 'd7fz1': + try: + searcher = rpc.getThriftSearcherClient(deploy.worker.lan_dns, int(deploy.base_port), timeout_ms) + start = 0 + while True: + rs = searcher.search('verified:1 AND cont_type:user', start, 1000, 0, {}, {}, {}, {}, {'fetch_fields':'author,fullname,name'}) + if len(rs.docs) == 0: + break + for d in rs.docs: + author = d.get('author') + if author: + indexer.promoteResult(d['docid'], author.lower().strip()) + name = d.get('name', d.get('fullname')) + if name: + indexer.promoteResult(d['docid'], name.lower().strip()) + start += len(rs.docs) + logger.info('WARNING: HACK! %s promotes were recovered for TwitVid.', start) + except Exception, e: + logger.error('HACK ERROR: applying TwitVid promotes', e) + + # Phew. End of HACK. Please let's not do this anymore. + + return DeployManager.INDEX_CONTROLLABLE + elif indexer_status == IndexerStatus.error: + logger.error('Deploy for %s (%s) on %d of %s reports it has failed recovering. MANUAL INTERVENTION REQUIRED.', deploy.index.name, deploy.index.code, deploy.base_port, deploy.worker.wan_dns) + #TODO: Send an alert. + except Exception, e: + logger.error('Index %s unreachable: %s, but its state is recovering', deploy.index.code, e) + return DeployManager.INDEX_RECOVERING + + def _handle_initializing(self, deploy): + logger.debug('Trying to reach %s (%s) on %d of %s.', deploy.index.name, deploy.index.code, deploy.base_port, deploy.worker.wan_dns) + try: + indexer = rpc.getThriftIndexerClient(deploy.worker.lan_dns, int(deploy.base_port), timeout_ms) + indexer.ping() + # successfully reported stats() + if deploy.index.status == Index.States.new: + deploy.update_status(Deploy.States.controllable) + index = deploy.index + index.status = Index.States.live + index.save() + logger.info('Deploy for %s (%s) on %d of %s contacted. New status moved to %s.', deploy.index.name, deploy.index.code, deploy.base_port, deploy.worker.wan_dns, Deploy.States.controllable) + return DeployManager.INDEX_CONTROLLABLE + else: + deploy.update_status(Deploy.States.recovering) + logger.info('Deploy for %s (%s) on %d of %s contacted. New status moved to %s.', deploy.index.name, deploy.index.code, deploy.base_port, deploy.worker.wan_dns, Deploy.States.recovering) + return DeployManager.INDEX_RECOVERING + except Exception, e: + # not ready yet, we'll leave it as initializing + logger.info('Index %s unreachable: %s', deploy.index.code, e) + return DeployManager.INDEX_INITIALIZING + + def _get_free_port(self, deploy): + while (True): + candidate = random.Random().randrange(20000, stop=23990, step=10) + if 0 == Deploy.objects.filter(worker=deploy.worker, base_port=candidate).count(): + return candidate + + def _handle_created(self, deploy): + if not deploy.worker.is_ready(): + logger.info('Waiting to initialize index "%s" (%s) on %s:%d. The worker is not ready yet', deploy.index.name, deploy.index.code, deploy.worker.instance_name, deploy.base_port) + return DeployManager.WORKER_NOT_READY_YET + + # else + controller = rpc.getThriftControllerClient(deploy.worker.lan_dns) + json_config = {} + json_config['functions'] = deploy.index.get_functions_dict() + + # there should be exactly one recovery service + recovery_service = Service.objects.get(name='recovery') + # log based storage + json_config['log_based_storage'] = True + json_config['log_server_host'] = recovery_service.host + json_config['log_server_port'] = recovery_service.port + + json_config.update(deploy.index.configuration.get_data()) + + proposed_port = self._get_free_port(deploy) + json_config['base_port'] = proposed_port + json_config['index_code'] = deploy.index.code + + analyzer_config = deploy.index.get_json_for_analyzer() + if analyzer_config: + json_config['analyzer_config'] = analyzer_config + + + logger.info('Initializing index "%s" (%s) on %s:%d', deploy.index.name, deploy.index.code, deploy.worker.instance_name, proposed_port) + + # override xmx with the one defined for this deploy + json_config['xmx'] = deploy.effective_xmx + + logger.debug("deploy: %r\n----\nindex: %r\n----\nstart args: %r", deploy, deploy.index, json_config) + started_ok = controller.start_engine(json.dumps(json_config)) + if started_ok: + qs = Deploy.objects.filter(id=deploy.id) + qs.update(base_port=proposed_port) + qs = Deploy.objects.filter(id=deploy.id,index__deleted=False) + qs.update(status=Deploy.States.initializing) + return DeployManager.INDEX_INITIALIZING + else: + logger.warn('Deploy failed starting. Will try again in next round.'); + return + + + @logerrors + def delete_index(self, index_code): + ''' + stops every running IndexEngine associated with the index_code, and + deletes from MySql deploys and index. + ''' + Index.objects.filter(code=index_code).update(name='[removed-%s]' % index_code) + Index.objects.get(code=index_code).mark_deleted() + + logger.info('Marked index %s as deleted', index_code) + return 1 + + def _delete_deploy(self, deploy): + logger.debug('Deleting deploy: %r', deploy) + if deploy.base_port: + try: + controller = rpc.getThriftControllerClient(deploy.worker.lan_dns) + controller.kill_engine(deploy.index.code,deploy.base_port) + except: + logger.exception('Failed when attempting to kill the IndexEngine for the deploy %s', deploy) + + index = deploy.index + deploy.delete() + + if index.deleted and index.deploys.count() == 0: + index.delete() + + @logerrors + def redeploy_index(self,index_code): + index = Index.objects.get(code=index_code) + deploys = index.deploys.all() + if len(deploys) != 1: + logger.error("Call to redeploy_index for index_code %s failed, this index has %i deploys...", index_code, len(deploys)) + else: + deploy = deploys[0] + if deploy.status != Deploy.States.controllable: + logger.error("Call to redeploy_index for index_code %s failed, this indexe's deploy is in state %s", index_code, deploy.status) + else: + deploy.update_status(Deploy.States.move_requested) + logger.info('Deploy for index_code %s changed from controllable to move_requested.', index_code) + + + @logerrors + def _hibernate_index(self,index): + if index.account.package.base_price == 0 or index.is_demo(): + for deploy in index.deploys.all(): + self._delete_deploy(deploy) + index.update_status(Index.States.hibernated) + else: + logger.error('Index_code %s was asked to hibernate. Only free or demo indexes can hibernate.', index.code) + + +if __name__ == "__main__": + handler = DeployManager() + processor = TDeployManager.Processor(handler) + transport = TSocket.TServerSocket(8899) + tfactory = TTransport.TBufferedTransportFactory() + pfactory = TBinaryProtocol.TBinaryProtocolFactory() + + server = TServer.TThreadPoolServer(processor, transport, tfactory, pfactory) + logger.info('Starting the deploy manager server...') + server.serve() + + diff --git a/nebu/deploy_manager_client.py b/nebu/deploy_manager_client.py new file mode 100644 index 0000000..d1330f0 --- /dev/null +++ b/nebu/deploy_manager_client.py @@ -0,0 +1,41 @@ +import sys + +from thrift.transport import TTransport, TSocket +from thrift.protocol import TBinaryProtocol + +from flaptor.indextank.rpc import DeployManager + + +# Missing a way to close transport +def getThriftDeployManagerClient(host): + protocol, transport = __getThriftProtocolTransport(host,8899) + client = DeployManager.Client(protocol) + transport.open() + return client + +def __getThriftProtocolTransport(host,port=0): + ''' returns protocol,transport''' + # Make socket + transport = TSocket.TSocket(host, port) + + # Buffering is critical. Raw sockets are very slow + transport = TTransport.TBufferedTransport(transport) + + # Wrap in a protocol + protocol = TBinaryProtocol.TBinaryProtocol(transport) + return protocol, transport + +if __name__ == "__main__": + client = getThriftDeployManagerClient('deploymanager') + while True: + line = sys.stdin.readline() + if not line : break + + line = line.strip() + if line.startswith('redeploy'): + code = line[8:].strip() + client.redeploy_index(code) + print 'redeploy executed' + else: + print 'invalid command' + diff --git a/nebu/inst.json b/nebu/inst.json new file mode 100644 index 0000000..785d707 --- /dev/null +++ b/nebu/inst.json @@ -0,0 +1,440 @@ +{"fields": {"url": "http://freebase.com/view/en/e-flat_clarinet", "text": "The E-flat clarinet is a member of the clarinet family. It is usually classed as a soprano clarinet, although some authors describe it as a \"sopranino\" or even \"piccolo\" clarinet. Smaller in size and higher in pitch than the more common B\u266d clarinet, it is a transposing instrument in E\u266d, sounding a minor third higher than written. In Italian it sometimes referred to as a quartino, generally appearing in scores as quartino in Mi\u266d.\nThe E\u266d clarinet is used in orchestras, concert bands, marching...", "image": "http://img.freebase.com/api/trans/raw/m/02cy_r6", "name": "E-flat clarinet", "thumbnail": "http://indextank.com/_static/common/demo/02cy_r6.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/e-flat_clarinet", "categories": {"family": "Clarinet"}} +{"fields": {"url": "http://freebase.com/view/en/swedish_bagpipes", "text": "Swedish bagpipes (Swedish: Svensk s\u00e4ckpipa) are a variety of bagpipes from Sweden. The term itself generically translates to \"bagpipes\" in Swedish, but is used in English to describe the specifically Swedish bagpipe from the Dalarna region.\nMedieval paintings in churches suggest that the instrument was spread all over Sweden. The instrument was practically extinct by the middle of the 20th century; the instrument that today is referred to as Swedish bagpipes is a construction based on...", "image": "http://img.freebase.com/api/trans/raw/m/029mx2d", "name": "Swedish bagpipes", "thumbnail": "http://indextank.com/_static/common/demo/029mx2d.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/swedish_bagpipes", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/portuguese_guitar", "text": "The Portuguese guitar or Portuguese guitarra (Portuguese: guitarra portuguesa) is a plucked string instrument with twelve steel strings, strung in six courses comprising two strings each. It has a distinctive tuning mechanism. It is most notably associated with fado.\nThe origin of the Portuguese guitar is a subject of debate. Throughout the 19th century the Portuguese guitar was being made in several sizes and shapes and subject to several regional aesthetic trends. A sizeable guitar making...", "image": "http://img.freebase.com/api/trans/raw/m/02b57zg", "name": "Portuguese guitar", "thumbnail": "http://indextank.com/_static/common/demo/02b57zg.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/portuguese_guitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/accordion", "text": "The accordion is a box-shaped musical instrument of the bellows-driven free-reed aerophone family, sometimes referred to as a squeezebox. A person who plays the accordion is called an accordionist.\nIt is played by compressing or expanding a bellows whilst pressing buttons or keys, causing valves, called pallets, to open, which allow air to flow across strips of brass or steel, called reeds, that vibrate to produce sound inside the body.\nThe instrument is sometimes considered a one-man-band...", "image": "http://img.freebase.com/api/trans/raw/m/03r6gyw", "name": "Accordion", "thumbnail": "http://indextank.com/_static/common/demo/03r6gyw.jpg"}, "variables": {"0": 233}, "docid": "http://freebase.com/view/en/accordion", "categories": {"family": "Keyboard instrument"}} +{"fields": {"url": "http://freebase.com/view/en/tuba", "text": "The tuba is the largest and lowest pitched brass instrument. Sound is produced by vibrating or \"buzzing\" the lips into a large cupped mouthpiece. It is one of the most recent additions to the modern symphony orchestra, first appearing in the mid-19th century, when it largely replaced the ophicleide. Tuba is Latin for trumpet or horn. The horn referred to would most likely resemble what is known as a baroque trumpet.\nPrussian Patent No. 19 was granted to Wilhelm Friedrich Wieprecht and Carl...", "image": "http://img.freebase.com/api/trans/raw/m/02bjtvj", "name": "Tuba", "thumbnail": "http://indextank.com/_static/common/demo/02bjtvj.jpg"}, "variables": {"0": 28}, "docid": "http://freebase.com/view/en/tuba", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/cuatro", "text": "The cuatro is any of several Latin American instruments of the guitar or lute family. The cuatro is smaller than a guitar. Cuatro means four in Spanish, although current instruments may have more than four strings.\nAn instrument of the guitar family, found in South America, Trinidad & Tobago and other territories of the West Indies. Its 15th century predecessor was the Portuguese Cavaquinho, which, like the cuatro had four strings. The cuatro is widely used in ensembles in Colombia, Jamaica,...", "image": "http://img.freebase.com/api/trans/raw/m/02fpz2k", "name": "Cuatro", "thumbnail": "http://indextank.com/_static/common/demo/02fpz2k.jpg"}, "variables": {"0": 9}, "docid": "http://freebase.com/view/en/cuatro", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/gudastviri", "text": "The gudastviri (Georgian: \u10d2\u10e3\u10d3\u10d0\u10e1\u10e2\u10d5\u10d8\u10e0\u10d8) is a droneless, double-chantered, horn-belled bagpipe played in Georgia. The term comes from the words guda (bag) and stviri (whistling). In some regions, the instrument is called the chiboni, stviri, or tulumi.\nThis type of bagpipe is found in many regions of Georgia, and is known by different names in various areas.\nThese variants differ from one another in timbre, capacity/size of the bag, and number of holes on the two pipes.\nThe gudastviri is made...", "image": "http://img.freebase.com/api/trans/raw/m/04pzpv_", "name": "Gudastviri", "thumbnail": "http://indextank.com/_static/common/demo/04pzpv_.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gudastviri", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/drum_kit", "text": "A drum kit (also drum set, or trap set) is a collection of drums, cymbals and often other percussion instruments, such as cowbells, wood blocks, triangles, chimes, or tambourines, arranged for convenient playing by a single person (drummer).\nThe individual instruments of a drum-set are hit by a variety of implements held in the hand, including sticks, brushes, and mallets. Two notable exceptions include the bass drum, played by a foot-operated pedal, and the hi-hat cymbals, which may be...", "image": "http://img.freebase.com/api/trans/raw/m/02bdbkf", "name": "Drum kit", "thumbnail": "http://indextank.com/_static/common/demo/02bdbkf.jpg"}, "variables": {"0": 1709}, "docid": "http://freebase.com/view/en/drum_kit", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/viol", "text": "The viol (also known as the Viola da gamba) is any one of a family of bowed, fretted and stringed musical instruments developed in the mid-late 15th century and used primarily in the Renaissance and Baroque periods. The family is related to and descends primarily from the Renaissance vihuela, a plucked instrument that preceded the guitar. An influence in the playing posture has been credited to the example of Moorish rabab players.\nViols are different in several respects from instruments of...", "image": "http://img.freebase.com/api/trans/raw/m/03slmhd", "name": "Viol", "thumbnail": "http://indextank.com/_static/common/demo/03slmhd.jpg"}, "variables": {"0": 8}, "docid": "http://freebase.com/view/en/viol", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/tiple", "text": "Tiple (pronounced as\u00a0:tee-pleh) is the Spanish word for treble or soprano, is often applied to specific instruments, generally to refer to a small chordophone of the guitar family.\nThe tiple is the smallest of the three string instruments of Puerto Rico that make up the orquesta jibara (i.e., the Cuatro, the Tiple and the Bordonua). According to investigations made by Jose Reyes Zamora, the tiple in Puerto Rico dates back to the 18th century. It is believed to have evolved from the Spanish...", "image": "http://img.freebase.com/api/trans/raw/m/042jl6k", "name": "Tiple", "thumbnail": "http://indextank.com/_static/common/demo/042jl6k.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/tiple", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/askomandoura", "text": "The askomandoura (Greek: \u03b1\u03c3\u03ba\u03bf\u03bc\u03b1\u03bd\u03c4\u03bf\u03cd\u03c1\u03b1) is a type of bagpipe played as a traditional instrument on the Greek island of Crete, similar to the tsampouna.\nIts use in Crete is attested in illustrations from the mid-15th Century.", "image": "http://img.freebase.com/api/trans/raw/m/0ccpq4r", "name": "Askomandoura", "thumbnail": "http://indextank.com/_static/common/demo/0ccpq4r.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/askomandoura", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/guiro", "text": "The g\u00fciro (Spanish pronunciation:\u00a0[\u02c8\u0261wi\u027eo]) is a Dominican [percussion instrument]] consisting of an open-ended, hollow gourd with parallel notches cut in one side. It is played by rubbing a wooden stick (\"pua\") along the notches to produce a ratchet-like sound. The g\u00fciro is commonly used in Latin-American music, and plays a key role in the typical cumbia rhythm section. The g\u00fciro is also known as calabazo, guayo, ralladera, or rascador. In Brazil it is commonly known as \"reco-reco\".\nThe...", "image": "http://img.freebase.com/api/trans/raw/m/03rh0wz", "name": "G\u00fciro", "thumbnail": "http://indextank.com/_static/common/demo/03rh0wz.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/guiro", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/piccolo_oboe", "text": "The piccolo oboe, also known as the piccoloboe, is the smallest and highest pitched member of the oboe family, historically known as the oboe musette. (It should not be confused with the similarly named musette, which is bellows-blown and characterized by a drone.) Pitched in E-flat or F above the regular oboe (which is a C instrument), the piccolo oboe is a sopranino version of the oboe, comparable to the E-flat clarinet.\nPiccolo oboes are produced by the French makers F. Lor\u00e9e and Marigaux...", "image": "http://img.freebase.com/api/trans/raw/m/02clxsj", "name": "Piccolo oboe", "thumbnail": "http://indextank.com/_static/common/demo/02clxsj.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/piccolo_oboe", "categories": {"family": "Oboe"}} +{"fields": {"url": "http://freebase.com/view/en/heckelphone", "text": "The Heckelphone (German: Heckelphon) is a musical instrument invented by Wilhelm Heckel and his sons. Introduced in 1904, it is similar to the cor anglais (English horn).\nThe Heckelphone is a double reed instrument of the oboe family, but with a wider bore and hence a heavier and more penetrating tone. It is pitched an octave below the oboe and furnished with an additional semitone taking its range down to A. It was intended to provide a broad oboe-like sound in the middle register of the...", "image": "http://img.freebase.com/api/trans/raw/m/044xfzb", "name": "Heckelphone", "thumbnail": "http://indextank.com/_static/common/demo/044xfzb.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/heckelphone", "categories": {"family": "Oboe"}} +{"fields": {"url": "http://freebase.com/view/en/harp_guitar", "text": "The harp guitar (or \"harp-guitar\") is a stringed instrument with a history of well over two centuries. While there are several unrelated historical stringed instruments that have appropriated the name \u201charp-guitar\u201d over the centuries, the term today is understood as the accepted vernacular to refer to a particular family of instruments defined as \"A guitar, in any of its accepted forms, with any number of additional unstopped strings that can accommodate individual plucking.\" Additionally,...", "image": "http://img.freebase.com/api/trans/raw/m/02d5pp1", "name": "Harp guitar", "thumbnail": "http://indextank.com/_static/common/demo/02d5pp1.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/harp_guitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/bandoneon", "text": "The bandone\u00f3n is a type of concertina particularly popular in Argentina and Uruguay. It plays an essential role in the orquesta t\u00edpica, the tango orchestra. The bandone\u00f3n, called bandonion by a German instrument dealer, Heinrich Band (1821\u20131860), was originally intended as an instrument for religious music and the popular music of the day, in contrast to its predecessor, the German concertina (or Konzertina), considered to be a folk instrument by some modern authors. German sailors and...", "image": "http://img.freebase.com/api/trans/raw/m/04pggqy", "name": "Bandone\u00f3n", "thumbnail": "http://indextank.com/_static/common/demo/04pggqy.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/bandoneon", "categories": {"family": "Accordion"}} +{"fields": {"url": "http://freebase.com/view/en/an_b_u", "text": "The \u0111\u00e0n b\u1ea7u (\u0111\u00e0n \u0111\u1ed9c huy\u1ec1n or \u0111\u1ed9c huy\u1ec1n c\u1ea7m, \u7368\u7d43\u7434) is a Vietnamese monochord. While the earliest written records of the Dan Bau date its origin to 1770, many scholars estimate its age to be up to one thousand years older than that. A popular legend of its beginning tells of a blind woman playing it in the market to earn a living for her family while her husband was at war. Whether this tale is based in fact or not, it remains true that the Dan Bau has historically been played by blind...", "image": "http://img.freebase.com/api/trans/raw/m/0c5bjvy", "name": "\u0110\u00e0n b\u1ea7u", "thumbnail": "http://indextank.com/_static/common/demo/0c5bjvy.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/an_b_u", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/positive_organ", "text": "A positive organ (pronounced \"positeev\"; also positiv organ, positif organ, portable organ, chair organ, or simply positive, positiv, positif, or chair) (from the Latin verb ponere, \"to place\") is a small, usually one-manual, pipe organ that is built to be more or less mobile. It was common in sacred and secular music between the 10th and the 18th centuries, in chapels and small churches, as a chamber organ and for the basso continuo in ensemble works. The smallest common kind of positive,...", "image": "http://img.freebase.com/api/trans/raw/m/044y95b", "name": "Positive organ", "thumbnail": "http://indextank.com/_static/common/demo/044y95b.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/positive_organ", "categories": {"family": "Organ"}} +{"fields": {"url": "http://freebase.com/view/en/basset-horn", "text": "The basset horn (sometimes written basset-horn) is a musical instrument, a member of the clarinet family.\nLike the clarinet, the instrument is a wind instrument with a single reed and a cylindrical bore. However, the basset horn is larger and has a bend near the mouthpiece rather than an entirely straight body (older instruments are typically curved or bent in the middle), and while the clarinet is typically a transposing instrument in B\u266d or A (meaning a written C sounds as a B\u266d or A), the...", "image": "http://img.freebase.com/api/trans/raw/m/02cpny7", "name": "Basset-horn", "thumbnail": "http://indextank.com/_static/common/demo/02cpny7.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/basset-horn", "categories": {"family": "Clarinet"}} +{"fields": {"url": "http://freebase.com/view/en/russian_guitar", "text": "The Russian guitar (sometimes referred to as a \"Gypsy guitar\") is a seven-string acoustic guitar that arrived in Russia toward the end of the 18th century and the beginning of the 19th century, most probably as an evolution of the cittern, kobza, and torban. It is known in Russian as the semistrunnaya gitara (\u0441\u0435\u043c\u0438\u0441\u0442\u0440\u0443\u043d\u043d\u0430\u044f \u0433\u0438\u0442\u0430\u0440\u0430), or affectionately as the semistrunka (\u0441\u0435\u043c\u0438\u0441\u0442\u0440\u0443\u043d\u043a\u0430), which translates to \"seven-string\". These guitars are typically tuned to an Open G chord as follows: DGBdgbd'....", "image": "http://img.freebase.com/api/trans/raw/m/03s3wln", "name": "Russian guitar", "thumbnail": "http://indextank.com/_static/common/demo/03s3wln.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/russian_guitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/organ", "text": "The organ (from Greek \u03cc\u03c1\u03b3\u03b1\u03bd\u03bf\u03bd organon, \"organ, instrument, tool\"), is a keyboard instrument of one or more divisions, each played with its own keyboard operated either with the hands or with the feet. The organ is a relatively old musical instrument in the Western musical tradition, dating from the time of Ctesibius of Alexandria who is credited with the invention of the hydraulis. By around the 8th century it had overcome early associations with gladiatorial combat and gradually assumed a...", "image": "http://img.freebase.com/api/trans/raw/m/029ffyk", "name": "Organ", "thumbnail": "http://indextank.com/_static/common/demo/029ffyk.jpg"}, "variables": {"0": 351}, "docid": "http://freebase.com/view/en/organ", "categories": {"family": "Keyboard instrument"}} +{"fields": {"url": "http://freebase.com/view/en/hammond_organ", "text": "The Hammond organ is an electric organ invented by Laurens Hammond in 1934 and manufactured by the Hammond Organ Company. While the Hammond organ was originally sold to churches as a lower-cost alternative to the wind-driven pipe organ, in the 1960s and 1970s it became a standard keyboard instrument for jazz, blues, rock music, church and gospel music.\nThe original Hammond organ used additive synthesis of waveforms from harmonic series made by mechanical tonewheels that rotate in front of...", "image": "http://img.freebase.com/api/trans/raw/m/04rrh5l", "name": "Hammond organ", "thumbnail": "http://indextank.com/_static/common/demo/04rrh5l.jpg"}, "variables": {"0": 200}, "docid": "http://freebase.com/view/en/hammond_organ", "categories": {"family": "Electronic organ"}} +{"fields": {"url": "http://freebase.com/view/en/morin_khuur", "text": "The morin khuur (Mongolian: \u043c\u043e\u0440\u0438\u043d \u0445\u0443\u0443\u0440) is a traditional Mongolian bowed stringed instrument. It is one of the most important musical instruments of the Mongolian people, and is considered a symbol of the Mongolian nation. The morin khuur is one of the Masterpieces of the Oral and Intangible Heritage of Humanity identified by UNESCO. It produces a sound which is poetically described as expansive and unrestrained, like a wild horse neighing, or like a breeze in the grasslands.\nThe full...", "image": "http://img.freebase.com/api/trans/raw/m/042d2j2", "name": "Morin khuur", "thumbnail": "http://indextank.com/_static/common/demo/042d2j2.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/morin_khuur", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/fender_precision_bass", "text": "The Fender Precision Bass (often shortened to \"P Bass\") is an Electric bass.\nDesigned by Leo Fender as a prototype in 1950 and brought to market in 1951, the Precision was the first bass to earn widespread attention and use. A revolutionary instrument for the time, the Precision Bass has made an immeasurable impact on the sound of popular music ever since.\nAlthough the Precision was the first mass-produced and widely-used bass, it was not the first model of the instrument, as is sometimes...", "image": "http://img.freebase.com/api/trans/raw/m/02bc45s", "name": "Fender Precision Bass", "thumbnail": "http://indextank.com/_static/common/demo/02bc45s.jpg"}, "variables": {"0": 122}, "docid": "http://freebase.com/view/en/fender_precision_bass", "categories": {"family": "Bass guitar"}} +{"fields": {"url": "http://freebase.com/view/m/07fdrb", "text": "Pipe describes a number of musical instruments, historically referring to perforated wind instruments. The word is an onomatopoeia, and comes from the tone which can resemble that of a bird chirping.\nFipple flutes are found in many cultures around the world. Often with six holes, the shepherd's pipe is a common pastoral image. Shepherds often piped both to soothe the sheep and to amuse themselves. Modern manufactured six-hole folk pipes are referred to as pennywhistle or tin whistle. The...", "image": "http://img.freebase.com/api/trans/raw/m/02g57g5", "name": "Pipe", "thumbnail": "http://indextank.com/_static/common/demo/02g57g5.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/07fdrb", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/western_concert_flute", "text": "The Western concert flute is a transverse (side-blown) woodwind instrument made of metal or wood. It is the most common variant of the flute. A musician who plays the flute is called a flautist, flutist, or flute player.\nThis type of flute is used in many ensembles including concert bands, orchestras, flute ensembles, and occasionally jazz bands and big bands. Other flutes in this family include the piccolo, alto flute, bass flute, contrabass flute and double contrabass flute. Millions of...", "image": "http://img.freebase.com/api/trans/raw/m/02bspr7", "name": "Western concert flute", "thumbnail": "http://indextank.com/_static/common/demo/02bspr7.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/western_concert_flute", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/pipa", "text": "The pipa (Chinese: \u7435\u7436; pinyin: p\u00edp\u00e1) is a four-stringed Chinese musical instrument, belonging to the plucked category of instruments (\u5f39\u62e8\u4e50\u5668/\u5f48\u64a5\u6a02\u5668). Sometimes called the Chinese lute, the instrument has a pear-shaped wooden body with a varying number of frets ranging from 12\u201326. Another Chinese 4 string plucked lute is the liuqin, which looks like a smaller version of the pipa.\nThe pipa appeared in the Qin Dynasty (221 - 206 BCE) and was developed during the Han Dynasty. It is one of the most...", "image": "http://img.freebase.com/api/trans/raw/m/02f1b99", "name": "Pipa", "thumbnail": "http://indextank.com/_static/common/demo/02f1b99.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/pipa", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/oboe_da_caccia", "text": "The oboe da caccia (literally \"hunting oboe\" in Italian) is a double reed woodwind instrument in the oboe family, pitched a fifth below the oboe and used primarily in the Baroque period of European classical music. It has a curved tube and a brass bell, unusual for an oboe.\nIts range is close to that of the English horn\u2014that is, from the F below middle C (notated C4 but sounding F3) to the G above the treble staff (notated D6 but sounding G5). The oboe da caccia is thus a transposing...", "image": "http://img.freebase.com/api/trans/raw/m/02bykkl", "name": "Oboe da caccia", "thumbnail": "http://indextank.com/_static/common/demo/02bykkl.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/oboe_da_caccia", "categories": {"family": "Oboe"}} +{"fields": {"url": "http://freebase.com/view/en/gittern", "text": "The gittern was a relatively small, quill-plucked, gut strung instrument that originated around the 13th century and came to Europe via Moorish Spain. It was also called the quinterne in Germany, the guitarra in Spain, and the chitarra in Italy. A popular instrument with the minstrels and amateur musicians of the 14th century, the gittern eventually out-competed its rival, the citole. Soon after, its popularity began to fade, giving rise to the larger and more evocative lute and guitar.\nUp...", "image": "http://img.freebase.com/api/trans/raw/m/04rqgmm", "name": "Gittern", "thumbnail": "http://indextank.com/_static/common/demo/04rqgmm.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gittern", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/m/03kljf", "text": "A serpent is a bass wind instrument, descended from the cornett, and a distant ancestor of the tuba, with a mouthpiece like a brass instrument but side holes like a woodwind. It is usually a long cone bent into a snakelike shape, hence the name. The serpent is closely related to the cornett, although it is not part of the cornett family, due to the absence of a thumb hole. It is generally made out of wood, with walnut being a particularly popular choice. The outside is covered with dark...", "image": "http://img.freebase.com/api/trans/raw/m/02b7nx9", "name": "Serpent", "thumbnail": "http://indextank.com/_static/common/demo/02b7nx9.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/03kljf", "categories": {"family": "Wind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/tar", "text": "The t\u0101r (Persian: \u062a\u0627\u0631) is a long-necked, waisted Iranian instrument. It has been adopted by other cultures and countries like Azerbaijan, Armenia, Georgia, and other areas near the Caucasus region. The word tar ( \u062a\u0627\u0631') itself means \"string\" in Persian, though it might have the same meaning in languages influenced by Persian or any other branches of Iranian languages like Kurdish. Therefore, Tar is common amongst all the Iranian people as well as the territories that are named as Iranian...", "image": "http://img.freebase.com/api/trans/raw/m/02ff9bp", "name": "Tar", "thumbnail": "http://indextank.com/_static/common/demo/02ff9bp.jpg"}, "variables": {"0": 5}, "docid": "http://freebase.com/view/en/tar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/fipple", "text": "A fipple is a constricted mouthpiece common to many end-blown woodwind instruments, such as the tin whistle and the recorder. These instruments are known variously as fipple flutes, duct flutes, or tubular-ducted flutes.\nIn the accompanying illustration of the head of a recorder, the wooden fipple plug (A), with a \"ducted flue\" windway above it in the mouthpiece of the instrument, compresses the player's breath, so that it travels along the duct (B), called the \"windway\". Exiting from the...", "image": "http://img.freebase.com/api/trans/raw/m/02cy2yq", "name": "Fipple", "thumbnail": "http://indextank.com/_static/common/demo/02cy2yq.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/fipple", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/electric_guitar", "text": "An electric guitar is a guitar that uses the principle of electromagnetic induction to convert vibrations of its metal strings into electric signals. Since the generated signal is too weak to drive a loudspeaker, it is amplified before sending it to a loudspeaker. Since the output of an electric guitar is an electric signal, the signal may easily be altered using electronic circuits to add color to the sound. Often the signal is modified using effects such as reverb and distortion. Conceived...", "image": "http://img.freebase.com/api/trans/raw/m/03qw7_m", "name": "Electric guitar", "thumbnail": "http://indextank.com/_static/common/demo/03qw7_m.jpg"}, "variables": {"0": 344}, "docid": "http://freebase.com/view/en/electric_guitar", "categories": {"family": "Guitar"}} +{"fields": {"url": "http://freebase.com/view/en/soprillo", "text": "The sopranissimo or soprillo saxophone is the smallest member of the saxophone family. It is pitched in B\u266d, one octave above the soprano saxophone. Because of the difficulties in building such a small instrument\u2014the soprillo is 12\u00a0inches long, 13\u00a0inches with the mouthpiece\u2014it is only recently that a true sopranissimo saxophone been produced. The keywork only extends to a written high E\u266d (rather than F like most saxophones) and the upper octave key has to be placed in the mouthpiece.\nThe...", "image": "http://img.freebase.com/api/trans/raw/m/042vsmy", "name": "Soprillo", "thumbnail": "http://indextank.com/_static/common/demo/042vsmy.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/soprillo", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/esraj", "text": "The esraj (Bengali: \u098f\u09b8\u09cd\u09b0\u09be\u099c; Hindi: \u0907\u0938\u0930\u093e\u091c; also called israj) is a string instrument found in two forms throughout the north, central, and east regions of India. It is a young instrument by Indian terms, being only about 200 years old. The dilruba is found in the north, where it is used in religious music and light classical songs in the urban areas. Its name is translated as \"robber of the heart.\" The esraj is found in the east and central areas, particularly Bengal (Bangladesh and Indian...", "image": "http://img.freebase.com/api/trans/raw/m/078kmxh", "name": "Esraj", "thumbnail": "http://indextank.com/_static/common/demo/078kmxh.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/esraj", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/harmonium", "text": "A harmonium is a free-standing keyboard instrument similar to a reed organ. Sound is produced by air being blown through sets of free reeds, resulting in a sound similar to that of an accordion. The air is usually supplied by bellows operated by foot, hand, or knees.\nIn North America, the most common pedal-pumped free-reed keyboard instrument is known as the American Reed Organ, (or parlor organ, pump organ, cabinet organ, cottage organ, etc.) and along with the earlier melodeon, is operated...", "image": "http://img.freebase.com/api/trans/raw/m/02f2d3m", "name": "Harmonium", "thumbnail": "http://indextank.com/_static/common/demo/02f2d3m.jpg"}, "variables": {"0": 39}, "docid": "http://freebase.com/view/en/harmonium", "categories": {"family": "Keyboard instrument"}} +{"fields": {"url": "http://freebase.com/view/en/pandeiro", "text": "The pandeiro (Portuguese pronunciation:\u00a0[p\u0250\u0303\u02c8dej\u027eu]) is a type of hand frame drum.\nThere are two important distinctions between a pandeiro and the common tambourine. The tension of the head on the pandeiro can be tuned, allowing the player a choice of high and low notes. Also, the metal jingles (called platinelas in Portuguese) are cupped, creating a crisper, drier and less sustained tone on the pandeiro than on the tambourine. This provides clarity when swift, complex rhythms are played.\nIt...", "image": "http://img.freebase.com/api/trans/raw/m/02dl9k2", "name": "Pandeiro", "thumbnail": "http://indextank.com/_static/common/demo/02dl9k2.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/pandeiro", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/appalachian_dulcimer", "text": "The Appalachian dulcimer (or mountain dulcimer) is a fretted string instrument of the zither family, typically with three or four strings. It is native to the Appalachian region of the United States. The body extends the length of the fingerboard, and its fretting is generally diatonic.\nAlthough the Appalachian dulcimer appeared in regions dominated by Irish and Scottish settlement, the instrument has no known precedent in Ireland or Scotland. However, several diatonic fretted zithers exist...", "image": "http://img.freebase.com/api/trans/raw/m/02cwrh0", "name": "Appalachian dulcimer", "thumbnail": "http://indextank.com/_static/common/demo/02cwrh0.jpg"}, "variables": {"0": 24}, "docid": "http://freebase.com/view/en/appalachian_dulcimer", "categories": {"family": "Zither"}} +{"fields": {"url": "http://freebase.com/view/en/bass_flute", "text": "The bass flute is the bass member of the flute family. It is in the key of C, pitched one octave below the concert flute. Because of the length of its tube (approximately 146\u00a0cm), it is usually made with a \"J\" shaped head joint, which brings the embouchure hole within reach of the player. It is usually only used in flute choirs, as it is easily drowned out by other instruments of comparable register, such as the clarinet.\nPrior to the mid-20th century, the term \"bass flute\" was sometimes...", "image": "http://img.freebase.com/api/trans/raw/m/02bkpyl", "name": "Bass flute", "thumbnail": "http://indextank.com/_static/common/demo/02bkpyl.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/bass_flute", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/saxotromba", "text": "The saxotromba is a valved brasswind instrument invented by the Belgian instrument-maker Adolphe Sax around 1844. It was designed for the mounted bands of the French military, probably as a substitute for the French horn. The saxotrombas comprised a family of half-tube instruments of different pitches. By about 1867 the saxotromba was no longer being used by the French military, but specimens of various sizes continued to be manufactured until the early decades of the twentieth century,...", "image": "http://img.freebase.com/api/trans/raw/m/05k84bg", "name": "Saxotromba", "thumbnail": "http://indextank.com/_static/common/demo/05k84bg.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/saxotromba", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/double_contrabass_flute", "text": "The double contrabass flute (sometimes also called the octobass flute or subcontrabass flute) is the largest and lowest pitched metal flute in the world (the hyperbass flute has an even lower range, though it is made out of PVC pipes and wood). It is pitched in the key of C, three octaves below the concert flute (two octaves below the bass flute and one octave below the contrabass flute). Its lowest note is C1, one octave below the cello's lowest C. This note is relatively easy to play in...", "image": "http://img.freebase.com/api/trans/raw/m/02d5lrf", "name": "Double contrabass flute", "thumbnail": "http://indextank.com/_static/common/demo/02d5lrf.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/double_contrabass_flute", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/oud", "text": "The oud (Arabic: \u0639\u0648\u062f\u200e \u02bf\u016bd, plural:\u0623\u0639\u0648\u0627\u062f, a\u2018w\u0101d; Assyrian:\u0725\u0718\u0715 \u016bd, Persian: \u0628\u0631\u0628\u0637 barbat; Turkish: ud or ut; Greek: \u03bf\u03cd\u03c4\u03b9; Armenian: \u0578\u0582\u0564, Azeri: ud; Hebrew: \u05e2\u05d5\u05d3 ud\u200e; Somali: cuud or kaban) is a pear-shaped stringed instrument commonly used in North Africa (Chaabi, Egyptian music, Andalusian, ...) and Middle Eastern music. The modern oud and the European lute both descend from a common ancestor via diverging evolutionary paths. The oud is readily distinguished by its lack of frets and smaller...", "image": "http://img.freebase.com/api/trans/raw/m/02btxfq", "name": "Oud", "thumbnail": "http://indextank.com/_static/common/demo/02btxfq.jpg"}, "variables": {"0": 38}, "docid": "http://freebase.com/view/en/oud", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/saxtuba", "text": "The saxtuba is an obsolete valved brasswind instrument conceived by the Belgian instrument-maker Adolphe Sax around 1845. The design of the instrument was inspired by the ancient Roman cornu and tuba. The saxtubas, which comprised a family of half-tube and whole-tube instruments of varying pitches, were first employed in Fromental Hal\u00e9vy's opera Le Juif errant (The Wandering Jew) in 1852. Their only other public appearance of note was at a military ceremony on the Champ de Mars in Paris in...", "image": "http://img.freebase.com/api/trans/raw/m/05l4224", "name": "Saxtuba", "thumbnail": "http://indextank.com/_static/common/demo/05l4224.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/saxtuba", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/sitar", "text": "The sitar (pronounced /s\u026a\u02c8t\u0251\u02d0(r)/; Hindi: \u0938\u093f\u0924\u093e\u0930, Bengali: \u09b8\u09c7\u09a4\u09be\u09b0, Urdu: \u0633\u062a\u0627\u0631, Persian: \u0633\u06cc\u200c\u062a\u0627\u0631\u00a0; Hindustani pronunciation:\u00a0[\u02c8s\u026a.t\u032aa\u02d0r]) is a plucked stringed instrument predominantly used in Hindustani classical music, where it has been ubiquitous since the Middle Ages. It derives its resonance from sympathetic strings, a long hollow neck and a gourd resonating chamber.\nUsed throughout the Indian subcontinent, particularly in Northern India, Pakistan, and Bangladesh, the sitar became known in...", "image": "http://img.freebase.com/api/trans/raw/m/03tdmc0", "name": "Sitar", "thumbnail": "http://indextank.com/_static/common/demo/03tdmc0.jpg"}, "variables": {"0": 71}, "docid": "http://freebase.com/view/en/sitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/pipe_organ", "text": "The pipe organ is a musical instrument that produces sound by driving pressurized air (called wind) through pipes selected via a keyboard. Because each organ pipe produces a single pitch, the pipes are provided in sets called ranks, each of which has a common timbre and volume throughout the keyboard compass. Most organs have multiple ranks of pipes of differing timbre, pitch and loudness that the player can employ singly or in combination through the use of controls called stops.\nA pipe...", "image": "http://img.freebase.com/api/trans/raw/m/029ffyk", "name": "Pipe organ", "thumbnail": "http://indextank.com/_static/common/demo/029ffyk.jpg"}, "variables": {"0": 5}, "docid": "http://freebase.com/view/en/pipe_organ", "categories": {"family": "Organ"}} +{"fields": {"url": "http://freebase.com/view/en/washtub_bass", "text": "The washtub bass, or \"gutbucket\", is a stringed instrument used in American folk music that uses a metal washtub as a resonator. Although it is possible for a washtub bass to have four or more strings and tuning pegs, traditional washtub basses have a single string whose pitch is adjusted by pushing or pulling on a staff or stick to change the tension.\nThe washtub bass was used in jug bands that were popular in some African Americans communities in the early 1900s. In the 1950s, British...", "image": "http://img.freebase.com/api/trans/raw/m/02bkd72", "name": "Washtub bass", "thumbnail": "http://indextank.com/_static/common/demo/02bkd72.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/washtub_bass", "categories": {"family": "Other string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/requinto", "text": "The term requinto is used in both Spanish and Portuguese to mean a smaller, higher-pitched version of another instrument. Thus, there are requinto guitars, drums, and several wind instruments.\nRequinto was 19th century Spanish for \"little clarinet\". Today, the word requinto, when used in relation to a clarinet, refers to the E-flat clarinet, also known as requint in Valencian language.\nRequinto can also mean a high-pitched flute (akin to a piccolo), or the person who plays it. In Galicia,...", "image": "http://img.freebase.com/api/trans/raw/m/05lw32q", "name": "Requinto", "thumbnail": "http://indextank.com/_static/common/demo/05lw32q.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/requinto", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/eight_string_guitar", "text": "An eight-string guitar is a guitar with eight strings instead of the commonly used six strings. Such guitars are not as common as the six string variety, but are used by classical, jazz, and metal guitarists to expand the range of their instrument by adding two strings.\nThere are several variants of this instrument, one probably originating from Russia along with the seven string guitar variant in the 19th century. The eight string guitar has recently begun to gain popularity, notably among...", "image": "http://img.freebase.com/api/trans/raw/m/02dg4vf", "name": "Eight string guitar", "thumbnail": "http://indextank.com/_static/common/demo/02dg4vf.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/eight_string_guitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/rebab", "text": "The rebab (Arabic \u0627\u0644\u0631\u0628\u0627\u0628\u0629 or \u0631\u0628\u0627\u0628\u0629 - \"a bowed (instrument)\"), also rebap, rabab, rebeb, rababah, or al-rababa) is a type of string instrument so named no later than the 8th century and spread via Islamic trading routes over much of North Africa, the Middle East, parts of Europe, and the Far East. The bowed variety often has a spike at the bottom to rest on the ground, and is thus called a spike fiddle in certain areas, but there exist plucked versions like the kabuli rebab (sometimes...", "image": "http://img.freebase.com/api/trans/raw/m/041hf7c", "name": "Rebab", "thumbnail": "http://indextank.com/_static/common/demo/041hf7c.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/rebab", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/an_nguy_t", "text": "The \u0111\u00e0n nguy\u1ec7t (also called nguy\u1ec7t c\u1ea7m, \u0111\u00e0n k\u00ecm, moon lute, or moon guitar) is a two-stringed Vietnamese traditional musical instrument. It is used in both folk and classical music, and remains popular throughout Vietnam (although during the 20th century many Vietnamese musicians increasingly gravitated toward the acoustic and electric guitar).\nThe \u0111\u00e0n nguy\u1ec7t's strings, formerly made of twisted silk, are today generally made of nylon or fishing line. They are kept at a fairly low tension in...", "image": "http://img.freebase.com/api/trans/raw/m/02cwm30", "name": "\u0110\u00e0n nguy\u1ec7t", "thumbnail": "http://indextank.com/_static/common/demo/02cwm30.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/an_nguy_t", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/pump_organ", "text": "The pump organ is a version of the reed organ where the player maintains the air pressure needed for creating the sound in the free reeds by pumping pedals with their feet.\nThe portative organ is a miniature version of the pipe organ where air pressure is also maintained by pumping, but in this case by hand.", "image": "http://img.freebase.com/api/trans/raw/m/04rcd28", "name": "Pump organ", "thumbnail": "http://indextank.com/_static/common/demo/04rcd28.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/pump_organ", "categories": {"family": "Organ"}} +{"fields": {"url": "http://freebase.com/view/en/theorbo", "text": "A theorbo (Italian: tiorba, also tuorbe; French: th\u00e9orbe, German: Theorbe) is a plucked string instrument. As a name, theorbo signifies a number of long-necked lutes with second pegboxes, such as the liuto attiorbato, the French th\u00e9orbe des pi\u00e8ces, the English theorbo, the archlute, the German baroque lute, the ang\u00e9lique or angelica. The etymology of the name tiorba has not yet been explained. It is hypothesized that its origin might have been in the Slavic or Turkish \"torba\", meaning \"bag\"...", "image": "http://img.freebase.com/api/trans/raw/m/029kp2s", "name": "Theorbo", "thumbnail": "http://indextank.com/_static/common/demo/029kp2s.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/theorbo", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/huqin", "text": "Huqin (\u80e1\u7434; pinyin: h\u00faq\u00edn) is a family of bowed string instruments, more specifically, a spike fiddle popularly used in Chinese music. The instruments consist of a round, hexagonal, or octagonal sound box at the bottom with a neck attached that protrudes upwards. They also have two strings (except the sihu, which has four strings tuned in pairs) and their soundboxes are typically covered with either snakeskin (most often python) or thin wood. Huqin instruments have either two (or, more...", "image": "http://img.freebase.com/api/trans/raw/m/02cpkr7", "name": "Huqin", "thumbnail": "http://indextank.com/_static/common/demo/02cpkr7.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/huqin", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/northumbrian_smallpipes", "text": "The Northumbrian smallpipes (also known as the Northumbrian pipes) are bellows-blown bagpipes from the North East of England. In , a survey of the bagpipes in the Pitt Rivers Museum, Oxford University, the organologist Anthony Baines wrote: It is perhaps the most civilized of the bagpipes, making no attempt to go farther than the traditional bagpipe music of melody over drone, but refining this music to the last degree.\nThe instrument consists of one chanter (generally with keys) and usually...", "image": "http://img.freebase.com/api/trans/raw/m/02cyw01", "name": "Northumbrian smallpipes", "thumbnail": "http://indextank.com/_static/common/demo/02cyw01.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/northumbrian_smallpipes", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/gravikord", "text": "The gravikord is an electric double bridge-harp invented by Robert Grawi in 1986.\nThe gravikord is a new instrument developed on the basis of the West African kora. It is made of welded stainless steel tubing, with 24 nylon strings but no resonating gourd or skin. The bridge is made from a machined synthetic material with an integral piezo-electric sensor. Two handles located in elevation near the middle of the bridge allow holding the instrument. The bridge is curved to follow the arc of a...", "image": "http://img.freebase.com/api/trans/raw/m/0dh91c5", "name": "Gravikord", "thumbnail": "http://indextank.com/_static/common/demo/0dh91c5.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gravikord", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/auditorium_organ", "text": "The Boardwalk Hall Auditorium Organ is the pipe organ in the Main Auditorium of the Boardwalk Hall (formerly known as the Atlantic City Convention Hall) in Atlantic City, New Jersey, built by the Midmer-Losh Organ Company. It is the largest organ in the world, as measured by the number of pipes. The Wanamaker Grand Court Organ has more ranks.\nThe Boardwalk Hall is a very large building, with a total floor area of 41,000\u00a0sq\u00a0ft (3,800\u00a0m). Consequently, the organ runs on much higher wind...", "image": "http://img.freebase.com/api/trans/raw/m/02dyvgw", "name": "Auditorium Organ", "thumbnail": "http://indextank.com/_static/common/demo/02dyvgw.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/auditorium_organ", "categories": {"family": "Pipe organ"}} +{"fields": {"url": "http://freebase.com/view/en/zampogna", "text": "Zampogna is a generic term for a number of Italian double chantered pipes that can be found as far north as the southern part of the Marche, throughout areas in Abruzzo, Latium, Molise, Basilicata, Campania, Calabria, and Sicily. The tradition is now mostly associated with Christmas, and the most famous Italian carol, \"Tu scendi dalle stelle\" (You Come Down From the Stars) is derived from traditional zampogna music. However, there is an ongoing resurgence of the instrument in secular use...", "image": "http://img.freebase.com/api/trans/raw/m/041dfcz", "name": "Zampogna", "thumbnail": "http://indextank.com/_static/common/demo/041dfcz.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/zampogna", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/hurdy_gurdy", "text": "The hurdy gurdy or hurdy-gurdy (also known as a wheel fiddle) is a stringed musical instrument that produces sound by a crank-turned rosined wheel rubbing against the strings. The wheel functions much like a violin bow, and single notes played on the instrument sound similar to a violin. Melodies are played on a keyboard that presses tangents (small wedges, usually made of wood) against one or more of the strings to change their pitch. Like most other acoustic string instruments, it has a...", "image": "http://img.freebase.com/api/trans/raw/m/029fyb2", "name": "Hurdy gurdy", "thumbnail": "http://indextank.com/_static/common/demo/029fyb2.jpg"}, "variables": {"0": 5}, "docid": "http://freebase.com/view/en/hurdy_gurdy", "categories": {"family": "String instrument"}} +{"fields": {"url": "http://freebase.com/view/en/djembe", "text": "A djembe ( /\u02c8d\u0292\u025bmbe\u026a/ JEM-bay) also known as jembe, jenbe, djimbe, jymbe, yembe, or jimbay, or sanbanyi in Susu; is a skin-covered drum meant to be played with bare hands. According to the Bamana people in Mali, the name of the djembe comes directly from the saying \"Anke dj\u00e9, anke b\u00e9\" which translates to \"everyone gather together in peace\" and defines the drum's purpose. In the Bambara language, \"dj\u00e9\" is the verb for \"gather\" and \"b\u00e9\" translates as \"peace\".\nIt is a member of the...", "image": "http://img.freebase.com/api/trans/raw/m/02chnm5", "name": "Djembe", "thumbnail": "http://indextank.com/_static/common/demo/02chnm5.jpg"}, "variables": {"0": 9}, "docid": "http://freebase.com/view/en/djembe", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/picco_pipe", "text": "The picco pipe is the smallest form of ducted flue tabor pipe or flute-a-bec.\nIt is 3\u00bd\" long, with the windway taking up 1\u00bd\". It has only three holes: two in front and a dorsal thumb hole. It has the same mouthpiece as a recorder. The bore end hole of the picco Pipe has a small flare, and the lowest notes were played with a finger blocking this end.\nThe range is from b to c3, using the slight frequency shift between registers to sound a full chromatic scale, like the tabor pipe.\nIt was...", "image": "http://img.freebase.com/api/trans/raw/m/03rk1x4", "name": "Picco pipe", "thumbnail": "http://indextank.com/_static/common/demo/03rk1x4.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/picco_pipe", "categories": {"family": "Pipe"}} +{"fields": {"url": "http://freebase.com/view/en/yangqin", "text": "The trapezoidal yangqin (simplified Chinese: \u626c\u7434; traditional Chinese: \u63da\u7434; pinyin: y\u00e1ngq\u00edn) is a Chinese hammered dulcimer, originally from Central Asia (Persia (modern-day Iran)). It used to be written with the characters \u6d0b\u7434 (lit. \"foreign zither\"), but over time the first character changed to \u63da (also pronounced \"y\u00e1ng\"), which means \"acclaimed\". It is also spelled yang quin or yang ch'in. Hammered dulcimers of various types are now very popular not only in China, but also Eastern Europe, the...", "image": "http://img.freebase.com/api/trans/raw/m/02b47fj", "name": "Yangqin", "thumbnail": "http://indextank.com/_static/common/demo/02b47fj.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/yangqin", "categories": {"family": "Struck string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/street_organ", "text": "A street organ is a mechanical organ designed to play in the street. The operator of a street organ is called an organ grinder. The two main types are the smaller German street organ and the larger Dutch street organ.\nIn the United Kingdom, street organ is often used to refer to a mechanically played piano like instrument. This is incorrect, and such instruments are called a Barrel piano.\nDutch street organs (unlike the simple street organ) are large organs that play book music. They are...", "image": "http://img.freebase.com/api/trans/raw/m/02gf070", "name": "Street organ", "thumbnail": "http://indextank.com/_static/common/demo/02gf070.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/street_organ", "categories": {"family": "Mechanical organ"}} +{"fields": {"url": "http://freebase.com/view/en/grand_piano", "text": "A grand piano is the concert form of a piano. A grand piano has the frame and strings placed horizontally, with the strings extending away from the keyboard. Grand pianos are distinguished from upright piano, which have their strings and frame arranged vertically. \n\nGrand Pianos are typically used for concerts and concert hall performances, although a baby grand piano can also be used in a household where space is limited. The strings on a Grand Piano are longer, resulting in a louder and...", "image": "http://img.freebase.com/api/trans/raw/m/064dhsl", "name": "Grand piano", "thumbnail": "http://indextank.com/_static/common/demo/064dhsl.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/grand_piano", "categories": {"family": "Piano"}} +{"fields": {"url": "http://freebase.com/view/en/dabakan", "text": "The dabakan is a single-headed Philippine drum, primarily used as a supportive instrument in the kulintang ensemble. Among the five main kulintang instruments, it is the only non-gong element of the Maguindanao ensemble.\nThe dabakan is frequently described as either hour-glass, conical, tubular, or goblet in shape Normally, the dabakan is found having a length of more than two feet and a diameter of more than a foot about the widest part of the shell. The shell is carved from wood either...", "image": "http://img.freebase.com/api/trans/raw/m/0419j5r", "name": "Dabakan", "thumbnail": "http://indextank.com/_static/common/demo/0419j5r.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/dabakan", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/kacapi", "text": "Kacapi is a zither-like Sundanese musical instrument played as the main accompanying instrument in the Tembang Sunda or Mamaos Cianjuran, kacapi suling (tembang Sunda without vocal accompaniment) genre (called kecapi seruling in Indonesian), pantun stories recitation or an additional instrument in Gamelan Degung performance.\nWord kacapi in Sundanese also refers to santol tree, from which initially the wood is believed to be used for building the zither instrument.\nAccording to its form or...", "image": "http://img.freebase.com/api/trans/raw/m/029lrtt", "name": "Kacapi", "thumbnail": "http://indextank.com/_static/common/demo/029lrtt.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/kacapi", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/doshpuluur", "text": "The doshpuluur (Tuvan: \u0414\u043e\u0448\u043f\u0443\u043b\u0443\u0443\u0440) is a long-necked Tuvan lute made from wood, usually pine or larch. The doshpuluur is played by plucking and strumming. There are two different versions of the doshpuluur. One version has a trapezoidal soundbox, which is covered on both sides by goat skin and is fretless. The other has a kidney-shaped soundbox mostly of wood with a small goat or snake skin roundel on the front and has frets.\nTraditionally the instrument has only two strings, but there exist...", "image": "http://img.freebase.com/api/trans/raw/m/02ddh5l", "name": "Doshpuluur", "thumbnail": "http://indextank.com/_static/common/demo/02ddh5l.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/doshpuluur", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/piano", "text": "The piano is a musical instrument played by means of a keyboard. It is one of the most popular instruments in the world. Widely used in classical music for solo performances, ensemble use, chamber music and accompaniment, the piano is also very popular as an aid to composing and rehearsal. Although not portable and often expensive, the piano's versatility and ubiquity have made it one of the world's most familiar musical instruments.\nPressing a key on the piano's keyboard causes a...", "image": "http://img.freebase.com/api/trans/raw/m/03s9dxr", "name": "Piano", "thumbnail": "http://indextank.com/_static/common/demo/03s9dxr.jpg"}, "variables": {"0": 3420}, "docid": "http://freebase.com/view/en/piano", "categories": {"family": "Keyboard instrument"}} +{"fields": {"url": "http://freebase.com/view/en/bodhran", "text": "The bodhr\u00e1n ( /\u02c8b\u0254r\u0251\u02d0n/ or /\u02c8ba\u028ar\u0251\u02d0n/; plural bodhr\u00e1ns or bodhr\u00e1in) is an Irish frame drum ranging from 25 to 65\u00a0cm (10\" to 26\") in diameter, with most drums measuring 35 to 45\u00a0cm (14\" to 18\"). The sides of the drum are 9 to 20\u00a0cm (3\u00bd\" to 8\") deep. A goatskin head is tacked to one side (synthetic heads, or other animal skins are sometimes used). The other side is open ended for one hand to be placed against the inside of the drum head to control the pitch and timbre.\nOne or two crossbars,...", "image": "http://img.freebase.com/api/trans/raw/m/0292qs1", "name": "Bodhr\u00e1n", "thumbnail": "http://indextank.com/_static/common/demo/0292qs1.jpg"}, "variables": {"0": 14}, "docid": "http://freebase.com/view/en/bodhran", "categories": {"family": "Drum"}} +{"fields": {"url": "http://freebase.com/view/en/gadulka", "text": "The gadulka (Bulgarian: \u0413\u044a\u0434\u0443\u043b\u043a\u0430) is a traditional Bulgarian bowed string instrument. Alternate spellings are \"gudulka\" and \"g'dulka\". Its name comes from a root meaning \"to make noise, hum or buzz\". The gadulka is an integral part of Bulgarian traditional instrumental ensembles, commonly played in the context of dance music.\nThe gadulka commonly has three (occasionally four) main strings with up to ten sympathetic resonating strings underneath, although there is a smaller variant of the...", "image": "http://img.freebase.com/api/trans/raw/m/029qs02", "name": "Gadulka", "thumbnail": "http://indextank.com/_static/common/demo/029qs02.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/gadulka", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/kora", "text": "The kora is a 21-string bridge-harp used extensively by people in West Africa.\nA kora is built from a large calabash cut in half and covered with cow skin to make a resonator, and has a notched bridge like a lute or guitar. It does not fit well into any one category of western instruments and would have to be described as a double bridge harp lute. The sound of a kora resembles that of a harp, though when played in the traditional style, it bears a closer resemblance to flamenco and delta...", "image": "http://img.freebase.com/api/trans/raw/m/03s1l8p", "name": "Kora", "thumbnail": "http://indextank.com/_static/common/demo/03s1l8p.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/kora", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/oboe_damore", "text": "The oboe d'amore (oboe of love in Italian), less commonly oboe d'amour, is a double reed woodwind musical instrument in the oboe family. Slightly larger than the oboe, it has a less assertive and more tranquil and serene tone, and is considered the mezzo-soprano of the oboe family, between the oboe itself (soprano) and the cor anglais, or English horn (alto). It is a transposing instrument, sounding a minor third lower than it is notated, i.e. in A. The bell is pear-shaped and the instrument...", "image": "http://img.freebase.com/api/trans/raw/m/02d0q36", "name": "Oboe d'amore", "thumbnail": "http://indextank.com/_static/common/demo/02d0q36.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/oboe_damore", "categories": {"family": "Oboe"}} +{"fields": {"url": "http://freebase.com/view/en/viola_bastarda", "text": "Viola bastarda refers to a highly virtuosic style of composition or extemporaneous performance, as well as to the altered viols created to maximize players' ability to play in this style. In the viola bastarda style, a polyphonic composition is reduced to a single line, while maintaining the same range as the original, and adding divisions, improvisations, and new counterpoint. The style flourished in Italy in the late 16th and early 17th centuries. Francesco Rognoni, a prominent composer of...", "image": "http://img.freebase.com/api/trans/raw/m/02g_6t5", "name": "Viola bastarda", "thumbnail": "http://indextank.com/_static/common/demo/02g_6t5.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/viola_bastarda", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/water_organ", "text": "The water organ or hydraulic organ (early types are sometimes called hydraulis, hydraulos, hydraulus or hydraula) is a type of pipe organ blown by air, where the power source pushing the air is derived by water from a natural source (e.g. by a waterfall) or by a manual pump. Consequently, the water organ lacks a bellows, blower, or compressor.\nOn the water organ, since the 15th century, the water is also used as a source of power to drive a mechanism similar to that of the Barrel organ,...", "image": "http://img.freebase.com/api/trans/raw/m/042v8v8", "name": "Water organ", "thumbnail": "http://indextank.com/_static/common/demo/042v8v8.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/water_organ", "categories": {"family": "Organ"}} +{"fields": {"url": "http://freebase.com/view/en/heckelphone_clarinet", "text": "The heckelphone-clarinet (or Heckelphon-Klarinette) is a rare woodwind instrument, invented in 1907 by Wilhelm Heckel in Wiesbaden-Biebrich, Germany. Despite its name, it is essentially a wooden saxophone with wide conical bore, built of red-stained maple wood, overblowing the octave, and with clarinet-like fingerings. It has a single-reed mouthpiece attached to a short metal neck, similar to an alto clarinet. The heckelphone-clarinet is a transposing instrument in B\u266d with sounding range of...", "image": "http://img.freebase.com/api/trans/raw/m/02h26f8", "name": "Heckelphone-clarinet", "thumbnail": "http://indextank.com/_static/common/demo/02h26f8.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/heckelphone_clarinet", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/three_hole_pipe", "text": "The three-hole pipe, also commonly known as Tabor pipe is a wind instrument designed to be played by one hand, leaving the other hand free to play a tabor drum, bell, psalterium or tambourin \u00e0 cordes, bones, triangle or other percussive instrument.\nThe three-hole pipe's origins are not known, but it dates back at least to the 11th Century. \nIt was popular from an early date in France, the Iberian Peninsula and Great Britain and remains in use there today. In the Basque Country it has...", "image": "http://img.freebase.com/api/trans/raw/m/03rlw81", "name": "Three-hole pipe", "thumbnail": "http://indextank.com/_static/common/demo/03rlw81.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/three_hole_pipe", "categories": {"family": "Pipe"}} +{"fields": {"url": "http://freebase.com/view/en/natural_trumpet", "text": "A natural trumpet is a valveless brass instrument that is able to play the notes of the harmonic series.\nThe natural trumpet was used as a military instrument to facilitate communication (e.g. break camp, retreat, etc).\nEven before the early Baroque period the (natural) trumpet had been accepted into Western Art Music. There is evidence, for example, of extensive use of trumpet ensembles in Venetian ceremonial music of the 16th century. Although neither Andrea nor Giovanni Gabrieli wrote...", "image": "http://img.freebase.com/api/trans/raw/m/03s5hzv", "name": "Natural trumpet", "thumbnail": "http://indextank.com/_static/common/demo/03s5hzv.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/natural_trumpet", "categories": {"family": "Trumpet"}} +{"fields": {"url": "http://freebase.com/view/en/flugelhorn", "text": "The flugelhorn (also spelled fluegelhorn, flugel horn or fl\u00fcgelhorn; German: \"wing horn\") is a brass instrument resembling a trumpet but with a wider, conical bore. Some consider it to be a member of the saxhorn family developed by Adolphe Sax (who also developed the saxophone); however, other historians assert that it derives from the keyed bugle designed by Michael Saurle (father), Munich 1832 (Royal Bavarian privilege for a \"chromatic Fl\u00fcgelhorn\" 1832), thus predating Adolphe Sax's...", "image": "http://img.freebase.com/api/trans/raw/m/02914lg", "name": "Flugelhorn", "thumbnail": "http://indextank.com/_static/common/demo/02914lg.jpg"}, "variables": {"0": 36}, "docid": "http://freebase.com/view/en/flugelhorn", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/m/05zsb7k", "text": "Variants of the bock, a type of bagpipe, were played in Central Europe in what are the modern states of Austria, Germany, Poland and the Czech Republic. The tradition of playing the instrument endured into the 20th century, primarily in the Blata, Chodsko, and Egerland regions of Bohemia, and among the Sorbs of Saxony. The name \"Bock\" (German for buck, i.e. male goat) refers to the use of goatskins in constructing the bag, similar to the common use of other goat-terms for bagpipes in other...", "image": "http://img.freebase.com/api/trans/raw/m/0788zlv", "name": "Bock", "thumbnail": "http://indextank.com/_static/common/demo/0788zlv.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/05zsb7k", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/fretless_guitar", "text": "A fretless guitar is a guitar without frets. It operates in the same manner as most other stringed instruments and traditional guitars, but does not have any frets to act as the lower end point (node) of the vibrating string. On a fretless guitar, the vibrating string length runs from the bridge, where the strings are attached, all the way up to the point where the fingertip presses the string down on the fingerboard. Fretless guitars are fairly uncommon in most forms of western music and...", "image": "http://img.freebase.com/api/trans/raw/m/03qxl43", "name": "Fretless guitar", "thumbnail": "http://indextank.com/_static/common/demo/03qxl43.jpg"}, "variables": {"0": 5}, "docid": "http://freebase.com/view/en/fretless_guitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/yueqin", "text": "This article is about the Chinese Yuequin. The Vietnamese \u0110\u00e0n nguy\u1ec7t is also often referred to as a 'moon guitar'.\nThe yueqin (Chinese: \u6708\u7434, pinyin: yu\u00e8q\u00edn; also spelled yue qin, or yueh-ch'in; and also called moon guitar, moon-zither, gekkin, la ch'in, or laqin) is a traditional Chinese string instrument. It is a lute with a round, hollow wooden body which gives it the nickname moon guitar. It has a short fretted neck and four strings tuned in courses of two (each pair of strings is tuned to...", "image": "http://img.freebase.com/api/trans/raw/m/02ctnn9", "name": "Yueqin", "thumbnail": "http://indextank.com/_static/common/demo/02ctnn9.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/yueqin", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/alto_flute", "text": "The alto flute is a type of Western concert flute, a musical instrument in the woodwind family. It is the next extension downward of the C flute after the fl\u00fbte d'amour. It is characterized by its distinct, mellow tone in the lower portion of its range. It is a transposing instrument in G and, like the piccolo and bass flute, uses the same fingerings as the C flute.\nThe tube of the alto flute is considerably thicker and longer than a C flute and requires more breath from the player. This...", "image": "http://img.freebase.com/api/trans/raw/m/02dwyx3", "name": "Alto flute", "thumbnail": "http://indextank.com/_static/common/demo/02dwyx3.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/alto_flute", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/lyra_viol", "text": "The lyra viol is a small bass viol, used primarily in England in the seventeenth century.\nWhile the instrument itself differs little physically from the standard consort viol, there is a large and important repertoire which was developed specifically for the lyra viol. Due to the number of strings and their rather flat layout, the lyra viol can approximate polyphonic textures, and because of its small size and large range, it is more suited to intricate and quick melodic lines than the...", "image": "http://img.freebase.com/api/trans/raw/m/02gw6kc", "name": "Lyra viol", "thumbnail": "http://indextank.com/_static/common/demo/02gw6kc.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/lyra_viol", "categories": {"family": "Viol"}} +{"fields": {"url": "http://freebase.com/view/m/07_hbw", "text": "Setar (Persian: \u0633\u0647 \u200c\u062a\u0627\u0631, from seh, meaning \"three\" and t\u0101r, meaning \"string\") is a Persian musical instrument. It is a member of the lute family. Two and a half centuries ago, a fourth string was added to the setar, which has 25 - 27 moveable frets. It originated in Persia before the spread of Islam.", "image": "http://img.freebase.com/api/trans/raw/m/02c_r2z", "name": "Setar", "thumbnail": "http://indextank.com/_static/common/demo/02c_r2z.jpg"}, "variables": {"0": 10}, "docid": "http://freebase.com/view/m/07_hbw", "categories": {"family": "Lute"}} +{"fields": {"url": "http://freebase.com/view/en/alto_horn", "text": "The alto horn (US English; tenor horn in British English, Althorn in Germany; occasionally referred to as E\u266d horn) is a brass instrument pitched in E\u266d. It has a predominantly conical bore (most tube extents gradually widening), and normally uses a deep, cornet-like mouthpiece.\nIt is most commonly used in marching bands, brass bands and similar ensembles, whereas the horn tends to take the corresponding parts in symphonic groupings and classical brass ensembles.\nThe alto horn is a valved...", "image": "http://img.freebase.com/api/trans/raw/m/029hg8p", "name": "Alto horn", "thumbnail": "http://indextank.com/_static/common/demo/029hg8p.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/alto_horn", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/torupill", "text": "The torupill (literally 'pipe instrument'; also known as kitsepill, lootspill, kotepill) is a type of bagpipe from Estonia.\nIt is not clear when the bagpipe became established in Estonia. It may have arrived with the Germans, but an analysis of the bagpipe tunes in West and North Estonia also show a strong Swedish influence.\nThe instrument was known throughout Estonia. The bagpipe tradition was longest preserved in West and North Estonia where folk music retained archaic characteristics for...", "image": "http://img.freebase.com/api/trans/raw/m/063cblf", "name": "Torupill", "thumbnail": "http://indextank.com/_static/common/demo/063cblf.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/torupill", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/m/02wylsp", "text": "The baglamas (Greek \u03bc\u03c0\u03b1\u03b3\u03bb\u03b1\u03bc\u03ac\u03c2) or baglamadaki (Greek \u03bc\u03c0\u03b1\u03b3\u03bb\u03b1\u03bc\u03b1\u03b4\u03ac\u03ba\u03b9), a long necked bowl-lute, is a plucked string instrument used in Greek music; it is a version of the bouzouki pitched an octave higher (nominally D-A-D), with unison pairs on the four highest strings and an octave pair on the lower D. Musically, the baglamas is most often found supporting the bouzouki in the Piraeus style of rembetika.\nThe body is often hollowed out from a piece of wood (skaftos construction) or else made...", "image": "http://img.freebase.com/api/trans/raw/m/05kpl3f", "name": "Baglama", "thumbnail": "http://indextank.com/_static/common/demo/05kpl3f.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/02wylsp", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/soprano_clarinet", "text": "The soprano clarinets are a sub-family of the clarinet family. They include the most common types of clarinets, and indeed are often referred to as simply \"clarinets\".\nAmong the soprano clarinets are the B\u266d clarinet, the most common type, whose range extends from D below middle C (written E) to about the C three octaves above middle C; the A and C clarinets, sounding respectively a semitone lower and a whole tone higher than the B\u266d clarinet; and the low G clarinet, sounding yet a whole tone...", "image": "http://img.freebase.com/api/trans/raw/m/02bctx9", "name": "Soprano clarinet", "thumbnail": "http://indextank.com/_static/common/demo/02bctx9.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/soprano_clarinet", "categories": {"family": "Clarinet"}} +{"fields": {"url": "http://freebase.com/view/en/rudra_veena", "text": "The rudra veena (also spelled rudra vina, and also called been or bin; Hindi: \u0930\u0941\u0926\u094d\u0930\u0935\u0940\u0923\u093e) is a large plucked string instrument used in Hindustani classical music. It is an ancient instrument rarely played today. The rudra veena declined in popularity in part due to the introduction of the surbahar in the early 19th century which allowed sitarists to more easily present the alap sections of slow dhrupad-style ragas.\nThe rudra veena has a long tubular body with a length ranging between 54 and...", "image": "http://img.freebase.com/api/trans/raw/m/02ddhg6", "name": "Rudra veena", "thumbnail": "http://indextank.com/_static/common/demo/02ddhg6.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/rudra_veena", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/gaida", "text": "The gaida (bagpipe) is a musical instrument, aerophone, using enclosed reeds fed from a constant reservoir of air in the form of a bag.\nThe gaida, and its variations, is a traditional musical instrument for entire Europe, Northern Africa and the Middle East.\nThe several variations of gaida are in: Albania (Gajde) Czech Republic (bock), Romania (cimpoi), Croatia (diple and surle), Hungary (duda), Slovenia (dude), Poland (duda, gaidu and koza), Russia (mih, sahrb, volinka and shapar), Turkey...", "image": "http://img.freebase.com/api/trans/raw/m/02dtz_4", "name": "Gaida", "thumbnail": "http://indextank.com/_static/common/demo/02dtz_4.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/gaida", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/reclam_de_xeremies", "text": "The reclam de xeremies, also known as the xeremia bessona or xeremieta, is a double clarinet with two single reeds, traditionally found on the Balearic island of Ibiza, off the east coast of Spain.\nIt consists of two cane tubes of equal length, bound together by cord and small pieces of lead to stabilise the tubes. On each tube are several finger holes, traditionally four in the front and one on the back, though in modern instruments the back hole is often omitted. At the top end of each...", "image": "http://img.freebase.com/api/trans/raw/m/0786zyh", "name": "Reclam de xeremies", "thumbnail": "http://indextank.com/_static/common/demo/0786zyh.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/reclam_de_xeremies", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/chalumeau", "text": "This article is about the historical musical instrument. For the register on the clarinet that is named for this instrument, see Clarinet#Range.\nThe chalumeau (plural chalumeaux; from Greek: \u03ba\u03ac\u03bb\u03b1\u03bc\u03bf\u03c2, kalamos, meaning \"reed\") is a woodwind instrument of the late baroque and early classical era, in appearance rather like a recorder, but with a mouthpiece like a clarinet's.\nThe word \"chalumeau\" was in use in French from the twelfth century to refer to various sorts of pipes, some of which were...", "image": "http://img.freebase.com/api/trans/raw/m/07b05t1", "name": "Chalumeau", "thumbnail": "http://indextank.com/_static/common/demo/07b05t1.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/chalumeau", "categories": {"family": "Clarinet"}} +{"fields": {"url": "http://freebase.com/view/en/square_piano", "text": "The square piano is a piano that has horizontal strings arranged diagonally across the rectangular case above the hammers and with the keyboard set in the long side. It is variously attributed to Silbermann and Frederici and was improved by Petzold and Babcock. Built in quantity through the 1890s (in the United States), Steinway's celebrated iron framed over strung squares were more than two and a half times the size of Zumpe's wood framed instruments that were successful a century before....", "image": "http://img.freebase.com/api/trans/raw/m/02h0hp7", "name": "Square piano", "thumbnail": "http://indextank.com/_static/common/demo/02h0hp7.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/square_piano", "categories": {"family": "Piano"}} +{"fields": {"url": "http://freebase.com/view/en/gaita_asturiana", "text": "The gaita asturiana is a type of bagpipe native to the autonomous communities of Asturias and parts of Cantabria on the northern coast of Spain.\nThe first evidence for the existence of the gaita asturiana dates back to the 13th century, as a piper can be seen carved into the capital of the church of Santa Mar\u00eda de Villaviciosa. Further evidence includes an illumination of a rabbit playing the gaita in the 14th century text Llibru la regla colorada. An early carving of a wild boar playing the...", "image": "http://img.freebase.com/api/trans/raw/m/0638m3c", "name": "Gaita asturiana", "thumbnail": "http://indextank.com/_static/common/demo/0638m3c.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gaita_asturiana", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/yamaha_dx7", "text": "The Yamaha DX7 is an FM Digital Synthesizer manufactured by the Yamaha Corporation from 1983 to 1986. It was the first commercially successful digital synthesizer. Its distinctive sound can be heard on many recordings, especially Pop music from the 1980s. The DX7 was the moderately priced model of the DX series of FM keyboards that included DX9, the smaller DX100, DX11, and DX21 and the larger DX1 and DX5. Over 160,000 DX7s were made.\nTone generation in the DX7 is based on linear Frequency...", "image": "http://img.freebase.com/api/trans/raw/m/03sz4t7", "name": "Yamaha DX7", "thumbnail": "http://indextank.com/_static/common/demo/03sz4t7.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/yamaha_dx7", "categories": {"family": "Synthesizer"}} +{"fields": {"url": "http://freebase.com/view/en/rebec", "text": "The rebec (sometimes rebeck, and originally various other spellings) is a bowed string musical instrument. In its most common form, it has narrowboat shaped body, three strings and is played on the arm or under the chin, like a violin.\nThe rebec dates back to the Middle Ages and was particularly popular in the 15th and 16th centuries. The instrument is European and derived from the Arabic bowed instrument rebab and the Byzantine lyra. The rebec was first referred to by that name around the...", "image": "http://img.freebase.com/api/trans/raw/m/02b_96j", "name": "Rebec", "thumbnail": "http://indextank.com/_static/common/demo/02b_96j.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/rebec", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/tin_whistle", "text": "The tin whistle also called the penny whistle, English Flageolet, Scottish penny whistle, Tin Flageolet, Irish whistle and Clarke London Flageolet is a simple six-holed woodwind instrument. It is an end blown fipple flute flageolet, putting it in the same category as the recorder, American Indian flute, and other woodwind instruments. A tin whistle player is called a tin whistler or whistler. The tin whistle is closely associated with Celtic music.\nThe penny whistle in its modern form stems...", "image": "http://img.freebase.com/api/trans/raw/m/0292329", "name": "Tin whistle", "thumbnail": "http://indextank.com/_static/common/demo/0292329.jpg"}, "variables": {"0": 41}, "docid": "http://freebase.com/view/en/tin_whistle", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/an_gao", "text": "The \u0111\u00e0n g\u00e1o is a Vietnamese bowed string instrument with two strings. Its body is made from half of a coconut shell covered with wood, with a small seashell used as bridge. The instrument's name literally means \"coconut shell dipper string instrument\" (\u0111\u00e0n is the generic term for \"string instrument\" and g\u00e1o means \"coconut shell dipper\").\nThe \u0111\u00e0n g\u00e1o is closely related to a similar Chinese instrument, the yehu, and was likely introduced to Vietnam by Chaozhou or Cantonese immigrants. It is...", "image": "http://img.freebase.com/api/trans/raw/m/04p94q0", "name": "\u0110\u00e0n g\u00e1o", "thumbnail": "http://indextank.com/_static/common/demo/04p94q0.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/an_gao", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/basler_drum", "text": "The Basler drum is a kind of snare drum traditionally used in Switzerland for marching music, and notably at the Carnival of Basel.\nIt has a height of between 40 and 60\u00a0cm and a diameter of about 40\u00a0cm.", "image": "http://img.freebase.com/api/trans/raw/m/02d3fr0", "name": "Basler drum", "thumbnail": "http://indextank.com/_static/common/demo/02d3fr0.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/basler_drum", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/baritone_horn", "text": "The baritone horn is a member of the brass instrument family. The baritone horn has a predominantly cylindrical bore as do the trumpet and trombone. A baritone horn uses a large mouthpiece much like those of a trombone or euphonium. It is pitched in B\u266d, one octave below the B\u266d trumpet. In the UK the baritone is frequently found in brass bands. The baritone horn is also a common instrument in high school and college bands, as older baritones are often found in schools' inventories. However,...", "image": "http://img.freebase.com/api/trans/raw/m/0293f7p", "name": "Baritone horn", "thumbnail": "http://indextank.com/_static/common/demo/0293f7p.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/baritone_horn", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/temple_block", "text": "The temple block is a percussion instrument originating in China, Japan and Korea where it is used in religious ceremonies.\nIt is a carved hollow wooden instrument with a large slit. In its traditional form, the wooden fish, the shape is somewhat bulbous; modern instruments are also used which are rectangular in shape. Several blocks of varying sizes are often used together to give a variety of pitches. In Western music, their use can be traced back to early jazz drummers, and they are not...", "image": "http://img.freebase.com/api/trans/raw/m/0c52jjc", "name": "Temple block", "thumbnail": "http://indextank.com/_static/common/demo/0c52jjc.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/temple_block", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/rainstick", "text": "A rainstick is a long, hollow tube partially filled with small pebbles or beans, and has small pins or thorns arranged helically on its inside surface. When the stick is upended, the pebbles fall to the other end of the tube, making a sound reminiscent of rain falling. Rainsticks are often sold to tourists visiting parts of Latin America.\nThe rainstick is believed to have been invented in Chile or Peru, and was played in the belief that it could bring about rainstorms. It is also said that...", "image": "http://img.freebase.com/api/trans/raw/m/02d3lbb", "name": "Rainstick", "thumbnail": "http://indextank.com/_static/common/demo/02d3lbb.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/rainstick", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/ophicleide", "text": "The ophicleide (pronounced /\u02c8\u0252f\u0268kla\u026ad/) is a family of conical bore, brass keyed-bugles. It has a similar shape to the sudrophone.\nThe ophicleide was invented in 1817 and patented in 1821 by French instrument maker Jean Hilaire Ast\u00e9 (also known as Halary or Haleri) as an extension to the keyed bugle or Royal Kent bugle family. It was the structural cornerstone of the brass section of the Romantic orchestra, often replacing the serpent, a Renaissance instrument which was thought to be...", "image": "http://img.freebase.com/api/trans/raw/m/03s7lcd", "name": "Ophicleide", "thumbnail": "http://indextank.com/_static/common/demo/03s7lcd.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/ophicleide", "categories": {"family": "Wind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/keyed_trumpet", "text": "The keyed trumpet is a brass instrument that, contrary to the traditional valved trumpet, uses keys. The keyed trumpet is rarely seen in modern performances, but was relatively common up until the introduction of the valved trumpet in the early nineteenth century. Prior to the invention of the keyed trumpet, the prominent trumpet of the time was the natural trumpet.\nThe keyed trumpet has holes in the wall of the tube that are closed by keys. The experimental E\u266d keyed trumpet was not confined...", "image": "http://img.freebase.com/api/trans/raw/m/0633w7q", "name": "Keyed trumpet", "thumbnail": "http://indextank.com/_static/common/demo/0633w7q.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/keyed_trumpet", "categories": {"family": "Trumpet"}} +{"fields": {"url": "http://freebase.com/view/en/guqin", "text": "The guqin (simplified/traditional: \u53e4\u7434; pinyin: g\u01d4q\u00edn; Wades-Giles ku-ch'in; pronounced\u00a0[k\u00f9t\u0255\u02b0\u01d0n]\u00a0 ( listen); literally \"ancient stringed instrument\") is the modern name for a plucked seven-string Chinese musical instrument of the zither family. It has been played since ancient times, and has traditionally been favored by scholars and literati as an instrument of great subtlety and refinement, as highlighted by the quote \"a gentleman does not part with his qin or se without good reason,\" as...", "image": "http://img.freebase.com/api/trans/raw/m/02b89d9", "name": "Guqin", "thumbnail": "http://indextank.com/_static/common/demo/02b89d9.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/guqin", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/tubular_bell", "text": "Tubular bells (also known as chimes) are musical instruments in the percussion family. Each bell is a metal tube, 30\u201338 mm (1\u00bc\u20131\u00bd inches) in diameter, tuned by altering its length. Tubular bells are often replaced by studio chimes, which are a smaller and usually less expensive instrument. Studio chimes are similar in appearance to tubular bells, but each bell has a smaller diameter than the corresponding bell on tubular bells.\nTubular bells are typically struck on the top edge of the tube...", "image": "http://img.freebase.com/api/trans/raw/m/029g1m6", "name": "Tubular bell", "thumbnail": "http://indextank.com/_static/common/demo/029g1m6.jpg"}, "variables": {"0": 4}, "docid": "http://freebase.com/view/en/tubular_bell", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/vox_continental", "text": "The Vox Continental is a transistor-based combo organ that was introduced in 1962. Known for its thin, bright, breathy sound, the \"Connie,\" as it was affectionately known, was designed to be used by touring musicians. It was also designed to replace heavy tonewheel organs, such as the Hammond B3.\nWhile this was not entirely accomplished, the Continental was used in many 1960s hit singles, and was probably the most popular and best-known combo organ among major acts. Although phased out of...", "image": "http://img.freebase.com/api/trans/raw/m/03t79cq", "name": "Vox Continental", "thumbnail": "http://indextank.com/_static/common/demo/03t79cq.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/vox_continental", "categories": {"family": "Electronic organ"}} +{"fields": {"url": "http://freebase.com/view/en/baryton", "text": "The baryton is a bowed string instrument in the viol family, in regular use in Europe up until the end of the 18th century. In London a performance at Marylebone Gardens was announced in 1744, when Mr Ferrand was to perform on \"the Pariton, an instrument never played on in publick before.\" It most likely fell out of favor due to its immense difficulty to play. Its size is comparable to that of a violoncello; it has seven or sometimes six bowed strings of gut, plus ten sympathetic wire...", "image": "http://img.freebase.com/api/trans/raw/m/03t02nm", "name": "Baryton", "thumbnail": "http://indextank.com/_static/common/demo/03t02nm.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/baryton", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/electric_upright_bass", "text": "The electric upright bass (abbreviated EUB and sometimes also called stick bass) is an electronically amplified version of the double bass that has a minimal or 'skeleton' body, which greatly reduces the size and weight of the instrument. The EUB retains enough of the features of the double bass so that double bass players are comfortable performing on it. While the EUB retains some of the tonal characteristics of the double bass, its electrically-amplified nature also gives it its own...", "image": "http://img.freebase.com/api/trans/raw/m/03s1c6v", "name": "Electric upright bass", "thumbnail": "http://indextank.com/_static/common/demo/03s1c6v.jpg"}, "variables": {"0": 6}, "docid": "http://freebase.com/view/en/electric_upright_bass", "categories": {"family": "Double bass"}} +{"fields": {"url": "http://freebase.com/view/en/veena", "text": "Veena (also spelled 'vina', Sanskrit: \u0935\u0940\u0923\u093e (v\u012b\u1e47\u0101), Tamil: \u0bb5\u0bc0\u0ba3\u0bc8, Kannada: \u0cb5\u0cc0\u0ca3\u0cc6, Malayalam: \u0d35\u0d40\u0d23, Telugu: \u0c35\u0c40\u0c23) is a plucked stringed instrument used mostly in Carnatic Indian classical music. There are several variations of the veena, which in its South Indian form is a member of the lute family. One who plays the veena is referred to as a vainika.\nThe veena has a recorded history that dates back to the Vedic period (approximately 1500 BCE)\nIn ancient times, the tone vibrating from the hunter's...", "image": "http://img.freebase.com/api/trans/raw/m/02cz_w2", "name": "Veena", "thumbnail": "http://indextank.com/_static/common/demo/02cz_w2.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/veena", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/clarsach", "text": "Cl\u00e0rsach or Cl\u00e1irseach (depending on Scottish Gaelic or Irish spellings), is the generic Gaelic word for 'a harp', as derived from Middle Irish. In English, the word is used to refer specifically to a variety of small Irish and Scottish harps.\nThe use of this word in English, and the varieties of harps that it describes, is very complex and is a cause of arguments or disagreements between different groups of harp-lovers.\nBy and large, in English, the word cl\u00e0rsach is equivalent to the term...", "image": "http://img.freebase.com/api/trans/raw/m/02dp7mp", "name": "Cl\u00e0rsach", "thumbnail": "http://indextank.com/_static/common/demo/02dp7mp.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/clarsach", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/tamburitza", "text": "Tamburica (pronounced /t\u00e6m\u02c8b\u028a\u0259r\u026ats\u0259/ or /\u02cct\u00e6mb\u0259\u02c8r\u026ats\u0259/) or Tamboura (Croatian: Tamburica, Serbian: \u0422\u0430\u043c\u0431\u0443\u0440\u0438\u0446\u0430, Tamburica, meaning Little Tamboura, Hungarian: Tambura, Greek: \u03a4\u03b1\u03bc\u03c0\u03bf\u03c5\u03c1\u03ac\u03c2, sometimes written tamburrizza) refers to any member of a family of long-necked lutes popular in Eastern and Southern Europe, particularly Croatia (especially Slavonia), northern Serbia (Vojvodina) and Hungary. It is also known in southern Slovenia and Burgenland. All took their name and some characteristics...", "image": "http://img.freebase.com/api/trans/raw/m/04pmtc5", "name": "Tamburitza", "thumbnail": "http://indextank.com/_static/common/demo/04pmtc5.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/tamburitza", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/baroque_violin", "text": "A baroque violin is, in common usage, any violin whose neck, fingerboard, bridge, and tailpiece are of the type used during the baroque period. Such an instrument may be an original built during the baroque and never changed to modern form; or a modern replica built as a baroque violin; or an older instrument which has been converted (or re-converted) to baroque form. \"Baroque cellos\" and \"baroque violas\" also exist, with similar modifications made to their form.\nFollowing period practices,...", "image": "http://img.freebase.com/api/trans/raw/m/02bxp8b", "name": "Baroque violin", "thumbnail": "http://indextank.com/_static/common/demo/02bxp8b.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/baroque_violin", "categories": {"family": "Violin"}} +{"fields": {"url": "http://freebase.com/view/en/cavaquinho", "text": "The cavaquinho (pronounced [kav\u0250\u02c8ki\u0272u] in Portuguese) is a small string instrument of the European guitar family with four wire or gut strings. It is also called machimbo, machim, machete (in the Portuguese Atlantic islands and Brazil), manchete or marchete, braguinha or braguinho, or cavaco.\nThe most common tuning is D-G-B-D (from lower to higher pitches); other tunings include D-A-B-E (Portuguese ancient tuning, made popular by Julio Pereira) and G-G-B-D and A-A-C#-E. Guitarists often use...", "image": "http://img.freebase.com/api/trans/raw/m/02d9kx8", "name": "Cavaquinho", "thumbnail": "http://indextank.com/_static/common/demo/02d9kx8.jpg"}, "variables": {"0": 4}, "docid": "http://freebase.com/view/en/cavaquinho", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/sarinda", "text": "A sarinda is a stringed Indian folk musical instrument similar to lutes or fiddles. It is played with a bow and has three strings. The bottom part of the front of its hollow wooden soundbox is covered with animal skin. It is played while sitting on the ground in a vertical orientation.\nThe sarinda seems to have its origin in tribal fiddle instruments called \"dhodro banam\" found throughout in central, north-western and eastern India. It is an important instrument in the culture of the...", "image": "http://img.freebase.com/api/trans/raw/m/02fwz73", "name": "Sarinda", "thumbnail": "http://indextank.com/_static/common/demo/02fwz73.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/sarinda", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/zil", "text": "Zills, also zils or finger cymbals, (from Turkish zil, \"cymbals\" ) are tiny metallic cymbals used in belly dancing and similar performances. They are called s\u0101j\u0101t (\u0635\u0627\u062c\u0627\u062a) in Arabic. They are similar to Tibetan tingsha bells.\nA set of zills consists of four cymbals, two for each hand. Modern zills come in a range of sizes, the most common having a diameter of about 5\u00a0cm (2\u00a0in). Different sizes and shapes of zills will produce sounds that differ in volume, tone and resonance. For instance, a...", "image": "http://img.freebase.com/api/trans/raw/m/03td8pl", "name": "Zil", "thumbnail": "http://indextank.com/_static/common/demo/03td8pl.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/zil", "categories": {"family": "Cymbal"}} +{"fields": {"url": "http://freebase.com/view/en/monochord", "text": "A monochord is an ancient musical and scientific laboratory instrument. The word \"monochord\" comes from the Greek and means literally \"one string.\" A misconception of the term lies within its name. Often a monochord has more than one string, most of the time two, one open string and a second string with a movable bridge. In a basic monochord, a single string is stretched over a sound box. The string is fixed at both ends while one or many movable bridges are manipulated to demonstrate...", "image": "http://img.freebase.com/api/trans/raw/m/02cx4g8", "name": "Monochord", "thumbnail": "http://indextank.com/_static/common/demo/02cx4g8.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/monochord", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/marimba", "text": "The marimba ( pronunciation (help\u00b7info)) (also: Marimbaphone) is a musical instrument in the percussion family. Keys or bars (usually made of wood) are struck with mallets to produce musical tones. The keys are arranged as those of a piano, with the accidentals raised vertically and overlapping the natural keys (similar to a piano) to aid the performer both visually and physically.\nThe chromatic marimba was developed in southern Mexico and northern Guatemala from the diatonic marimba, an...", "image": "http://img.freebase.com/api/trans/raw/m/03sw9r1", "name": "Marimba", "thumbnail": "http://indextank.com/_static/common/demo/03sw9r1.jpg"}, "variables": {"0": 43}, "docid": "http://freebase.com/view/en/marimba", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/ajaeng", "text": "The ajaeng is a Korean string instrument. It is a wide zither with strings made of twisted silk, played by means of a slender stick made of forsythia wood, which is scraped against the strings in the manner of a bow. The original version of the instrument, and that used in court music (called the jeongak ajaeng), has seven strings, while the ajaeng used for sanjo and sinawi (called the sanjo ajaeng) has eight strings; some instruments may have up to nine strings.\nThe ajaeng is generally...", "image": "http://img.freebase.com/api/trans/raw/m/044jqcy", "name": "Ajaeng", "thumbnail": "http://indextank.com/_static/common/demo/044jqcy.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/ajaeng", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/cymbal", "text": "Cymbals are a common percussion instrument. Cymbals consist of thin, normally round plates of various alloys; see cymbal making for a discussion of their manufacture. The greater majority of cymbals are of indefinite pitch, although small disc-shaped cymbals based on ancient designs sound a definite note (see: crotales). Cymbals are used in many ensembles ranging from the orchestra, percussion ensembles, jazz bands, heavy metal bands, and marching groups. Drum kits usually incorporate a...", "image": "http://img.freebase.com/api/trans/raw/m/0290zn9", "name": "Cymbal", "thumbnail": "http://indextank.com/_static/common/demo/0290zn9.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/cymbal", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/bass_oboe", "text": "The bass oboe or baritone oboe is a double reed instrument in the woodwind family. It is about twice the size of a regular (soprano) oboe and sounds an octave lower; it has a deep, full tone not unlike that of its higher-pitched cousin, the English horn. The bass oboe is notated in the treble clef, sounding one octave lower than written. Its lowest note is B2 (in scientific pitch notation), one octave and a semitone below middle C, although an extension may be inserted between the lower...", "image": "http://img.freebase.com/api/trans/raw/m/041gwbz", "name": "Bass oboe", "thumbnail": "http://indextank.com/_static/common/demo/041gwbz.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/bass_oboe", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/shamisen", "text": "The shamisen or samisen (\u4e09\u5473\u7dda, literally \"three flavor strings\"), also called sangen (\u4e09\u7d43, literally \"three strings\") is a three-stringed musical instrument played with a plectrum called a bachi. The pronunciation in Japanese is usually \"shamisen\" (in western Japan, and often in Edo-period sources \"samisen\") but sometimes \"jamisen\" when used as a suffix (e.g., Tsugaru-jamisen).\nThe shamisen is similar in length to a guitar, but its neck is much much slimmer and has no frets. Its drum-like...", "image": "http://img.freebase.com/api/trans/raw/m/02bg242", "name": "Shamisen", "thumbnail": "http://indextank.com/_static/common/demo/02bg242.jpg"}, "variables": {"0": 4}, "docid": "http://freebase.com/view/en/shamisen", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/stroh_violin", "text": "A Stroh violin, Stro(h)viol, violinophone, or horn-violin is a violin that amplifies its sound through a metal resonator and metal horns rather than a wooden sound box as on a standard violin. The instrument is named after its German designer, Johannes Matthias Augustus Stroh, who patented it in 1899. The Stroh violin is also closely related to other horned violins using a mica sheet resonating diaphragm known as Phonofiddles.\nIn the present day, many types of horn-violin exist, especially...", "image": "http://img.freebase.com/api/trans/raw/m/05mbh5x", "name": "Stroh violin", "thumbnail": "http://indextank.com/_static/common/demo/05mbh5x.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/stroh_violin", "categories": {"family": "Violin"}} +{"fields": {"url": "http://freebase.com/view/en/nose_flute", "text": "The nose flute is a popular musical instrument played in Polynesia and the Pacific Rim countries. Other versions are found in Africa, China, and India.\nIn the North Pacific, in the Hawaiian islands the nose flute was a common courting instrument. In Hawaiian, it is variously called hano, \"nose flute,\" (Pukui and Elbert 1986), by the more specific term 'ohe hano ihu, \"bamboo flute [for] nose,\" or `ohe hanu `ihu, \"bamboo [for] nose breath\" (Nona Beamer lectures).\nIt is made from a single...", "image": "http://img.freebase.com/api/trans/raw/m/03sxt9m", "name": "Nose flute", "thumbnail": "http://indextank.com/_static/common/demo/03sxt9m.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/nose_flute", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/konghou", "text": "The konghou (Chinese: \u7b9c\u7bcc; pinyin: k\u014dngh\u00f3u) is an ancient Chinese harp. The konghou, also known as kanhou, went extinct sometime in the Ming Dynasty, but was revived in the 20th century. The modern version of the instrument does not resemble the ancient one.\nThe main feature that distinguishes the contemporary konghou from the Western concert harp is that the modern konghou's strings are folded over to make two rows, which enables players to use advanced playing techniques such as vibrato and...", "image": "http://img.freebase.com/api/trans/raw/m/03sgm8f", "name": "Konghou", "thumbnail": "http://indextank.com/_static/common/demo/03sgm8f.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/konghou", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/cello", "text": "The cello (pronounced /\u02c8t\u0283\u025blo\u028a/ CHEL-oh; plural cellos or celli) is a bowed string instrument with four strings tuned in perfect fifths. It is a member of the violin family of musical instruments, which also includes the violin, viola and the contrabass.\nThe word derives from the Italian 'violoncello'. The word derives ultimately from vitula, meaning a stringed instrument. A person who plays a cello is called a cellist. The cello is used as a solo instrument, in chamber music, in a string...", "image": "http://img.freebase.com/api/trans/raw/m/0290__v", "name": "Violoncello", "thumbnail": "http://indextank.com/_static/common/demo/0290__v.jpg"}, "variables": {"0": 274}, "docid": "http://freebase.com/view/en/cello", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/buzuq", "text": "The buzuq (Arabic: \u0628\u0632\u0642\u200e; also transliterated bozuq, bouzouk, buzuk etc.) is a long-necked fretted lute related to the Greek bouzouki and Turkish saz. It is an essential instrument in the Rahbani repertoire, but it is not classified among the classical instruments of Arab or Turkish music. However, this instrument may be looked upon as a larger and deeper-toned relative of the saz, to which it could be compared in the same way as the viola to the violin in Western music. Before the Rahbanis...", "image": "http://img.freebase.com/api/trans/raw/m/04p8wkb", "name": "Buzuq", "thumbnail": "http://indextank.com/_static/common/demo/04p8wkb.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/buzuq", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/pipe_organ_of_the_lds_conference_center", "text": "The Schoenstein Organ at the Conference Center is a pipe organ built by Schoenstein & Co., San Francisco, California located in the Conference Center of The Church of Jesus Christ of Latter-day Saints in Salt Lake City, Utah. The organ was completed in 2003. It is composed of five manuals and pedal. Along with the nearby Salt Lake Tabernacle organ, it is typically used to accompany the Mormon Tabernacle Choir.\nThe Conference Center organ is heard semi-annually each year at the Church\u2019s...", "image": "http://img.freebase.com/api/trans/raw/m/08bc354", "name": "Pipe organ of the LDS Conference Center", "thumbnail": "http://indextank.com/_static/common/demo/08bc354.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/pipe_organ_of_the_lds_conference_center", "categories": {"family": "Pipe organ"}} +{"fields": {"url": "http://freebase.com/view/en/cittern", "text": "The cittern or cither is a stringed instrument dating from the Renaissance. Modern scholars debate its exact history, but it is generally accepted that it is descended from the Medieval Citole, or Cytole. It looks much like the modern-day flat-back mandolin and the modern Irish bouzouki and cittern. Its flat-back design was simpler and cheaper to construct than the lute. It was also easier to play, smaller, less delicate and more portable. Played by all classes, the cittern was a premier...", "image": "http://img.freebase.com/api/trans/raw/m/02ch7s3", "name": "Cittern", "thumbnail": "http://indextank.com/_static/common/demo/02ch7s3.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/cittern", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/chord_organ", "text": "A chord organ is an organ (a free-reed musical instrument), similar to a small reed organ, in which sound is produced by the flow of air, usually driven by an electric motor, over plastic or metal reeds. Much like the accordion, the chord organ has both a keyboard and a set of chord buttons, enabling the musician to play a melody or lead with one hand and accompanying chords with the other. Chord organs were generally designed as toys, like those made by the Magnus Harmonica Corporation and...", "image": "http://img.freebase.com/api/trans/raw/m/03s8cq2", "name": "Chord organ", "thumbnail": "http://indextank.com/_static/common/demo/03s8cq2.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/chord_organ", "categories": {"family": "Organ"}} +{"fields": {"url": "http://freebase.com/view/en/saz", "text": "Saz can be a nickname for the given name Sarah, or may refer to:", "image": "http://img.freebase.com/api/trans/raw/m/03s6v1m", "name": "Saz", "thumbnail": "http://indextank.com/_static/common/demo/03s6v1m.jpg"}, "variables": {"0": 4}, "docid": "http://freebase.com/view/en/saz", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/clavinet", "text": "A Clavinet is an electrophonic keyboard instrument manufactured by the Hohner company. It is essentially an electronically amplified clavichord, analogous to an electric guitar. Its distinctive bright staccato sound has appeared particularly in funk, disco, rock, and reggae songs.\nVarious models were produced over the years, including the models I, II, L, C, D6, and E7. Most models consist of 60 keys and 60 associated strings, giving it a five-octave range from F1 to E6.\nEach key uses a...", "image": "http://img.freebase.com/api/trans/raw/m/029s8y_", "name": "Clavinet", "thumbnail": "http://indextank.com/_static/common/demo/029s8y_.jpg"}, "variables": {"0": 22}, "docid": "http://freebase.com/view/en/clavinet", "categories": {"family": "Clavichord"}} +{"fields": {"url": "http://freebase.com/view/en/paraguayan_harp", "text": "The Paraguayan harp is the national instrument of Paraguay, and similar instruments are used elsewhere in South America, particularly Venezuela.\nIt is a diatonic harp with 32, 36, 38 or 40 strings, made from tropical wood, with an exaggerated neck-arch, played with the fingernail. It accompanies songs in the Guarani language.", "image": "http://img.freebase.com/api/trans/raw/m/07s_y8b", "name": "Paraguayan harp", "thumbnail": "http://indextank.com/_static/common/demo/07s_y8b.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/paraguayan_harp", "categories": {"family": "Harp"}} +{"fields": {"url": "http://freebase.com/view/en/timbales", "text": "Timbales (or pailas criollas) are shallow single-headed drums with metal casing, invented in Cuba. They are shallower in shape than single-headed tom-toms, and usually much higher tuned. The player (known as a timbalero) uses a variety of stick strokes, rim shots, and rolls on the skins to produce a wide range of percussive expression during solos and at transitional sections of music, and usually plays the shells of the drum or auxiliary percussion such as a cowbell or cymbal to keep time...", "image": "http://img.freebase.com/api/trans/raw/m/041l8m6", "name": "Timbales", "thumbnail": "http://indextank.com/_static/common/demo/041l8m6.jpg"}, "variables": {"0": 8}, "docid": "http://freebase.com/view/en/timbales", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/post_horn", "text": "The post horn (also posthorn, post-horn, or coach horn) is a valveless cylindrical brass or copper instrument with cupped mouthpiece, used to signal the arrival or departure of a post rider or mail coach. It was used especially by postilions of the 18th and 19th centuries.\nThe instrument commonly had a circular or coiled shape with three turns of the tubing, though sometimes it was straight. It is therefore an example of a natural instrument. The cornet was developed from the post horn by...", "image": "http://img.freebase.com/api/trans/raw/m/02g8_mb", "name": "Post horn", "thumbnail": "http://indextank.com/_static/common/demo/02g8_mb.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/post_horn", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/division_viol", "text": "The division viol is an English type of bass viol, which was originally popular in the mid-17th century, but is currently experiencing a renaissance of its own due to the movement for historically informed performance. John Playford mentions the division viol in his A Brief Introduction of 1667, describing it as smaller than a consort bass viol, but larger than a lyra viol.\nAs suggested by its name, (divisions were a type of variations), the division viol is intended for highly ornamented...", "image": "http://img.freebase.com/api/trans/raw/m/02g_6ql", "name": "Division viol", "thumbnail": "http://indextank.com/_static/common/demo/02g_6ql.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/division_viol", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/bass_trumpet", "text": "The bass trumpet is a type of low trumpet which was first developed during the 1820s in Germany. It is usually pitched in 8' C or 9' B\u266d today, but is sometimes built in E\u266d and is treated as a transposing instrument sounding either an octave, a sixth or a ninth lower than written, depending on the pitch of the instrument. Although almost identical in length to the trombone, the bass trumpet possesses a tone which is harder and more metallic than that of the trombone. Although it has valves...", "image": "http://img.freebase.com/api/trans/raw/m/02937mn", "name": "Bass trumpet", "thumbnail": "http://indextank.com/_static/common/demo/02937mn.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/bass_trumpet", "categories": {"family": "Trumpet"}} +{"fields": {"url": "http://freebase.com/view/en/plucked_string_instrument", "text": "Plucked string instruments are a subcategory of string instruments that are played by plucking the strings. Plucking is a way of pulling and releasing the string in such as way as to give it an impulse that causes the string to vibrate. Plucking can be done with either a finger or a plectrum.\nMost plucked string instruments belong to the lute family (such as guitar, bass guitar, mandolin, banjo, balalaika, sitar, pipa, etc.), which generally consist of a resonating body, and a neck; the...", "image": "http://img.freebase.com/api/trans/raw/m/02dd6jj", "name": "Plucked string instrument", "thumbnail": "http://indextank.com/_static/common/demo/02dd6jj.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/plucked_string_instrument", "categories": {"family": "String instrument"}} +{"fields": {"url": "http://freebase.com/view/en/castanet", "text": "Castanets are percussion instrument (idiophone), used in Moorish, Ottoman, ancient Roman, Italian, Spanish, Portuguese, Latin American music, and Irish Folk Music. The instrument consists of a pair of concave shells joined on one edge by string. These are held in the hand and used to produce clicks for rhythmic accents or a ripping or rattling sound consisting of a rapid series of clicks. They are traditionally made of hardwood, although fibreglass is becoming increasingly popular.\nIn...", "image": "http://img.freebase.com/api/trans/raw/m/03tpmfd", "name": "Castanet", "thumbnail": "http://indextank.com/_static/common/demo/03tpmfd.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/castanet", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/bouzouki", "text": "The bouzouki (Greek \u03c4\u03bf \u03bc\u03c0\u03bf\u03c5\u03b6\u03bf\u03cd\u03ba\u03b9; pl. \u03c4\u03b1 \u03bc\u03c0\u03bf\u03c5\u03b6\u03bf\u03cd\u03ba\u03b9\u03b1) (plural sometimes transliterated as bouzoukia) is a musical instrument in the lute family, with a pear-shaped body and a long neck. A mainstay of modern Greek music, the front of the body is flat and is usually heavily inlaid with mother-of-pearl. The instrument is played with a plectrum and has a sharp metallic sound, reminiscent of a mandolin but pitched lower.\nThere are two main types of bouzouki:\nIn Greece, there had been an instrument...", "image": "http://img.freebase.com/api/trans/raw/m/029gy47", "name": "Bouzouki", "thumbnail": "http://indextank.com/_static/common/demo/029gy47.jpg"}, "variables": {"0": 25}, "docid": "http://freebase.com/view/en/bouzouki", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/suspended_cymbal", "text": "A suspended cymbal is any single cymbal played with a stick or beater rather than struck against another cymbal. A common abbreviation used is sus. cym., or sus. cymb. (with, or without the period).\nThe term comes from the modern orchestra, in which the term cymbals normally refers to a pair of clash cymbals. The first suspended cymbals used in the modern orchestra were one of a pair of orchestral cymbals, supported by hanging it bell upwards by its strap. This technique is still used, at...", "image": "http://img.freebase.com/api/trans/raw/m/03tdbhm", "name": "Suspended cymbal", "thumbnail": "http://indextank.com/_static/common/demo/03tdbhm.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/suspended_cymbal", "categories": {"family": "Cymbal"}} +{"fields": {"url": "http://freebase.com/view/en/electric_violin", "text": "An electric violin is a violin equipped with an electronic output of its sound. The term most properly refers to an instrument purposely made to be electrified with built-in pickups, usually with a solid body. It can also refer to a violin fitted with an electric pickup of some type, although \"amplified violin\" or \"electro-acoustic violin\" are more accurate in that case.\nElectrically amplified violins have been used in one form or another since the 1920s; jazz and blues artist Stuff Smith is...", "image": "http://img.freebase.com/api/trans/raw/m/03sz_t5", "name": "Electric violin", "thumbnail": "http://indextank.com/_static/common/demo/03sz_t5.jpg"}, "variables": {"0": 41}, "docid": "http://freebase.com/view/en/electric_violin", "categories": {"family": "Violin"}} +{"fields": {"url": "http://freebase.com/view/en/oboe", "text": "The oboe (English pronunciation:\u00a0/\u02c8o\u028abo\u028a/) is a double reed musical instrument of the woodwind family. In English, prior to 1770, the instrument was called \"hautbois\" (French, meaning \"high wood\"), \"hoboy\", or \"French hoboy\". The spelling \"oboe\" was adopted into English ca. 1770 from the Italian obo\u00e8, a transliteration in that language's orthography of the 17th-century pronunciation of the French word hautbois, a compound word made of haut (\"high, loud\") and bois (\"wood, woodwind\"). A...", "image": "http://img.freebase.com/api/trans/raw/m/0291jbh", "name": "Oboe", "thumbnail": "http://indextank.com/_static/common/demo/0291jbh.jpg"}, "variables": {"0": 144}, "docid": "http://freebase.com/view/en/oboe", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/kemenche", "text": "The term kemenche (Turkish: kemen\u00e7e, Adyghe: \u0428\u044b\u043a1\u044d \u043f\u0449\u044b\u043d, Armenian: \u0584\u0561\u0574\u0561\u0576\u0579\u0561 k\u2019aman\u010da, Laz: \u00c7'ilili - \u10ed\u10d8\u10da\u10d8\u10da\u10d8, Azerbaijani: kaman\u00e7a, Persian: \u06a9\u0645\u0627\u0646\u0686\u0647, Greek: \u03bb\u03cd\u03c1\u03b1) is used to describe two types of three-stringed bowed musical instruments:\nBoth types of kemenche are played in the downright position, either by resting it on the knee when sitting, or held in front of the player when standing. It is always played \"braccio\", that is, with the tuning head uppermost. The kemenche bow is called the...", "image": "http://img.freebase.com/api/trans/raw/m/044m0vc", "name": "Kemenche", "thumbnail": "http://indextank.com/_static/common/demo/044m0vc.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/kemenche", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/ride_cymbal", "text": "The ride cymbal is a standard cymbal in most drum kits. It maintains a steady rhythmic pattern, sometimes called a ride pattern, rather than the accent of a crash. It is normally placed on the extreme right (or dominant hand) of a drum kit, above the floor tom.\nThe ride can fulfill any function or rhythm the hi-hat does, with the exclusion of an open and closed sound.\nThe term ride means to ride with the music, describing the cymbal's sustain after it is struck. The term may depict either...", "image": "http://img.freebase.com/api/trans/raw/m/02bdbkf", "name": "Ride cymbal", "thumbnail": "http://indextank.com/_static/common/demo/02bdbkf.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/ride_cymbal", "categories": {"family": "Cymbal"}} +{"fields": {"url": "http://freebase.com/view/en/kithara", "text": "The cithara or kithara (Greek: \u03ba\u03b9\u03b8\u03ac\u03c1\u03b1, kith\u0101ra, Latin: cithara) was an ancient Greek musical instrument in the lyre or lyra family. In modern Greek the word kithara has come to mean \"guitar\" (a word whose origins are found in kithara).\nThe kithara was a professional version of the two-stringed lyre. As opposed to the simpler lyre, which was a folk-instrument, the cithara was primarily used by professional musicians, called citharedes. The barbiton was a bass version of the kithara popular in...", "image": "http://img.freebase.com/api/trans/raw/m/02f7_fw", "name": "Kithara", "thumbnail": "http://indextank.com/_static/common/demo/02f7_fw.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/kithara", "categories": {"family": "Lyre"}} +{"fields": {"url": "http://freebase.com/view/en/scottish_smallpipes", "text": "The Scottish smallpipe, in its modern form, is a bellows-blown bagpipe developed by Colin Ross and others, to be playable according to the Great Highland Bagpipe fingering system. There are surviving examples of similar historical instruments such as the mouth-blown Montgomery smallpipes in E, dated 1757, which are now in the National Museum of Scotland. There is some discussion of the historical Scottish smallpipes in Collinson's history of the bagpipes. Some instruments are being built as...", "image": "http://img.freebase.com/api/trans/raw/m/042smv6", "name": "Scottish smallpipes", "thumbnail": "http://indextank.com/_static/common/demo/042smv6.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/scottish_smallpipes", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/contrabassoon", "text": "The contrabassoon, also known as the double bassoon or double-bassoon, is a larger version of the bassoon, sounding an octave lower. Its technique is similar to its smaller cousin, with a few notable differences.\nThe reed is considerably larger, at 65\u201375\u00a0mm in total length as compared to 53\u201358\u00a0mm for most bassoon reeds. Fingering is slightly different, particularly at the register change and in the extreme high range. The instrument is twice as long, curves around on itself twice, and, due...", "image": "http://img.freebase.com/api/trans/raw/m/044ln56", "name": "Contrabassoon", "thumbnail": "http://indextank.com/_static/common/demo/044ln56.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/contrabassoon", "categories": {"family": "Bassoon"}} +{"fields": {"url": "http://freebase.com/view/en/rmi_368_electra-piano_and_harpsichord", "text": "The RMI 368 Electra-Piano and Harpsichord was an electronic piano and the most popular instrument created by RMI. Often serving as a substitute for a grand piano in live performance, it didn't actually sound like one. It had its own distinctive sound that separated it from other electric alternatives to the piano, like the Fender Rhodes or the Wurlitzer, in that its sound was generated by transistors (like an electronic organ, which RMI started out making), instead of a hammer hitting a reed...", "image": "http://img.freebase.com/api/trans/raw/m/044yr_6", "name": "RMI 368 Electra-Piano and Harpsichord", "thumbnail": "http://indextank.com/_static/common/demo/044yr_6.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/rmi_368_electra-piano_and_harpsichord", "categories": {"family": "Electronic piano"}} +{"fields": {"url": "http://freebase.com/view/en/barrel_piano", "text": "A barrel piano (also known as a \"roller piano\") is a forerunner of the modern player piano. Unlike the pneumatic player piano, a barrel piano is usually powered by turning a hand crank, though coin operated models powered by clockwork were used to provide music in establishments such as pubs and caf\u00e9s. Barrel pianos were popular with street musicians, who sought novel instruments that were also highly portable. They are frequently confused with barrel organs, but are quite different...", "image": "http://img.freebase.com/api/trans/raw/m/02gsf7c", "name": "Barrel piano", "thumbnail": "http://indextank.com/_static/common/demo/02gsf7c.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/barrel_piano", "categories": {"family": "Piano"}} +{"fields": {"url": "http://freebase.com/view/en/langeleik", "text": "The langeleik also called langleik is a Norwegian stringed folklore musical instrument, a droned zither\nThe langeleik has only one melody string and up to 8 drone strings. Under the melody string there are seven frets per octave, forming a diatonic major scale. The drone strings are tuned to a triad. The langeleik is tuned to about an A, though on score the C major key is used, as if the instrument were tuned in C. This is for simplification of both writing and reading, by circumventing the...", "image": "http://img.freebase.com/api/trans/raw/m/04rh8gk", "name": "Langeleik", "thumbnail": "http://indextank.com/_static/common/demo/04rh8gk.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/langeleik", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/ektara", "text": "Ektara (Bengali: \u098f\u0995\u09a4\u09be\u09b0\u09be, Punjabi: \u0a07\u0a15 \u0a24\u0a3e\u0a30; literally \"one-string\", also called iktar, ektar, yaktaro gopichand) is a one-string instrument used in Bangladesh, India, Egypt, and Pakistan.\nIn origin the ektara was a regular string instrument of wandering bards and minstrels from India and is plucked with one finger. The ektara usually has a stretched single string, an animal skin over a head (made of dried pumpkin/gourd, wood or coconut) and pole neck or split bamboo cane neck.\nPressing the two...", "image": "http://img.freebase.com/api/trans/raw/m/041ykv2", "name": "Ektara", "thumbnail": "http://indextank.com/_static/common/demo/041ykv2.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/ektara", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/flexatone", "text": "The flexatone is a modern percussion instrument (an indirectly struck idiophone) consisting of a small flexible metal sheet suspended in a wire frame ending in a handle. \nAn invention for a flexatone occurs in the British Patent Records of 1922 and 1923. In 1924 the 'Flex-a-tone' was patented in the USA by the Playatone Company of New York.\nA wooden knob mounted on a strip of spring steel lies on each side of the metal sheet. The player holds the flexatone in one hand with the palm around...", "image": "http://img.freebase.com/api/trans/raw/m/029ydt8", "name": "Flexatone", "thumbnail": "http://indextank.com/_static/common/demo/029ydt8.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/flexatone", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/psalmodicon", "text": "The psalmodicon, or psalmodikon, is a single-stringed musical instrument. It was developed in Scandinavia for simplifying music in churches and schools. Beginning in the early 19th century, it was adopted by many rural churches in Scandinavia; later, immigrants brought the instrument to the United States. At the time, many congregations could not afford organs. Dance instruments were considered inappropriate for sacred settings, so violins were not allowed. The psalmodikon, on the other...", "image": "http://img.freebase.com/api/trans/raw/m/03sf05n", "name": "Psalmodicon", "thumbnail": "http://indextank.com/_static/common/demo/03sf05n.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/psalmodicon", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/santur", "text": "The santur (also sant\u016br, santour, santoor ) (Persian: \u0633\u0646\u062a\u0648\u0631) is a hammered dulcimer, of Persian origin it has strong resemblances to the Indian santoor. It is a trapezoid-shaped box often made of walnut or different exotic woods. The original classical santur has 72 strings. The can be roughly described as one hundred strings in Persian. The oval-shaped mallets (Mezrabs) are feather-weight and are held between the index and middle fingers. A typical santur has two sets of bridges, providing...", "image": "http://img.freebase.com/api/trans/raw/m/02f61nd", "name": "Santur", "thumbnail": "http://indextank.com/_static/common/demo/02f61nd.jpg"}, "variables": {"0": 4}, "docid": "http://freebase.com/view/en/santur", "categories": {"family": "Struck string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/tromba_marina", "text": "A tromba marina, or marine trumpet (Fr. trompette marine; Ger. Marientrompete, Trompetengeige, Nonnengeige or Trumscheit, Pol. tubmaryna) is a triangular bowed string instrument used in medieval and Renaissance Europe that was highly popular in the 15th century in England and survived into the 18th century. The tromba marina consists of a body and neck in the shape of a truncated cone resting on a triangular base. It is usually four to seven feet long, and is a monochord (although some...", "image": "http://img.freebase.com/api/trans/raw/m/02g9ls1", "name": "Tromba marina", "thumbnail": "http://indextank.com/_static/common/demo/02g9ls1.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/tromba_marina", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/uilleann_pipes", "text": "The uilleann (pronounced /\u02c8\u026al\u0259n/) pipes are the characteristic national bagpipe of Ireland. Their current name (they were earlier known in English as \"union pipes\") is a part translation of the Irish-language term p\u00edoba uilleann (literally, \"pipes of the elbow\"), from their method of inflation. The bag of the uilleann pipes is inflated by means of a small set of bellows strapped around the waist and the right arm. The bellows not only relieve the player from the effort needed to blow into a...", "image": "http://img.freebase.com/api/trans/raw/m/03twg5s", "name": "Uilleann pipes", "thumbnail": "http://indextank.com/_static/common/demo/03twg5s.jpg"}, "variables": {"0": 5}, "docid": "http://freebase.com/view/en/uilleann_pipes", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/electronic_keyboard", "text": "An electronic keyboard (also called digital keyboard, portable keyboard and home keyboard) is an electronic or digital keyboard instrument.\nThe major components of a typical modern electronic keyboard are:\nElectronic keyboards typically use MIDI signals to send and receive data, a standard format now universally used across most digital electronic musical instruments. On the simplest example of an electronic keyboard, MIDI messages would be sent when a note is pressed on the keyboard, and...", "image": "http://img.freebase.com/api/trans/raw/m/029n2xk", "name": "Electronic keyboard", "thumbnail": "http://indextank.com/_static/common/demo/029n2xk.jpg"}, "variables": {"0": 98}, "docid": "http://freebase.com/view/en/electronic_keyboard", "categories": {"family": "Keyboard instrument"}} +{"fields": {"url": "http://freebase.com/view/en/barrel_organ", "text": "A barrel organ (or roller organ) is a mechanical musical instrument consisting of bellows and one or more ranks of pipes housed in a case, usually of wood, and often highly decorated. The basic principle is the same as a traditional pipe organ, but rather than being played by an organist, the barrel organ is activated either by a person turning a crank, or by clockwork driven by weights or springs. The pieces of music are encoded onto wooden barrels (or cylinders), which are analogous to the...", "image": "http://img.freebase.com/api/trans/raw/m/02bcy90", "name": "Barrel organ", "thumbnail": "http://indextank.com/_static/common/demo/02bcy90.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/barrel_organ", "categories": {"family": "Mechanical organ"}} +{"fields": {"url": "http://freebase.com/view/en/subcontrabass_flute", "text": "The subcontrabass flute is one of the largest instruments in the flute family, measuring over 15\u00a0feet (4.6 m) long. The instrument can be made in the key of G, pitched a fourth below the contrabass flute in C and two octaves below the alto flute in G; which is sometimes also called double contra-alto flute, or in C, which will sound three octaves lower than the C flute.\nThe subcontrabass flute is rarely used outside of flute ensembles. It is sometimes called the \"gentle giant\" of the flute...", "image": "http://img.freebase.com/api/trans/raw/m/05m243v", "name": "Subcontrabass flute", "thumbnail": "http://indextank.com/_static/common/demo/05m243v.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/subcontrabass_flute", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/gaita_de_saco", "text": "The gaita de saco (or de bota) is a type of bagpipe native to the provinces of Soria, La Rioja, Alava, and Burgos in north-central Spain. In the past, it may also have been played in Segovia and \u00c1vila. According to some experts, the gaita de boto is the same as the gaita de fuelle of Old Castile.\nIt consists of a single chanter (puntero) holding a double reed which plays the melody, and single drone (ronco), which has a single reed and plays a constant bass note.\nIn La Rioja, the instrument...", "image": "http://img.freebase.com/api/trans/raw/m/063850_", "name": "Gaita de saco", "thumbnail": "http://indextank.com/_static/common/demo/063850_.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gaita_de_saco", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/gong", "text": "A gong is an East and South East Asian musical percussion instrument that takes the form of a flat metal disc which is hit with a malleta.\nGongs are broadly of three types. Suspended gongs are more or less flat, circular discs of metal suspended vertically by means of a cord passed through holes near to the top rim. Bossed or nipple gongs have a raised center boss and are often suspended and played horizontally. Bowl gongs are bowl-shaped, and rest on cushions and belong more to bells than...", "image": "http://img.freebase.com/api/trans/raw/m/041nzgy", "name": "Gong", "thumbnail": "http://indextank.com/_static/common/demo/041nzgy.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/gong", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/vielle", "text": "The vielle is a European bowed stringed instrument used in the Medieval period, similar to a modern violin but with a somewhat longer and deeper body, five (rather than four) gut strings, and a leaf-shaped pegbox with frontal tuning pegs. The instrument was also known as a fidel or a viuola, although the French name for the instrument, vielle, is generally used. It was one of the most popular instruments of the Medieval period, and was used by troubadours and jongleurs from the 13th through...", "image": "http://img.freebase.com/api/trans/raw/m/02bgn1z", "name": "Vielle", "thumbnail": "http://indextank.com/_static/common/demo/02bgn1z.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/vielle", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/horn", "text": "The horn is a brass instrument consisting of about 12\u201313 feet (3.66\u20133.96 meters) of tubing wrapped into a coil with a flared bell. A musician who plays the horn is called a horn player (or less frequently, a hornist).\nDescended from the natural horn, the instrument is often informally and incorrectly known as the French horn. Since 1971 the International Horn Society has recommended the use of the word horn alone, as the commonly played instrument is not, in fact, the French horn, but rather...", "image": "http://img.freebase.com/api/trans/raw/m/0593gps", "name": "Horn", "thumbnail": "http://indextank.com/_static/common/demo/0593gps.jpg"}, "variables": {"0": 58}, "docid": "http://freebase.com/view/en/horn", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/irish_flute", "text": "The term Irish Flute refers to a conical-bore, simple-system wooden flute of the type favored by classical flautists of the early 19th century, or to a flute of modern manufacture derived from this design (often with modifications to optimize its use in Irish Traditional Music or Scottish Traditional Music).\nThe Irish flute is a simple system, transverse flute which plays a diatonic (Major) scale as the tone holes are successively uncovered. Most flutes from the Classical era, and some of...", "image": "http://img.freebase.com/api/trans/raw/m/05tr2mh", "name": "Irish flute", "thumbnail": "http://indextank.com/_static/common/demo/05tr2mh.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/irish_flute", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/cumbus", "text": "The c\u00fcmb\u00fc\u015f (Turkish pronunciation:\u00a0[d\u0292ym\u02c8by\u0283]; sometimes approximated as /d\u0292u\u02d0m\u02c8bu\u02d0\u0283/ by English speakers) is a Turkish stringed instrument of relatively modern origin. Developed in the early 20th century by Zeynelabidin C\u00fcmb\u00fc\u015f as an oud-like instrument that could be heard as part of a larger ensemble. In construction it resembles both the American banjo and the Middle Eastern oud. A fretless instrument, it has six courses of doubled-strings, and is generally tuned like an oud. In shape,...", "image": "http://img.freebase.com/api/trans/raw/m/02b8q91", "name": "C\u00fcmb\u00fc\u015f", "thumbnail": "http://indextank.com/_static/common/demo/02b8q91.jpg"}, "variables": {"0": 4}, "docid": "http://freebase.com/view/en/cumbus", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/gaita_de_boto", "text": "The gaita de boto is a type of bagpipe native to the Aragon region of northern Spain.\nIts use and construction were nearly extinct by the 1970s, when a revival of folk music began. Today there are various gaita builders, various schools and associations for gaita players, and more than a dozen Aragonese folk music groups which include the instrument in their ensemble. Most importantly, there are now several hundred gaiteros within Aragon.\nThe gaita de boto consists of\nThe bag is...", "image": "http://img.freebase.com/api/trans/raw/m/0636rx_", "name": "Gaita de boto", "thumbnail": "http://indextank.com/_static/common/demo/0636rx_.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gaita_de_boto", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/cristal_baschet", "text": "The Cristal Baschet is a musical instrument that produces sound from oscillating glass cylinders. The Cristal Baschet is also known as the Crystal Organ and the Crystal Baschet, and composed of 54 chromatically-tuned glass rods. The glass rods are rubbed with moistened fingers to produce vibrations. The sound of the Cristal Baschet is similar to that of the glass harmonica.\nThe vibration of the glass rods in the Cristal Baschet is passed to a heavy block of metal by a metal stem whose...", "image": "http://img.freebase.com/api/trans/raw/m/041d9tn", "name": "Cristal baschet", "thumbnail": "http://indextank.com/_static/common/demo/041d9tn.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/cristal_baschet", "categories": {"family": "Crystallophone"}} +{"fields": {"url": "http://freebase.com/view/en/lauterbach_stradivarius", "text": "The Lauterbach Stradivarius of 1719 is an antique violin fabricated by Italian luthier, Antonio Stradivari of Cremona (1644-1737). The instrument derives its name from previous owner, German virtuoso, Johann Christoph Lauterbach.\nComposer and violinist Charles Philippe Lafont owned the violin. On his death, the violin was acquired by luthier and expert Jean-Baptiste Vuillaume. Vuillaume sold the violin to Johann Christoph Lauterbach.\nPolish textile manufacturer Henryk Grohman acquired the...", "image": "http://img.freebase.com/api/trans/raw/m/0bbdvgl", "name": "Lauterbach Stradivarius", "thumbnail": "http://indextank.com/_static/common/demo/0bbdvgl.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/lauterbach_stradivarius", "categories": {"family": "Violin"}} +{"fields": {"url": "http://freebase.com/view/en/goblet_drum", "text": "The Goblet drum (also Chalice drum, Darbuka Doumbek or Tablah\"') is a goblet shaped hand drum used mostly in the Middle East, North Africa, and Eastern Europe.\nThough it is not known exactly when these drums were first made, they are known to be of ancient origin. Some say that it has been around for thousands of years, used in Mesopotamian and Ancient Egyptian cultures.There has also has been some debates that it has actually originated in Europe and was brought to the Middle East by...", "image": "http://img.freebase.com/api/trans/raw/m/03sll6p", "name": "Goblet drum", "thumbnail": "http://indextank.com/_static/common/demo/03sll6p.jpg"}, "variables": {"0": 9}, "docid": "http://freebase.com/view/en/goblet_drum", "categories": {"family": "Drum"}} +{"fields": {"url": "http://freebase.com/view/en/synthesizer", "text": "A synthesizer (often abbreviated \"synth\") is an electronic instrument capable of producing sounds by generating electrical signals of different frequencies. These electrical signals are played through a loudspeaker or set of headphones. Synthesizers can usually produce a wide range of sounds, which may either imitate other instruments (\"imitative synthesis\") or generate new timbres.\nSynthesizers use a number of different technologies or programmed algorithms, each with their own strengths...", "image": "http://img.freebase.com/api/trans/raw/m/03q_gc1", "name": "Synthesizer", "thumbnail": "http://indextank.com/_static/common/demo/03q_gc1.jpg"}, "variables": {"0": 442}, "docid": "http://freebase.com/view/en/synthesizer", "categories": {"family": "Keyboard instrument"}} +{"fields": {"url": "http://freebase.com/view/en/ceng", "text": "The \u00e7eng is a Turkish harp. Descended from ancient Near Eastern instruments, it was a popular Ottoman instrument until the last quarter of the 17th century. The word comes from the Persian word \"chang,\" which means \"harp\" (and also \"five fingers\").\nThe ancestor of the Ottoman harp is thought to be an instrument seen in ancient Assyrian tablets. While a similar instrument also appears in Egyptian drawings.\nIn the late 20th century, instrument makers and performers began to revive the \u00e7eng,...", "image": "http://img.freebase.com/api/trans/raw/m/02d6s41", "name": "\u00c7eng", "thumbnail": "http://indextank.com/_static/common/demo/02d6s41.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/ceng", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/banhu", "text": "The banhu (\u677f\u80e1, pinyin: b\u01cenh\u00fa) is a Chinese traditional bowed string instrument in the huqin family of instruments. It is used primarily in northern China. Ban means a piece of wood and hu is short for huqin.\nLike the more familiar erhu and gaohu, the banhu has two strings, is held vertically, and the bow hair passes in between the two strings. The banhu differs in construction from the erhu in that its soundbox is generally made from a coconut shell rather than wood, and instead of a...", "image": "http://img.freebase.com/api/trans/raw/m/0bch7yq", "name": "Banhu", "thumbnail": "http://indextank.com/_static/common/demo/0bch7yq.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/banhu", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/contrabass_bugle", "text": "The contrabass bugle, usually shortened to contra, is the lowest-pitched instrument in the drum and bugle corps hornline. It is essentially the drum corps' counterpart to the marching band's sousaphone: the lowest-pitched member of the hornline, and a replacement for the concert tuba on the marching field.\nIt is different from the other members of the marching band and drum corps hornlines in that it rests on the shoulder of the player, rather than being held in front of the body. Because...", "image": "http://img.freebase.com/api/trans/raw/m/03tb_w9", "name": "Contrabass Bugle", "thumbnail": "http://indextank.com/_static/common/demo/03tb_w9.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/contrabass_bugle", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/sallaneh", "text": "The sallaneh (\u0633\u0644\u0627\u0646\u0647) is a newly developed plucked string instrument made under the supervision of the Iranian musician Hossein Alizadeh, and constructed by Siamak Afshari. It is inspired by the ancient Persian lute called barbat. The barbat used to have three strings but Sallaneh has six melody and six harmonic strings giving Alizadeh a new realm in lower tones.", "image": "http://img.freebase.com/api/trans/raw/m/03svz1c", "name": "Sallaneh", "thumbnail": "http://indextank.com/_static/common/demo/03svz1c.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/sallaneh", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/vichitra_veena", "text": "The vichitra veena (Sanskrit: \u0935\u093f\u091a\u093f\u0924\u094d\u0930 \u0935\u0940\u0923\u093e) is a plucked string instrument used in Hindustani music. It is similar to the Carnatic gottuvadhyam (chitra vina). It has no frets and is played with a slide.\nThe Vichitra Veena is the modern form of ancient Ektantri Veena. It is made of a broad, fretless, horizontal arm or crossbar (dand) around three feet long and six inches wide, with two large resonating gourds (tumba), which are inlaid with ivory and attached underneath at either end. The...", "image": "http://img.freebase.com/api/trans/raw/m/02dz2m_", "name": "Vichitra veena", "thumbnail": "http://indextank.com/_static/common/demo/02dz2m_.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/vichitra_veena", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/mellophone", "text": "The mellophone is a brass instrument that is typically used in place of the horn (sometimes called a French horn) in marching bands or drum and bugle corps.\nOwing to its use primarily outside of concert music, there is not much solo literature for the mellophone, other than that used within drum and bugle corps.\nThe present-day mellophone has three valves, operated with the right hand. Mellophone fingering is identical to that of a trumpet. Mellophones are typically pitched in the key of F....", "image": "http://img.freebase.com/api/trans/raw/m/029hskr", "name": "Mellophone", "thumbnail": "http://indextank.com/_static/common/demo/029hskr.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/mellophone", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/flute", "text": "The flute is a musical instrument of the woodwind family. Unlike woodwind instruments with reeds, a flute is an aerophone or reedless wind instrument that produces its sound from the flow of air across an opening. According to the instrument classification of Hornbostel-Sachs, flutes are categorized as Edge-blown aerophones.\nA musician who plays the flute can be referred to as a flute player, a flautist, a flutist, or less commonly a fluter.\nAside from the voice, flutes are the earliest...", "image": "http://img.freebase.com/api/trans/raw/m/02bdpv9", "name": "Flute (transverse)", "thumbnail": "http://indextank.com/_static/common/demo/02bdpv9.jpg"}, "variables": {"0": 197}, "docid": "http://freebase.com/view/en/flute", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/serinette", "text": "A serinette is a type of mechanical musical instrument consisting of a small barrel organ. It appeared in the first half of the 18th century in eastern France, and was used to teach tunes to canaries. Its name is derived from the French serin, meaning \u201ccanary.\u201d\nSerinettes are housed in a wooden case, normally of walnut, and typically measuring 265 \u00d7 200 \u00d7 150 mm. The instrument is played by turning a crank mounted on the front. The crank pumps a bellows to supply air to the pipes, and also...", "image": "http://img.freebase.com/api/trans/raw/m/02frq4g", "name": "Serinette", "thumbnail": "http://indextank.com/_static/common/demo/02frq4g.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/serinette", "categories": {"family": "Organ"}} +{"fields": {"url": "http://freebase.com/view/en/psaltery", "text": "A psaltery is a stringed musical instrument of the harp or the zither family. The psaltery of Ancient Greece (Epigonion) dates from at least 2800 BC, when it was a harp-like instrument. Etymologically the word derives from the Ancient Greek \u03c8\u03b1\u03bb\u03c4\u03ae\u03c1\u03b9\u03bf\u03bd (psalterion) \u201cstringed instrument, psaltery, harp\u201d and that from the verb \u03c8\u03ac\u03bb\u03bb\u03c9 (psallo) \u201cto touch sharply, to pluck, pull, twitch\u201d and in the case of the strings of musical instruments, \u201cto play a stringed instrument with the fingers, and not...", "image": "http://img.freebase.com/api/trans/raw/m/02cd9nz", "name": "Psaltery", "thumbnail": "http://indextank.com/_static/common/demo/02cd9nz.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/psaltery", "categories": {"family": "Zither"}} +{"fields": {"url": "http://freebase.com/view/en/novachord", "text": "The Novachord is often considered to be the world's first commercial polyphonic synthesizer. All-electronic, incorporating many circuit and control elements found in modern synths, and using subtractive synthesis to generate tones, it was designed by John Hanert, Laurens Hammond and C. N. Williams and manufactured by the Hammond company. Only some 1069 examples were built over a period from 1939 to 1942. It was one of very few electronic products released by Hammond that was not intended to...", "image": "http://img.freebase.com/api/trans/raw/m/0bd1s41", "name": "Novachord", "thumbnail": "http://indextank.com/_static/common/demo/0bd1s41.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/novachord", "categories": {"family": "Electronic keyboard"}} +{"fields": {"url": "http://freebase.com/view/en/steel-string_acoustic_guitar", "text": "A steel-string acoustic guitar is a modern form of guitar descended from the classical guitar, but strung with steel strings for a brighter, louder sound. It is often referred to simply as an acoustic guitar, although strictly speaking the nylon-strung classical guitar is acoustic as well.\nThe most common type can be called a flat-top guitar to distinguish it from the more specialized archtop guitar and other variations.\nThe standard tuning for an acoustic guitar is E-A-D-G-B-E (low to...", "image": "http://img.freebase.com/api/trans/raw/m/0290vfk", "name": "Steel-string acoustic guitar", "thumbnail": "http://indextank.com/_static/common/demo/0290vfk.jpg"}, "variables": {"0": 34}, "docid": "http://freebase.com/view/en/steel-string_acoustic_guitar", "categories": {"family": "Guitar"}} +{"fields": {"url": "http://freebase.com/view/en/electric_harp", "text": "Like electric guitars, electric harps are based on their acoustic originals, and there are both solid-body and electro-acoustic models available.\nA solid-body electric harp has no hollow soundbox, and thus makes very little noise when not amplified. Alan Stivell writes in his book Telenn, la harpe bretonne of his first dreams of electric harps going back to the late 1950s. He designed and had made a solid body (after different electric-acoustic harps) electric harp at the turn of the...", "image": "http://img.freebase.com/api/trans/raw/m/02cqk_x", "name": "Electric harp", "thumbnail": "http://indextank.com/_static/common/demo/02cqk_x.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/electric_harp", "categories": {"family": "Harp"}} +{"fields": {"url": "http://freebase.com/view/m/04czcxw", "text": "Fue (\u7b1b, hiragana: \u3075\u3048) is the Japanese word for flute, and refers to a class of flutes native to Japan. Fue come in many varieties, but are generally high-pitched and made of a bamboo called shinobue. The most popular of the fue is the shakuhachi.\nFue are traditionally broken up into two basic categories \u2013 the transverse flute and the end-blown flute. Transverse flutes are held to the side, with the musician blowing across a hole near one end; end-blown flutes are held vertically and the...", "image": "http://img.freebase.com/api/trans/raw/m/0cckk27", "name": "Fue", "thumbnail": "http://indextank.com/_static/common/demo/0cckk27.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/04czcxw", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/m/02npsp", "text": "A ratchet, also called a noisemaker (or, when used in Judaism, a gragger or grogger (etymologically from Yiddish: \u05d2\u05e8\u05d0\u05b7\u05d2\u05e2\u05e8) or ra'ashan (Hebrew: \u05e8\u05e2\u05e9\u05df\u200e)), is an orchestral musical instrument played by percussionists. Operating on the principle of the ratchet device, a gearwheel and a stiff board is mounted on a handle, which can be freely rotated. The handle is held and the whole mechanism is swung around, the momentum causing the board to click against the gearwheel, making a clicking and...", "image": "http://img.freebase.com/api/trans/raw/m/02c4758", "name": "Ratchet", "thumbnail": "http://indextank.com/_static/common/demo/02c4758.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/02npsp", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/ondes_martenot", "text": "The ondes Martenot (pronounced:\u00a0[\u0254\u0303d ma\u0281t\u0259no], OHND mar-t\u0259-NOH, French for \"Martenot waves\"), also known as the ondium Martenot, Martenot and ondes musicales, is an early electronic musical instrument invented in 1928 by Maurice Martenot. The original design was similar in sound to the theremin. The sonic capabilities of the instrument were later expanded by the addition of timbral controls and switchable loudspeakers.\nThe instrument's eerie wavering notes are produced by varying the...", "image": "http://img.freebase.com/api/trans/raw/m/044vh28", "name": "Ondes Martenot", "thumbnail": "http://indextank.com/_static/common/demo/044vh28.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/ondes_martenot", "categories": {"family": "Electronic keyboard"}} +{"fields": {"url": "http://freebase.com/view/en/yehu", "text": "The yehu (\u6930\u80e1; pinyin: y\u0113h\u00fa) is a Chinese bowed string instrument in the huqin family of musical instruments. Ye means coconut and hu is short for huqin. It is used particularly in the southern coastal provinces of China and in Taiwan. The instrument's soundbox is made from a coconut shell, which is cut on the playing end and covered with a piece of coconut wood instead of the snakeskin commonly used on other huqin instruments such as the erhu or gaohu. As with most huqin the bow hair passes...", "image": "http://img.freebase.com/api/trans/raw/m/02d9q5z", "name": "Yehu", "thumbnail": "http://indextank.com/_static/common/demo/02d9q5z.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/yehu", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/m/05zl9hy", "text": "The Sarangi (Nepali/Hindi: \u0938\u093e\u0930\u0919\u094d\u0917\u0940) is a folk Nepalese string instrument. Unlike Classical Indian Sarangi, it has four strings and all of them are played. Traditionally, in Nepal, Sarangi was only played by people of Gandarva or Gaine cast, who sings narrative tales and folk song. However, in present days, its widely used and played by many.\nTraditional Nepali Sarangi is made up of single piece of wood having a neck and hollowed out body. Sarangi is carved out from a very light wood, locally...", "image": "http://img.freebase.com/api/trans/raw/m/08bgn5k", "name": "Sarangi", "thumbnail": "http://indextank.com/_static/common/demo/08bgn5k.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/05zl9hy", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/tambura", "text": "The tambura, tanpura, or tambora is a long-necked plucked lute (a stringed instrument found in different forms and in many places). The body shape of the tambura somewhat resembles that of the sitar, but it has no frets \u2013 only the open strings are played to accompany other musicians. It has four or five (rarely six) wire strings, which are plucked one after another in a regular pattern to create a harmonic resonance on the basic note (bourdon or drone function).\nTamburas come in different...", "image": "http://img.freebase.com/api/trans/raw/m/05lng4p", "name": "Tambura", "thumbnail": "http://indextank.com/_static/common/demo/05lng4p.jpg"}, "variables": {"0": 11}, "docid": "http://freebase.com/view/en/tambura", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/welsh_pipes", "text": "Welsh bagpipes (Welsh pibau, pipa c\u0175d, pibau c\u0175d, pibgod, cotbib, pibau cyrn, chwibanogl a chod, sachbib, backpipes or bacbib) have been documented, represented or described in Wales since the fourteenth century. In 1376, the poet Iolo Goch describes the instrument in his Cywydd to Syr Hywel y Fwyall.. Also, in the same century, Brut y Tywysogion (\"Chronicle of the Princes\"), written around 1330 AD, states that there are three types of wind instrument: Organ, a Phibeu a Cherd y got (\"organ,...", "image": "http://img.freebase.com/api/trans/raw/m/05mfh7c", "name": "Welsh bagpipes", "thumbnail": "http://indextank.com/_static/common/demo/05mfh7c.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/welsh_pipes", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/roman_tuba", "text": "The tuba of ancient Rome is a military signal trumpet, quite different from the modern tuba. The tuba (from Latin tubus, \"tube\") was produced around 500 BC. Its shape was straight, in contrast to the military buccina or cornu, which was more like the modern tuba in curving around the body. Its origin is thought to be Etruscan, and it is similar to the Greek salpinx. About four feet in length, it was made usually of bronze, and was played with a detachable bone mouthpiece.", "image": "http://img.freebase.com/api/trans/raw/m/02ghk4m", "name": "Roman tuba", "thumbnail": "http://indextank.com/_static/common/demo/02ghk4m.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/roman_tuba", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/musette_de_cour", "text": "The musette de cour or baroque musette is a musical instrument of the bagpipe family. Visually, the musette is characterised by the short, cylindrical shuttle-drone and the two chalumeaux. Both the chanters and the drones have a cylindrical bore and use a double reed, giving a quiet tone similar to the oboe. The instrument is always bellows-blown.\nNote: the qualified name de cour does not appear in original music for the instrument; title-pages refer to it simply as musette, allowing...", "image": "http://img.freebase.com/api/trans/raw/m/02dxqsx", "name": "Musette de cour", "thumbnail": "http://indextank.com/_static/common/demo/02dxqsx.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/musette_de_cour", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/silent_piano", "text": "A silent piano is an acoustic piano where there is an option to silence the strings by means of an interposing hammer bar. A silent piano is designed for private silent practice. In the silent mode, sensors pick up the piano key movement. Older models used mechanical sensors which affected the touch and produced a clicking sound, whereas newer models use optical sensors which do not affect the feel or sound of the piano. The key movement is then converted to a midi signal and can link to a...", "image": "http://img.freebase.com/api/trans/raw/m/09hq9r4", "name": "Silent piano", "thumbnail": "http://indextank.com/_static/common/demo/09hq9r4.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/silent_piano", "categories": {"family": "Piano"}} +{"fields": {"url": "http://freebase.com/view/en/electronic_drum", "text": "An electronic drum is a percussion instrument in which the sound is generated by an electronic waveform generator or sampler instead of by acoustic vibration.\nWhen an electronic drum pad is struck, a voltage change is triggered in the embedded piezoelectric transducer (piezo) or force sensitive resistor (FSR). The resultant signals are transmitted to an electronic \"drum brain\" via TS or TRS cables, and are translated into digital waveforms, which produce the desired percussion sound assigned...", "image": "http://img.freebase.com/api/trans/raw/m/02f88sx", "name": "Electronic drum", "thumbnail": "http://indextank.com/_static/common/demo/02f88sx.jpg"}, "variables": {"0": 12}, "docid": "http://freebase.com/view/en/electronic_drum", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/piano_accordion", "text": "A piano accordion is an accordion equipped with a right-hand keyboard similar to a piano or organ. Its acoustic mechanism is more similar to that of an organ than a piano, as they are both wind instruments, but the term \"piano accordion\"\u2014coined by Guido Deiro in 1910\u2014has remained the popular nomenclature. It may be equipped with any of the available systems for the left-hand manual.\nIn comparison to a piano keyboard, the keys are more rounded, smaller, and lighter to the touch. These go...", "image": "http://img.freebase.com/api/trans/raw/m/04xzfth", "name": "Piano accordion", "thumbnail": "http://indextank.com/_static/common/demo/04xzfth.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/piano_accordion", "categories": {"family": "Accordion"}} +{"fields": {"url": "http://freebase.com/view/en/zetland_pipes", "text": "The Zetland pipes were a type of bagpipe designed and crafted by Pipe Major Royce Lerwick in the 1990s.\nLerwick believed that the bagpipes had been introduced to the British Isles by the Vikings. His \"Zetland pipes\" were intended to resemble single-drone, single-reeded pipes such as might have been brought to the Shetland Islands by the Vikings. The term \"Zetland\" is an antiquated variant of \"Shetland\".\nThe original impetus for the design, according to Lerwick, was the Lady Maket pipes, or...", "image": "http://img.freebase.com/api/trans/raw/m/063bbz6", "name": "Zetland pipes", "thumbnail": "http://indextank.com/_static/common/demo/063bbz6.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/zetland_pipes", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/igil", "text": "An igil (Tuvan- \u0438\u0433\u0438\u043b) is a two-stringed Tuvan musical instrument, played by bowing the strings. (It is called \"ikili\" in Western Mongolia.) The neck and lute-shaped sound box are usually made of a solid piece of pine or larch. The top of the sound box may be covered with skin or a thin wooden plate. The strings, and those of the bow, are traditionally made of hair from a horse's tail (strung parallel), but may also be made of nylon. Like the morin khuur of Mongolia, the igil typically...", "image": "http://img.freebase.com/api/trans/raw/m/03tck74", "name": "Igil", "thumbnail": "http://indextank.com/_static/common/demo/03tck74.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/igil", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/alto_clarinet", "text": "The alto clarinet is a wind instrument of the clarinet family. It is a transposing instrument pitched in the key of E\u266d, though instruments in F (and in the 19th century, E) have been made. It is sometimes known as a tenor clarinet; this name especially is applied to the instrument in F. In size it lies between the soprano clarinet and the bass clarinet, to which it bears a greater resemblance in that it typically has a straight body (made of Grenadilla or other wood, hard rubber, or...", "image": "http://img.freebase.com/api/trans/raw/m/02bvl5n", "name": "Alto clarinet", "thumbnail": "http://indextank.com/_static/common/demo/02bvl5n.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/alto_clarinet", "categories": {"family": "Clarinet"}} +{"fields": {"url": "http://freebase.com/view/en/hellier_stradivari", "text": "The Hellier Stradivarius of circa 1679 is a violin made by Antonio Stradivari of Cremona, Italy. It derives its name from the Hellier family, who might well have bought it directly from the luthier himself.\nThe Hellier Stradivarius has had a convoluted ownership history. It seems to have been in the possession of the Hellier family from the beginning of the 18th century. Sir Samuel Hellier, High Sheriff of Staffordshire 1745-1749, brought the violin to England, and through various wills it...", "image": "http://img.freebase.com/api/trans/raw/m/03rb6yt", "name": "Hellier Stradivarius", "thumbnail": "http://indextank.com/_static/common/demo/03rb6yt.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/hellier_stradivari", "categories": {"family": "Violin"}} +{"fields": {"url": "http://freebase.com/view/en/rubab", "text": "Rubab or robab (Persian: \u0631\u064f\u0628\u0627\u0628 rub\u0101b, Urdu And Pashto \u0631\u0628\u0627\u0628, Tajik and Uzbek \u0440\u0443\u0431\u043e\u0431) is a lute-like musical instrument originally from Afghanistan but is also played in the neighbouring countries, especially Pakistan. It derives its name from the Arab rebab which means \"played with a bow\" but the Central Asian instrument is plucked, and is distinctly different in construction. The rubab is mainly used by Pashtun, Tajik, Kashmiri and Iranian Kurdish classical musicians.\nThe rubab is a...", "image": "http://img.freebase.com/api/trans/raw/m/02h1k1w", "name": "Rubab", "thumbnail": "http://indextank.com/_static/common/demo/02h1k1w.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/rubab", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/great_irish_warpipes", "text": "The Great Irish Warpipes (Irish: p\u00edob mh\u00f3r; literally \"great pipes\") are an instrument that in modern practice is identical, and historically was analogous or identical to the Great Highland Bagpipe. \"Warpipes\" is an English term; The first use of the Gaelic term in Ireland is recorded in a poem by John O'Naughton (c. 1650-1728), in which the bagpipes are referred to as p\u00edb mh\u00f3r. The p\u00edob mh\u00f3r has a long and significant history in Ireland.\nIn Gaelic Ireland and Scotland, the bagpipe seems to...", "image": "http://img.freebase.com/api/trans/raw/m/03s1xj3", "name": "Great Irish Warpipes", "thumbnail": "http://indextank.com/_static/common/demo/03s1xj3.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/great_irish_warpipes", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/flageolet", "text": "A flageolet is a woodwind musical instrument and a member of the fipple flute family. Its invention is ascribed to the 16th century Sieur Juvigny in 1581. It had 4 holes on the front and 2 on the back. The English instrument maker William Bainbridge developed it further and patented the \"improved English flageolet\" in 1803 as well as the double flageolet around 1805. They were continued to be made until the 19th century when it was succeeded by the tin whistle.\nFlageolets have varied greatly...", "image": "http://img.freebase.com/api/trans/raw/m/02f9ht0", "name": "Flageolet", "thumbnail": "http://indextank.com/_static/common/demo/02f9ht0.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/flageolet", "categories": {"family": "Duct flutes"}} +{"fields": {"url": "http://freebase.com/view/en/arp_string_ensemble", "text": "The ARP String Ensemble, also known as the Solina String Ensemble, is a fully polyphonic multi-orchestral ARP Instruments, Inc. synthesizer with a 49-key keyboard, produced by Solina from 1974 to 1981. The sounds it incorporates are violin, viola, trumpet, horn, cello and contrabass. The keyboard uses 'organ style' divide-down technology to make it polyphonic. The built-in chorus effect gives the instrument its famous sound.\nThe core technology is based on the String Section of the Eminent...", "image": "http://img.freebase.com/api/trans/raw/m/02bv07p", "name": "ARP String Ensemble", "thumbnail": "http://indextank.com/_static/common/demo/02bv07p.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/arp_string_ensemble", "categories": {"family": "Synthesizer"}} +{"fields": {"url": "http://freebase.com/view/en/crash_cymbal", "text": "A crash cymbal is a type of cymbal that produces a loud, sharp \"crash\" and is used mainly for occasional accents, as opposed to in ostinato. The term \"crash\" may have been first used by Zildjian in 1928. They can be mounted on a stand and played with a drum stick, or by hand in pairs. One or two crash cymbals are a standard part of a drum kit. Suspended crash cymbals are also used in bands and orchestras, either played with a drumstick or rolled with a pair of mallets to produce a slower,...", "image": "http://img.freebase.com/api/trans/raw/m/02bdbkf", "name": "Crash cymbal", "thumbnail": "http://indextank.com/_static/common/demo/02bdbkf.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/crash_cymbal", "categories": {"family": "Cymbal"}} +{"fields": {"url": "http://freebase.com/view/en/trautonium", "text": "The trautonium is a monophonic electronic musical instrument invented about 1929 by Friedrich Trautwein in Berlin at the Musikhochschule's music and radio lab, the Rundfunkversuchstelle. Soon Oskar Sala joined him, continuing development until Sala's death in 2002. Instead of a keyboard, its manual is made of a resistor wire over a metal plate which is pressed to create a sound. Expressive playing was possible with this wire by gliding on it, creating vibrato with small movements. Volume was...", "image": "http://img.freebase.com/api/trans/raw/m/05kqj3y", "name": "Trautonium", "thumbnail": "http://indextank.com/_static/common/demo/05kqj3y.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/trautonium", "categories": {"family": "Electronic keyboard"}} +{"fields": {"url": "http://freebase.com/view/m/0151b0", "text": "The triangle is an idiophone type of musical instrument in the percussion family. It is a bar of metal, usually steel but sometimes other metals such as beryllium copper, bent into a triangle shape. The instrument is usually held by a loop of some form of thread or wire at the top curve. It was first made around the 16th century.\nOn a triangle instrument, one of the angles is left open, with the ends of the bar not quite touching. This causes the instrument to be of indeterminate or not...", "image": "http://img.freebase.com/api/trans/raw/m/02c091h", "name": "Triangle", "thumbnail": "http://indextank.com/_static/common/demo/02c091h.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/m/0151b0", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/glass_harmonica", "text": "The glass harmonica, also known as the glass armonica, bowl organ, hydrocrystalophone, or simply the armonica (derived from \"harmonia,\" the Greek word for harmony), is a type of musical instrument that uses a series of glass bowls or goblets graduated in size to produce musical tones by means of friction (instruments of this type are known as friction idiophones).\nBecause its sounding portion is made of glass, the glass harmonica is a crystallophone. The phenomenon of rubbing a wet finger...", "image": "http://img.freebase.com/api/trans/raw/m/02cl0sp", "name": "Glass harmonica", "thumbnail": "http://indextank.com/_static/common/demo/02cl0sp.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/glass_harmonica", "categories": {"family": "Crystallophone"}} +{"fields": {"url": "http://freebase.com/view/en/shakuhachi", "text": "The shakuhachi (\u5c3a\u516b, pronounced\u00a0[\u0255ak\u026fhat\u0255i]) is a Japanese end-blown flute. It is traditionally made of bamboo, but versions now exist in ABS and hardwoods. It was used by the monks of the Fuke school of Zen Buddhism in the practice of suizen (\u5439\u7985, blowing meditation). Its soulful sound made it popular in 1980s pop music in the English-speaking world.\nThey are often made in the minor pentatonic scale.\nThe name shakuhachi means \"1.8 shaku\", referring to its size. It is a compound of two...", "image": "http://img.freebase.com/api/trans/raw/m/02c3rxm", "name": "Shakuhachi", "thumbnail": "http://indextank.com/_static/common/demo/02c3rxm.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/shakuhachi", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/treble_flute", "text": "The treble flute is a member of the flute family. It is in the key of G, pitched a fifth above the concert flute and is a transposing instrument, sounding a fifth up from the written note. The instrument is rare today, only occasionally found in flute choirs, some marching bands or private collections. Some 19th century operas, such as Ivanhoe include the instrument in their orchestrations.\nA limited number of manufacturers produce G treble flute, including Myall-Allen and Flutemakers Guild....", "image": "http://img.freebase.com/api/trans/raw/m/05mg4hy", "name": "Treble flute", "thumbnail": "http://indextank.com/_static/common/demo/05mg4hy.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/treble_flute", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/jinghu", "text": "The jinghu (\u4eac\u80e1; pinyin: j\u012bngh\u00fa) is a Chinese bowed string instrument in the huqin family, used primarily in Beijing opera. It is the smallest and highest pitched instrument in the huqin family.\nLike most of its relatives, the jinghu has two strings that are customarily tuned to the interval of a fifth which the hair of the non-detachable bow passes in between. The strings were formerly made of silk, but in modern times are increasingly made of steel or nylon. Unlike other huqin instruments...", "image": "http://img.freebase.com/api/trans/raw/m/02fgb_1", "name": "Jinghu", "thumbnail": "http://indextank.com/_static/common/demo/02fgb_1.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/jinghu", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/maraca", "text": "Maracas ( pronunciation (help\u00b7info), sometimes called rumba shakers) are a native instrument of Puerto Rico, Cuba, Colombia, Guatemala and several nations of the Caribbean and Latin America. They are simple percussion instruments (idiophones), usually played in pairs, consisting of a dried calabash or gourd shell (cuia \"cue-ya\") or coconut shell filled with seeds or dried beans. They may also be made of leather, wood, or plastic.\nOften one ball is pitched high and the other is pitched low....", "image": "http://img.freebase.com/api/trans/raw/m/05mgb3m", "name": "Maracas", "thumbnail": "http://indextank.com/_static/common/demo/05mgb3m.jpg"}, "variables": {"0": 7}, "docid": "http://freebase.com/view/en/maraca", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/khim", "text": "The khim (Thai: \u0e02\u0e34\u0e21, Thai pronunciation:\u00a0[k\u02b0\u01d0m]; Khmer: \u1783\u17b9\u1798) is a hammered dulcimer from Thailand and Cambodia. It is made of wood and trapezoidal in shape, with brass strings that are laid across the instrument. There are 14 groups of strings on the khim, and each group has 3 strings. Overall, the khim has a total of 42 strings. It is played with two flexible bamboo sticks with soft leather at the tips to produce the soft tone. It is used as both a solo and ensemble instrument.\nThe...", "image": "http://img.freebase.com/api/trans/raw/m/02dbwq_", "name": "Khim", "thumbnail": "http://indextank.com/_static/common/demo/02dbwq_.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/khim", "categories": {"family": "Struck string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/guzheng", "text": "The guzheng, also spelled gu zheng or gu-zheng (Chinese: \u53e4\u7b8f; pinyin: g\u01d4zh\u0113ng, with gu \u53e4 meaning \"ancient\"); and also called zheng (\u7b8f) is a Chinese plucked zither. It has 13-21 strings and movable bridges.\nThe guzheng is a similar instrument to many Asian instruments such as the Japanese koto, the Mongolian yatga, the Korean gayageum and the Vietnamese \u0111\u00e0n tranh.\nThe guzheng should not to be confused with the guqin (another ancient Chinese zither but a fewer number of strings and without...", "image": "http://img.freebase.com/api/trans/raw/m/02b89jl", "name": "Guzheng", "thumbnail": "http://indextank.com/_static/common/demo/02b89jl.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/guzheng", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/gaita_sanabresa", "text": "The gaita sanabresa is a type of bagpipe native to Sanabria, a comarca of the province of Zamora in northwestern Spain.\nThe gaita sanabresa features a single drone. The scale of this chanter is distinct from others in Spain, more resembling the gaita transmontana in the neighboring regions of Portugal, as well as the gaita alistana of Aliste. In playing, the fingering is generally open, though some players use semi-closed touches.\nThe instrument was in decline in the 20th century and nearly...", "image": "http://img.freebase.com/api/trans/raw/m/0637vmv", "name": "Gaita sanabresa", "thumbnail": "http://indextank.com/_static/common/demo/0637vmv.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gaita_sanabresa", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/pastoral_pipes", "text": "The Pastoral Pipe (also known as the Scottish Pastoral pipes, Hybrid Union pipes, Organ pipe and Union pipe) was a bellows-blown bagpipe, widely recognised as the forerunner and ancestor of the 19th-century Union pipes, which became the Uilleann Pipes of today. Similar in design and construction, it had a foot joint in order to play a low leading note and plays a two octave chromatic scale. There is a tutor for the \"Pastoral or New Bagpipe\" by J. Geoghegan, published in London in 1745.. It...", "image": "http://img.freebase.com/api/trans/raw/m/04rvbry", "name": "Pastoral pipes", "thumbnail": "http://indextank.com/_static/common/demo/04rvbry.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/pastoral_pipes", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/gusli", "text": "Gusli (Russian: \u0413\u0443\u0441\u043b\u0438) is the oldest Russian multi-string plucked instrument. Its exact history is unknown, but it may have derived from a Byzantine form of the Greek kythare, which in turn derived from the ancient lyre. It has its relatives throughout the world - kantele in Finland, kannel in Estonia, kankles and kokle in Lithuania and Latvia. Furthermore, we can find kanun in Arabic countries and the autoharp in the USA. It is also related to such ancient instruments as Chinese gu zheng...", "image": "http://img.freebase.com/api/trans/raw/m/04252vn", "name": "Gusli", "thumbnail": "http://indextank.com/_static/common/demo/04252vn.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gusli", "categories": {"family": "Zither"}} +{"fields": {"url": "http://freebase.com/view/en/piccolo_clarinet", "text": "The piccolo clarinets are members of the clarinet family, smaller and higher pitched than the more familiar high soprano clarinets in E\u266d and D. None are common, but the most often used piccolo clarinet is the A\u266d clarinet, sounding a minor seventh higher than the B\u266d clarinet. Shackleton also lists obsolete instruments in C, B\u266d, and A\u266e. Some writers call these sopranino clarinets or octave clarinets. The boundary between the piccolo and soprano clarinets is not well-defined, and the rare...", "image": "http://img.freebase.com/api/trans/raw/m/0425c82", "name": "Piccolo clarinet", "thumbnail": "http://indextank.com/_static/common/demo/0425c82.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/piccolo_clarinet", "categories": {"family": "Clarinet"}} +{"fields": {"url": "http://freebase.com/view/en/tabla", "text": "The tabla (or tabl, tabla) (Hindi: \u0924\u092c\u0932\u093e, Marathi: \u0924\u092c\u0932\u093e, Kannada: \u0ca4\u0cac\u0cb2, Telugu: \u0c24\u0c2c\u0c32, Tamil: \u0ba4\u0baa\u0bc7\u0bb2\u0bbe, Bengali: \u09a4\u09ac\u09b2\u09be, Nepali: \u0924\u092c\u0932\u093e, Urdu: \u0637\u0628\u0644\u06c1, Arabic: \u0637\u0628\u0644\u060c \u0637\u0628\u0644\u0629\u200e) is a popular Indian percussion instrument (of the membranophone family) used in Hindustani classical music and in popular and devotional music of the Indian subcontinent. The instrument consists of a pair of hand drums of contrasting sizes and timbres. The term 'tabla is derived from an Arabic word, tabl, which simply means \"drum.\"...", "image": "http://img.freebase.com/api/trans/raw/m/029l5st", "name": "Tabla", "thumbnail": "http://indextank.com/_static/common/demo/029l5st.jpg"}, "variables": {"0": 23}, "docid": "http://freebase.com/view/en/tabla", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/gandingan_a_kayo", "text": "The gandingan a kayo (translated means, \u201cwooden gandingan,\u201d or \u201cgandingan made of wood\u201d) is a Philippine xylophone and considered the wooden version of the real gandingan. This instrument is a relatively new instrument coming of age due to the increasing popularity of the \u201cwooden kulintang ensemble,\u201d but unfortunately, there is nothing traditional about it and they cannot be used for apad, communicating long distances like the real gandingan.", "image": "http://img.freebase.com/api/trans/raw/m/0428rh2", "name": "Gandingan a Kayo", "thumbnail": "http://indextank.com/_static/common/demo/0428rh2.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gandingan_a_kayo", "categories": {"family": "Xylophone"}} +{"fields": {"url": "http://freebase.com/view/en/gusle", "text": "The gusle or lahuta (or gusla) (Albanian: lahuta, Bulgarian: \u0433\u0443\u0441\u043b\u0430, Croatian: gusle, Romanian: guzl\u0103, Serbian: \u0433\u0443\u0441\u043b\u0435), is a single-stringed musical instrument used in the Balkans and in the Dinarides region.\nThe term gusle/gusli/husli/husla is common term to all Slavic languages and denotes a musical instrument with strings. The gusle should, however, not be confused with the Russian gusli, which is a psaltery-like instrument; nor with the Czech term for violin, housle.\nThe Gusle has many...", "image": "http://img.freebase.com/api/trans/raw/m/029tqpb", "name": "Gusle", "thumbnail": "http://indextank.com/_static/common/demo/029tqpb.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gusle", "categories": {"family": "String instrument"}} +{"fields": {"url": "http://freebase.com/view/en/bugle", "text": "The bugle is one of the simplest brass instruments, having no valves or other pitch-altering devices. All pitch control is done by varying the player's embouchure, since the bugle has no other mechanism for controlling pitch. Consequently, the bugle is limited to notes within the harmonic series. See bugle call for scores to standard bugle calls, which all consist of only five notes.\nThe bugle developed from early musical or communication instruments made of animal horns, with the word...", "image": "http://img.freebase.com/api/trans/raw/m/02994bd", "name": "Bugle", "thumbnail": "http://indextank.com/_static/common/demo/02994bd.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/bugle", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/qinqin", "text": "The qinqin (\u79e6\u7434; pinyin: q\u00ednq\u00edn) is a plucked Chinese lute. It was originally manufactured with a wooden body, a slender fretted neck, and three strings. Its body can be either round, hexagonal (with rounded sides), or octagonal. Often, only two strings were used, as in certain regional silk-and-bamboo ensembles. In its hexagonal form (with rounded sides), it is also referred to as meihuaqin (\u6885\u82b1\u7434, literally \"plum blossom instrument\"). \nThe qinqin is particularly popular in southern China: in...", "image": "http://img.freebase.com/api/trans/raw/m/05mmdp1", "name": "Qinqin", "thumbnail": "http://indextank.com/_static/common/demo/05mmdp1.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/qinqin", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/masenqo", "text": "The masenqo (also spelled masenko or masinqo) is a single-string violin . The square- or diamond-shaped resonator is normally covered with parchment or rawhide. The instrument is tuned by means of a large tuning peg. As with the krar, this is an instrument used by Ethiopian minstrels, or azmaris (\"singer\" in Amharic). The masenqo requires considerable virtuosity.", "image": "http://img.freebase.com/api/trans/raw/m/02f0470", "name": "Masenqo", "thumbnail": "http://indextank.com/_static/common/demo/02f0470.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/masenqo", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/subcontrabass_saxophone", "text": "The subcontrabass saxophone is a type of saxophone that Adolphe Sax patented and planned to build but never constructed. Sax called this imagined instrument saxophone bourdon (named after the lowest stop on the pipe organ). It would have been a transposing instrument pitched in B\u266d, one octave below the bass saxophone and two octaves below the tenor saxophone.\nUntil 1999, no genuine, playable subcontrabass saxophones were made, though at least two gigantic saxophones seem to have been built...", "image": "http://img.freebase.com/api/trans/raw/m/02fx9nl", "name": "Subcontrabass saxophone", "thumbnail": "http://indextank.com/_static/common/demo/02fx9nl.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/subcontrabass_saxophone", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/galician_gaita", "text": "The (Galician) gaita or gaita de foles is a traditional bagpipe of Galicia, Asturias and northern Portugal.\nThe name gaita is used in Galician, Spanish, Leonese and Portuguese languages as a generic term for \"bagpipe\".\nJust like \"Northumbrian smallpipe\"' or \"Great Highland Bagpipe\", each country and region attributes its toponym to the respective gaita name: gaita galega (Galicia), gaita transmontana (Tr\u00e1s-os-Montes), gaita asturiana (Asturias), gaita sanabresa (Sanabria), sac de gemecs...", "image": "http://img.freebase.com/api/trans/raw/m/02c9kgg", "name": "Galician gaita", "thumbnail": "http://indextank.com/_static/common/demo/02c9kgg.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/galician_gaita", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/celesta", "text": "The celesta ( /s\u026a\u02c8l\u025bst\u0259/) or celeste ( /s\u026a\u02c8l\u025bst/) is a struck idiophone operated by a keyboard. Its appearance is similar to that of an upright piano (four- or five-octave) or of a large wooden music box (three-octave). The keys are connected to hammers which strike a graduated set of metal (usually steel) plates suspended over wooden resonators. On four or five octave models one pedal is usually available to sustain or dampen the sound. The three-octave instruments do not have a pedal, due...", "image": "http://img.freebase.com/api/trans/raw/m/0292dqw", "name": "Celesta", "thumbnail": "http://indextank.com/_static/common/demo/0292dqw.jpg"}, "variables": {"0": 7}, "docid": "http://freebase.com/view/en/celesta", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/conga", "text": "The conga (pronounced c\u016bnga), or more properly the tumbadora, is a tall, narrow, single-headed Cuban drum with African antecedents. It is thought to be derived from the Makuta drums or similar drums associated with Afro-Cubans of Central African descent. A person who plays conga is called a conguero. Although ultimately derived from African drums made from hollowed logs, the Cuban conga is staved, like a barrel. These drums were probably made from salvaged barrels originally. They are used...", "image": "http://img.freebase.com/api/trans/raw/m/029w55f", "name": "Conga", "thumbnail": "http://indextank.com/_static/common/demo/029w55f.jpg"}, "variables": {"0": 19}, "docid": "http://freebase.com/view/en/conga", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/archlute", "text": "The archlute (Spanish archila\u00fad, Italian arciliuto, German Erzlaute, Russian \u0410\u0440\u0445\u0438\u043b\u044e\u0442\u043d\u044f) is a European plucked string instrument developed around 1600 as a compromise between the very large theorbo, the size and re-entrant tuning of which made for difficulties in the performance of solo music, and the Renaissance tenor lute, which lacked the bass range of the theorbo. Essentially a tenor lute with the theorbo's neck-extension, the archlute lacks the power in the tenor and the bass that the...", "image": "http://img.freebase.com/api/trans/raw/m/02d1d7j", "name": "Archlute", "thumbnail": "http://indextank.com/_static/common/demo/02d1d7j.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/archlute", "categories": {"family": "Lute"}} +{"fields": {"url": "http://freebase.com/view/en/kantele", "text": "A kantele (pronounced [\u02c8k\u0251ntele] in Finnish) or kannel ([\u02c8k\u0251n\u02d0el] in Estonian) is a traditional plucked string instrument of the zither family native to Finland, Estonia, and Karelia. It is related to the Russian gusli, the Latvian kokle and the Lithuanian kankl\u0117s. Together these instruments make up the family known as Baltic psalteries.\nThe oldest forms of kantele have 5 or 6 horsehair strings and a wooden body carved from one piece; more modern instruments have metal strings and often a...", "image": "http://img.freebase.com/api/trans/raw/m/0291bv5", "name": "Kantele", "thumbnail": "http://indextank.com/_static/common/demo/0291bv5.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/kantele", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/bongo_drum", "text": "Bongo or bongos are a Cuban percussion instrument consisting of a pair of single-headed, open-ended drums attached to each other. The drums are of different size: the larger drum is called in Spanish the hembra (female) and the smaller the macho (male). It is most often played by hand and is especially associated in Cuban music with a steady pattern or ostinato of eighth-notes known as the martillo or \"hammer\". They are membranophones, or instruments that create sound by a vibration against...", "image": "http://img.freebase.com/api/trans/raw/m/029sm8s", "name": "Bongo drum", "thumbnail": "http://indextank.com/_static/common/demo/029sm8s.jpg"}, "variables": {"0": 13}, "docid": "http://freebase.com/view/en/bongo_drum", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/fairground_organ", "text": "A fairground organ is a pipe organ designed for use in a commercial public fairground setting to provide loud music to accompany fairground rides and attractions. Unlike organs intended for indoor use, they are designed to produce a large volume of sound to be heard over and above the noise of crowds of people and fairground machinery.\nFrom the development of street entertainment and fairs, musicians and entertainers had both mixed and creatively bounced ideas off one other. As the industry...", "image": "http://img.freebase.com/api/trans/raw/m/02f0g8z", "name": "Fairground organ", "thumbnail": "http://indextank.com/_static/common/demo/02f0g8z.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/fairground_organ", "categories": {"family": "Mechanical organ"}} +{"fields": {"url": "http://freebase.com/view/m/074z58", "text": "The regal was a small portable organ, furnished with beating reeds and having two bellows. The instrument enjoyed its greatest popularity during the Renaissance. The name was also sometimes given to the reed stops of a pipe organ, and more especially the vox humana stop.\nThe sound of the regal was produced by brass reeds held in resonators. The length of the vibrating portion of the reed determined its pitch and was regulated by means of a wire passing through the socket, the other end...", "image": "http://img.freebase.com/api/trans/raw/m/07bjnmm", "name": "Regal", "thumbnail": "http://indextank.com/_static/common/demo/07bjnmm.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/074z58", "categories": {"family": "Organ"}} +{"fields": {"url": "http://freebase.com/view/en/recorder", "text": "The recorder or English flute is a woodwind musical instrument of the family known as fipple flutes or internal duct flutes\u2014whistle-like instruments which include the tin whistle and ocarina. The recorder is end-blown and the mouth of the instrument is constricted by a wooden plug, known as a block or fipple. It is distinguished from other members of the family by having holes for seven fingers (the lower one or two often doubled to facilitate the production of semitones) and one for the...", "image": "http://img.freebase.com/api/trans/raw/m/0291nst", "name": "Recorder", "thumbnail": "http://indextank.com/_static/common/demo/0291nst.jpg"}, "variables": {"0": 75}, "docid": "http://freebase.com/view/en/recorder", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/hammered_dulcimer", "text": "The hammered dulcimer is a stringed musical instrument with the strings stretched over a trapezoidal sounding board. Typically, the hammered dulcimer is set on a stand, at an angle, before the musician, who holds small mallet hammers in each hand to strike the strings (cf. Appalachian dulcimer). The Graeco-Roman dulcimer (sweet song), derives from the Latin dulcis (sweet) and the Greek melos (song). The dulcimer's origin is uncertain, but tradition holds it was invented in Persia (Iran), as...", "image": "http://img.freebase.com/api/trans/raw/m/02bfht8", "name": "Hammered dulcimer", "thumbnail": "http://indextank.com/_static/common/demo/02bfht8.jpg"}, "variables": {"0": 44}, "docid": "http://freebase.com/view/en/hammered_dulcimer", "categories": {"family": "Struck string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/oval_spinet", "text": "The oval spinet is a type of harpsichord invented in the late 17th century by Bartolomeo Cristofori, the Italian instrument maker who later achieved fame for inventing the piano. The oval spinet was unusual for its shape, the arrangement of its strings, and for its mechanism for changing registration.\nThe two oval spinets built by Cristofori survive today. One, built in 1690, is kept in the Museo degli strumenti musicali, part of the Galleria del Accademia in Florence. The other, from 1693,...", "image": "http://img.freebase.com/api/trans/raw/m/041ng_b", "name": "Oval spinet", "thumbnail": "http://indextank.com/_static/common/demo/041ng_b.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/oval_spinet", "categories": {"family": "Harpsichord"}} +{"fields": {"url": "http://freebase.com/view/m/03cljxs", "text": "The helicon is a brass musical instrument in the tuba family. Most are BB\u266d basses, but they also commonly exist in EE\u266d, F, and tenor sizes, as well as other types to a lesser extent.\nThe sousaphone is a specialized version of helicon, differing primarily in two ways: a Sousaphone bell is shaped to face forwards and has a larger flare, a Sousaphone has a \"goose-neck\" leadpipe which offers greater adjustability of mouthpiece position at the expense of tone quality, while both instruments have...", "image": "http://img.freebase.com/api/trans/raw/m/02d7_c9", "name": "Helicon", "thumbnail": "http://indextank.com/_static/common/demo/02d7_c9.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/03cljxs", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/english_concertina", "text": "The English concertina is a free-reed instrument invented by Charles Wheatstone in 1829. It is fully chromatic, and plays the same tones whether contracting or expanding the bellows.\nKeys are arranged in four horizontal rows on each side. The natural notes are in the middle two rows; low notes are closest to the player, and alternate between the left and right side as they ascend (e.g., C may be on the left, then D on the right). Any two notes next to each other in the middle two rows form a...", "image": "http://img.freebase.com/api/trans/raw/m/0292mbm", "name": "English concertina", "thumbnail": "http://indextank.com/_static/common/demo/0292mbm.jpg"}, "variables": {"0": 5}, "docid": "http://freebase.com/view/en/english_concertina", "categories": {"family": "Concertina"}} +{"fields": {"url": "http://freebase.com/view/en/chamberlin", "text": "The Chamberlin is an electro-mechanical keyboard instrument that was a precursor to the Mellotron. It was developed and patented by Iowa, Wisconsin inventor Harry Chamberlin from 1949 to 1956, when the first model was introduced. Various models and versions of these Chamberlin music instruments exist. While most are keyboard-based instruments, there were also early drum machines produced and sold. Some of these drums patterns feature Harry Chamberlin's son Richard on them.\nHarry Chamberlin's...", "image": "http://img.freebase.com/api/trans/raw/m/02b965p", "name": "Chamberlin", "thumbnail": "http://indextank.com/_static/common/demo/02b965p.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/chamberlin", "categories": {"family": "Electric piano"}} +{"fields": {"url": "http://freebase.com/view/en/bass_guitar", "text": "The bass guitar (also called electric bass, or simply bass; /\u02c8be\u026as/) is a stringed instrument played primarily with the fingers or thumb (by plucking, slapping, popping, tapping, or thumping), or by using a pick.\nThe bass guitar is similar in appearance and construction to an electric guitar, but with a longer neck and scale length, and four, five, or six strings. The four-string bass\u2014by far the most common\u2014is usually tuned the same as the double bass, which corresponds to pitches one...", "image": "http://img.freebase.com/api/trans/raw/m/02bc8zj", "name": "Bass guitar", "thumbnail": "http://indextank.com/_static/common/demo/02bc8zj.jpg"}, "variables": {"0": 2274}, "docid": "http://freebase.com/view/en/bass_guitar", "categories": {"family": "Guitar"}} +{"fields": {"url": "http://freebase.com/view/en/saxophone", "text": "The saxophone (also referred to as the sax) is a conical-bore transposing musical instrument that is a member of the woodwind family. Saxophones are usually made of brass and played with a single-reed mouthpiece similar to that of the clarinet. The saxophone was invented by the Belgian Adolphe Sax in 1841. He wanted to create an instrument that would both be the most powerful and vocal of the woodwinds and the most adaptive of the brass, which would fill the then vacant middle ground between...", "image": "http://img.freebase.com/api/trans/raw/m/05kh825", "name": "Saxophone", "thumbnail": "http://indextank.com/_static/common/demo/05kh825.jpg"}, "variables": {"0": 814}, "docid": "http://freebase.com/view/en/saxophone", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/pipe_and_tabor", "text": "Pipe and tabor is a pair of instruments played by a single player, consisting of a three-hole pipe played with one hand, and a small drum played with the other. The tabor (drum) hangs on the performer's left arm or around the neck, leaving the hands free to beat the drum with a stick in the right hand and play the pipe with thumb and first two fingers of the left hand.\nThe pipe is made out of wood, metal or plastic and consists of a cylindrical tube of narrow bore (1:40 diameter:length...", "image": "http://img.freebase.com/api/trans/raw/m/02g57g5", "name": "Pipe and Tabor", "thumbnail": "http://indextank.com/_static/common/demo/02g57g5.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/pipe_and_tabor", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/melodica", "text": "The melodica, also known as the \"blow-organ\" or \"key-flute\", is a free-reed instrument similar to the melodeon and harmonica. It has a musical keyboard on top, and is played by blowing air through a mouthpiece that fits into a hole in the side of the instrument. Pressing a key opens a hole, allowing air to flow through a reed. The keyboard is usually two or three octaves long. Melodicas are small, light, and portable. They are popular in music education, especially in Asia.\nThe modern form...", "image": "http://img.freebase.com/api/trans/raw/m/02bpzhj", "name": "Melodica", "thumbnail": "http://indextank.com/_static/common/demo/02bpzhj.jpg"}, "variables": {"0": 50}, "docid": "http://freebase.com/view/en/melodica", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/m/0fwc52", "text": "The triple harp, or more often referred to as the Welsh Triple Harp, (Welsh: Telyn deires) is a type of harp employing three rows of strings instead of the common single row. The Welsh triple harp today is found mainly among players of traditional Welsh folk music.\nThe triple harp first originated in Italy, under the form of two rows of strings and later three, as the baroque harp (Italian: Arpa Doppia). It appeared in the British Isles early in the 17th century. In 1629, the French harpist...", "image": "http://img.freebase.com/api/trans/raw/m/0bcj7yk", "name": "Triple Harp", "thumbnail": "http://indextank.com/_static/common/demo/0bcj7yk.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/0fwc52", "categories": {"family": "Harp"}} +{"fields": {"url": "http://freebase.com/view/en/glockenspiel", "text": "A glockenspiel (German pronunciation:\u00a0[\u02c8\u0261l\u0254k\u0259n\u02cc\u0283pi\u02d0l]) is a percussion instrument composed of a set of tuned keys arranged in the fashion of the keyboard of a piano. In this way, it is similar to the xylophone; however, the xylophone's bars are made of wood, while the glockenspiel's are metal plates or tubes, thus making it a metallophone. The glockenspiel, moreover, is usually smaller and higher in pitch.\nIn German, a carillon is also called a Glockenspiel.\nWhen used in a marching or...", "image": "http://img.freebase.com/api/trans/raw/m/02bnygk", "name": "Glockenspiel", "thumbnail": "http://indextank.com/_static/common/demo/02bnygk.jpg"}, "variables": {"0": 15}, "docid": "http://freebase.com/view/en/glockenspiel", "categories": {"family": "Tuned percussion"}} +{"fields": {"url": "http://freebase.com/view/m/02f5t8", "text": "A fife is a small, high-pitched, transverse flute that is similar to the piccolo, but louder and shriller due to its narrower bore. The fife originated in medieval Europe and is often used in military and marching bands. Someone who plays the fife is called a fifer. The word fife comes from the German Pfeife, or pipe, ultimately derived from the Latin word pipare.\nThe fife is a simple instrument usually consisting of a tube with 6 finger holes, and diatonically tuned. Some have 10 or 11...", "image": "http://img.freebase.com/api/trans/raw/m/02cw2vb", "name": "Fife", "thumbnail": "http://indextank.com/_static/common/demo/02cw2vb.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/m/02f5t8", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/cimpoi", "text": "Cimpoi, the Romanian bagpipe, has a single drone and straight bore chanter and is less strident than its Balkan relatives.\nThe number of finger holes varies from five to eight and there are two types of cimpoi with a double chanter. The bag is often covered with embroidered cloth. The bagpipe can be found in most of Romania apart from the central, northern and eastern parts of Transylvania, but at present (the early 21st century) is only played by a few elderly people.\nA well-known player of...", "image": "http://img.freebase.com/api/trans/raw/m/041xg96", "name": "Cimpoi", "thumbnail": "http://indextank.com/_static/common/demo/041xg96.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/cimpoi", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/kulintang_a_kayo", "text": "The kulintang a kayo (literally, \u201cwooden kulintang\u201d) is a Philippine xylophone of the Maguindanaon people with eight tuned slabs arranged horizontally atop a wooden antangan (rack). Made of soft wood such as bayug, the kulintang a kayo is a common found among Maguindanaon households with a musical background. Traditionally, it was used for self-entertainment purpose inside the house, so beginners could practice kulintang pieces before performing them on the real kulintang and only recently...", "image": "http://img.freebase.com/api/trans/raw/m/041rgl8", "name": "Kulintang a Kayo", "thumbnail": "http://indextank.com/_static/common/demo/041rgl8.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/kulintang_a_kayo", "categories": {"family": "Xylophone"}} +{"fields": {"url": "http://freebase.com/view/en/ranat_ek", "text": "The ranat ek (Thai: \u0e23\u0e30\u0e19\u0e32\u0e14\u0e40\u0e2d\u0e01, pronounced\u00a0[ran\u00e2\u02d0t \u0294\u00e8\u02d0k], \"alto xylophone\") is a Thai xylophone. It has 21 or 22 wooden bars suspended by cords over a boat-shaped trough resonator, and is played with two mallets. It is used as a leading instrument in the piphat ensemble.\nThe ranat ek is played by two types of mallets. The hard mallets create the sharp bright sound when they keys are hit.The hard mallets are used for more faster playing. The soft mallets create a mellow and more softer tone...", "image": "http://img.freebase.com/api/trans/raw/m/05t3hzy", "name": "Ranat ek", "thumbnail": "http://indextank.com/_static/common/demo/05t3hzy.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/ranat_ek", "categories": {"family": "Xylophone"}} +{"fields": {"url": "http://freebase.com/view/en/mandola", "text": "The mandola (US and Canada) or tenor mandola (Ireland, and UK) is a fretted, stringed musical instrument. It is to the mandolin what the viola is to the violin: the four double courses of strings tuned in fifths to the same pitches as the viola (C-G-D-A low-to-high), a fifth lower than a mandolin. However, the mandola, though now rarer, is the ancestor of the mandolin, the name of which means simply \"little mandola\".\nThe name mandola may originate with the ancient pandura, and was also...", "image": "http://img.freebase.com/api/trans/raw/m/02f0bjz", "name": "Mandola", "thumbnail": "http://indextank.com/_static/common/demo/02f0bjz.jpg"}, "variables": {"0": 6}, "docid": "http://freebase.com/view/en/mandola", "categories": {"family": "Mandolin"}} +{"fields": {"url": "http://freebase.com/view/en/clarinet", "text": "The clarinet is a musical instrument of woodwind type. The name derives from adding the suffix -et (meaning little) to the Italian word clarino (meaning a type of trumpet), as the first clarinets had a strident tone similar to that of a trumpet. The instrument has an approximately cylindrical bore, and uses a single reed. In jazz contexts, it has sometimes been informally referred to as the \"licorice stick.\"\nClarinets comprise a family of instruments of differing sizes and pitches. The...", "image": "http://img.freebase.com/api/trans/raw/m/02bctx9", "name": "Clarinet", "thumbnail": "http://indextank.com/_static/common/demo/02bctx9.jpg"}, "variables": {"0": 291}, "docid": "http://freebase.com/view/en/clarinet", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/gayageum", "text": "The gayageum or kayagum is a traditional Korean zither-like string instrument, with 12 strings, although more recently variants have been constructed with 21 or other numbers of strings. It is probably the best known traditional Korean musical instrument. It is in the zither family. It is very similar to the Chinese guzheng, or table harp, from which it is derived.\nAccording to the Samguksagi (1145), a history of the Three Kingdoms of Korea, the gayageum is supposed to have been developed...", "image": "http://img.freebase.com/api/trans/raw/m/02cp5bd", "name": "Gayageum", "thumbnail": "http://indextank.com/_static/common/demo/02cp5bd.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gayageum", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/mohan_veena", "text": "The Mohan veena (or Vishwa veena) is a stringed musical instrument used in Indian classical music. It derives its name from its inventor Pandit Vishwa Mohan Bhatt\nThe instrument is actually a modified Archtop guitar and consists of 20 strings viz. three melody strings, five drone strings strung to the peghead, and twelve sympathetic strings strung to the tuners mounted on the side of the neck. A gourd (or the tumba) is screwed into the back of the neck for improved sound quality and...", "image": "http://img.freebase.com/api/trans/raw/m/02cnzd7", "name": "Mohan veena", "thumbnail": "http://indextank.com/_static/common/demo/02cnzd7.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/mohan_veena", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/cornet", "text": "The cornet is a brass instrument very similar to the trumpet, distinguished by its conical bore, compact shape, and mellower tone quality. The most common cornet is a transposing instrument in B\u266d. It is not related to the renaissance and early baroque cornett or cornetto.\nThe cornet was originally derived from the post horn around 1820 in France. Among the first manufacturers of modern cornets were Parisian Jean Aste' in 1928. Cornets first appear as separate instrumental parts in 19th...", "image": "http://img.freebase.com/api/trans/raw/m/02910xc", "name": "Cornet", "thumbnail": "http://indextank.com/_static/common/demo/02910xc.jpg"}, "variables": {"0": 64}, "docid": "http://freebase.com/view/en/cornet", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/semi-acoustic_guitar", "text": "A semi-acoustic guitar or hollow-body electric is a type of electric guitar with both a sound box and one or more electric pickups. This is not the same as an electric acoustic guitar, which is an acoustic guitar with the addition of pickups or other means of amplification, either added by the manufacturer or the player.\nOther semi-acoustic or acoustic electric instruments include basses and mandolins. These are similarly constructed to semi-acoustic guitars, and are used in the same ways...", "image": "http://img.freebase.com/api/trans/raw/m/044njy7", "name": "Semi-acoustic guitar", "thumbnail": "http://indextank.com/_static/common/demo/044njy7.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/semi-acoustic_guitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/daxophone", "text": "The daxophone, invented by Hans Reichel, is a experimental musical instrument of the friction idiophones category. It consists of a thin wooden blade fixed in a wooden block (often attached to a tripod), which holds one or more contact microphones. Normally, it is played by bowing the free end, but it can also be struck or plucked, which propagates sound in the same way a ruler halfway off a table does. These vibrations then continue to the wooden-block base, which in turn is amplified by...", "image": "http://img.freebase.com/api/trans/raw/m/07876hm", "name": "Daxophone", "thumbnail": "http://indextank.com/_static/common/demo/07876hm.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/daxophone", "categories": {"family": "Idiophone"}} +{"fields": {"url": "http://freebase.com/view/en/cor_anglais", "text": "The cor anglais (British English and French), or English horn (American English), is a double-reed woodwind instrument in the oboe family.\nThe cor anglais is a transposing instrument pitched in F, a perfect fifth lower than the oboe (a C instrument), and is consequently approximately one and a half times the length of the oboe. The fingering and playing technique used for the cor anglais are essentially the same as those of the oboe. Music for the cor anglais is thus written a perfect fifth...", "image": "http://img.freebase.com/api/trans/raw/m/02btjlp", "name": "Cor anglais", "thumbnail": "http://indextank.com/_static/common/demo/02btjlp.jpg"}, "variables": {"0": 4}, "docid": "http://freebase.com/view/en/cor_anglais", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/chemnitzer_concertina", "text": "A Chemnitzer concertina is a musical instrument of the hand-held bellows-driven free-reed category, sometimes called squeezeboxes. The Chemnitzer concertina is most closely related to the Bandone\u00f3n (German spelling: Bandonion), more distantly to the other concertinas, and accordions.\nIt is roughly square in cross-section, with the keyboards consisting of cylindrical buttons on each end arranged in curving rows. Like other concertinas, the buttons travel in a direction approximately parallel...", "image": "http://img.freebase.com/api/trans/raw/m/02bldk5", "name": "Chemnitzer concertina", "thumbnail": "http://indextank.com/_static/common/demo/02bldk5.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/chemnitzer_concertina", "categories": {"family": "Concertina"}} +{"fields": {"url": "http://freebase.com/view/en/ghaychak", "text": "The Ghaychak or Ghijak is a round-bodied musical instrument with 3 or 4 metal strings and a short fretless neck. It is used by Iranians, Afghans, Uzbeks, Uyghurs, Tajiks, Turkmens and Qaraqalpaks.\nThe ghidjak, or violin, is the only bow instrument found in the Pamirs. The ghidjak is very popular throughout Central Asia. Its sound box is metal or wooden, and it has three or four metal strings and a neck made of willow, apricot or mulberry wood. It is tuned in intervals of fourths.\nThe...", "image": "http://img.freebase.com/api/trans/raw/m/02gfwwq", "name": "Ghaychak", "thumbnail": "http://indextank.com/_static/common/demo/02gfwwq.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/ghaychak", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/contrabass_flute", "text": "The contrabass flute is one of the rarer members of the flute family. It is used mostly in flute ensembles. Its range is similar to that of the regular concert flute, except that it is pitched two octaves lower; the lowest performable note is two octaves below middle C (the lowest C on the cello). Many contrabass flutes in C are also equipped with a low B, (in the same manner as many modern standard sized flutes are.) Contrabass flutes are only available from select flute makers.\nSometimes...", "image": "http://img.freebase.com/api/trans/raw/m/02cffny", "name": "Contrabass flute", "thumbnail": "http://indextank.com/_static/common/demo/02cffny.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/contrabass_flute", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/bass_drum", "text": "The bass drums are of variable sizes and are used in several musical genres (see usage below). Three major types of bass drums can be distinguished. The type usually seen or heard in orchestral, ensemble or concert band music is the orchestral, or concert bass drum (in Italian: gran cassa, gran tamburo). It is the largest drum of the orchestra. The kick drum, struck with a beater attached to a pedal, is usually seen on drum kits. The third type, the pitched bass drum, is generally used in...", "image": "http://img.freebase.com/api/trans/raw/m/02bdbkf", "name": "Bass drum", "thumbnail": "http://indextank.com/_static/common/demo/02bdbkf.jpg"}, "variables": {"0": 6}, "docid": "http://freebase.com/view/en/bass_drum", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/surbahar", "text": "Surbahar (Urdu: \u0633\u0631\u0628\u06c1\u0627\u0631; Hindi: \u0938\u0941\u0930 \u092c\u0939\u093e\u0930), sometimes known as bass sitar, is a plucked string instrument used in the Hindustani classical music of North India. It is closely related to sitar, but it has a lower tone. Depending on the instrument's size, it is usually pitched two to five whole steps below the standard sitar, but Indian classical music having no concept of absolute pitch, this may vary. The instrument can emit frequencies lower than 20\u00a0Hz.\nSurbahar is over 130\u00a0cm (51\u00a0inches). It...", "image": "http://img.freebase.com/api/trans/raw/m/02d411z", "name": "Surbahar", "thumbnail": "http://indextank.com/_static/common/demo/02d411z.jpg"}, "variables": {"0": 4}, "docid": "http://freebase.com/view/en/surbahar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/crwth", "text": "The crwth is an archaic stringed musical instrument, associated particularly with Welsh music, once widely-played in Europe.\nCrwth is originally a Welsh word, in English pronounced /\u02c8kru\u02d0\u03b8/ (rhyming with \"truth\") or /\u02c8kr\u028a\u03b8/ (with the vowel of \"push\"). The traditional English name is crowd (or rote), and the variants crwd, crout and crouth are little used today. In Medieval Latin it is called the chorus or crotta. The Welsh word crythor means a performer on the crwth. The Irish word is cruit,...", "image": "http://img.freebase.com/api/trans/raw/m/029gk01", "name": "Crwth", "thumbnail": "http://indextank.com/_static/common/demo/029gk01.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/crwth", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/cymbalum", "text": "The cimbalom is a concert hammered dulcimer: a type of chordophone composed of a large, trapezoidal box with metal strings stretched across its top. It is a musical instrument commonly found throughout the group of East European nations and cultures which composed Austria-Hungary (1867\u20131918), namely contemporary Belarus, Hungary, Romania, Moldova, Ukraine, Poland, the Czech Republic and Slovakia. The cimbalom is (typically) played by striking two beaters against the strings. The steel treble...", "image": "http://img.freebase.com/api/trans/raw/m/029sh3x", "name": "Cymbalum", "thumbnail": "http://indextank.com/_static/common/demo/029sh3x.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/cymbalum", "categories": {"family": "Struck string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/kutiyapi", "text": "The kutiyapi, is a Philippine two-stringed, fretted boat-lute. It is the only stringed instrument among the Maguindanao people, and one of several among other groups such as the Maranao and Manobo. It is four to six feet long with nine frets made of hardened beeswax. The instrument is carved out of solid soft wood such as from the jackfruit tree.\nCommon to all kutiyapi instruments, a constant drone is played with one string while the other, an octave above the drone, plays the melody with a...", "image": "http://img.freebase.com/api/trans/raw/m/042m46_", "name": "Kutiyapi", "thumbnail": "http://indextank.com/_static/common/demo/042m46_.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/kutiyapi", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/sac_de_gemecs", "text": "The sac de gemecs (Catalan: literally \"bag of moans\", alternately buna in Andorra or coixinera, gaita or botella) is a type of bagpipe found in Catalonia (eastern Spain spilling over into southern France).\nThe instrument consists of a chanter, a mouthblown blowpipe, and three drones. The lowest drone (bord\u00f3 llarg) plays a note two octaves below the tonic of the chanter. The middle drone (bord\u00f3 mitj\u00e0 ) plays a fifth above the bass. The high drone (bord\u00f3 petit) plays an octave below the...", "image": "http://img.freebase.com/api/trans/raw/m/0635qpl", "name": "Sac de gemecs", "thumbnail": "http://indextank.com/_static/common/demo/0635qpl.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/sac_de_gemecs", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/octobass", "text": "The octobass is an extremely large bowed string instrument constructed about 1850 in Paris by the French luthier Jean Baptiste Vuillaume (1798-1875). It has three strings and is essentially a larger version of the double bass (the specimen in the collection of the Mus\u00e9e de la Musique in Paris measures 3.48 meters in length, whereas a full size double bass is generally approximately 2 meters in length). Because of the impractically large size of its fingerboard and thickness of its strings,...", "image": "http://img.freebase.com/api/trans/raw/m/02cmycm", "name": "Octobass", "thumbnail": "http://indextank.com/_static/common/demo/02cmycm.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/octobass", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/dulcian", "text": "The dulcian is a Renaissance bass woodwind instrument, with a double reed and a folded conical bore. Equivalent terms include \"curtal\" in English, \"dulzian\" in German, \"baj\u00f3n\" in Spanish, \"dou\u00e7aine\"' in French, \"dulciaan\" in Dutch, and \"dulciana\" in Italian.\nThe predecessor of the modern bassoon, it flourished between 1550 and 1700, but was probably invented earlier. Towards the end of this period it co-existed with, and was then superseded by the baroque bassoon, although it continued to be...", "image": "http://img.freebase.com/api/trans/raw/m/03r1glg", "name": "Dulcian", "thumbnail": "http://indextank.com/_static/common/demo/03r1glg.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/dulcian", "categories": {"family": "Bassoon"}} +{"fields": {"url": "http://freebase.com/view/en/huemmelchen", "text": "The h\u00fcmmelchen is a type of small German bagpipe, attested in Syntagma Musicum by Michael Praetorius during the Renaissance.\nEarly versions are believed to have double-reeded chanters, most likely with single-reeded drones.\nThe word \"h\u00fcmmelchen\" probably comes from the Low German word h\u00e4meln meaning \"trim\". This may refer to the h\u00fcmmelchen's small size, resembling a trimmed-down version of a larger bagpipe. Another possibly etymology comes from the word hummel (\"bumble-bee\"), referring to...", "image": "http://img.freebase.com/api/trans/raw/m/07blxb9", "name": "Huemmelchen", "thumbnail": "http://indextank.com/_static/common/demo/07blxb9.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/huemmelchen", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/hi-hat", "text": "A hi-hat, or hihat, is a type of cymbal and stand used as a typical part of a drum kit by percussionists in R&B, hip-hop, disco, jazz, rock and roll, house, reggae and other forms of contemporary popular music.\nThe hi-hat consists of two cymbals that are mounted on a stand, one on top of the other, and clashed together using a pedal on the stand. A narrow metal shaft or rod runs through both cymbals into a hollow tube and connects to the pedal. The top cymbal is connected to the rod with a...", "image": "http://img.freebase.com/api/trans/raw/m/02bdbkf", "name": "Hi-hat", "thumbnail": "http://indextank.com/_static/common/demo/02bdbkf.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/hi-hat", "categories": {"family": "Cymbal"}} +{"fields": {"url": "http://freebase.com/view/en/sihu", "text": "The sihu (\u56db\u80e1; pinyin: s\u00ech\u00fa) is a Chinese bowed string instrument with four strings. It is a member of the huqin family of instruments.\nThe instrument's name comes from the words s\u00ec (\u56db, meaning \"four\" in Chinese, referring to the instrument's number of strings) and h\u00fa (\u80e1, short for huqin, the family of instruments of which the sihu is a member). Its soundbox and neck are made from hardwood and the playing end of the soundbox is covered with python, cow, or sheep skin.\nThere are several sizes...", "image": "http://img.freebase.com/api/trans/raw/m/04pr37b", "name": "Sihu", "thumbnail": "http://indextank.com/_static/common/demo/04pr37b.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/sihu", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/tricordia", "text": "A tricordia (also trichordia or tricordio) or mandriola is a twelve-stringed variation of the mandolin. The tricordia is used in Mexican folk music, while its European cousin, the mandriola, is used identically to the mandolin. It differs from a standard mandolin in that it has three strings per course. Tricordias only use unison tuning (ggg d'd'd' a'a'a' e\"e\"e\"), while mandriolas use either unison tuning or octave tuning (Ggg dd'd' aa'a' e'e\"e\").", "image": "http://img.freebase.com/api/trans/raw/m/04rymhg", "name": "Tricordia", "thumbnail": "http://indextank.com/_static/common/demo/04rymhg.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/tricordia", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/m/04gtkv6", "text": "The word la\u00fad is the Spanish word for lute. It is most commonly used to refer to a plectrum-plucked chordophone from Spain. It belongs to the cittern family of instruments. It has six double courses (i.e. twelve strings in pairs), similarly to the bandurria, but its neck is longer. Traditionally it is used folk string musical groups, together with the guitar and the bandurria.\nLike the bandurria, it is tuned in fourths, but its range is 1 octave lower.\nTuning:\nThere is also a Cuban variety...", "image": "http://img.freebase.com/api/trans/raw/m/05m1djr", "name": "La\u00fad", "thumbnail": "http://indextank.com/_static/common/demo/05m1djr.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/04gtkv6", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/kobza", "text": "The kobza (Ukrainian: \u043a\u043e\u0431\u0437\u0430) is a Ukrainian folk music instrument of the lute family (Hornbostel-Sachs classification number 321.321-5+6), a relative of the Central European mandora. The term kobza however, has also been applied to a number of other Eastern European instruments distinct from the Ukrainian kobza.\nThe Ukrainian kobza was traditionally gut-strung, with a body hewn from a single block of wood. Instruments with a staved assembly also exist. The kobza has a medium length neck...", "image": "http://img.freebase.com/api/trans/raw/m/02bnwxh", "name": "Kobza", "thumbnail": "http://indextank.com/_static/common/demo/02bnwxh.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/kobza", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/gudok", "text": "The gudok or hudok (Russian: \u0433\u0443\u0434\u043e\u043a, Ukrainian: \u0433\u0443\u0434\u043e\u043a) is an ancient Eastern Slavic string musical instrument, played with a bow.\nA gudok usually had three strings, two of them tuned in unison and played as a drone, the third tuned a fifth higher. All three strings were in the same plane at the bridge, so that a bow could make them all sound simultaneously. Sometimes the gudok also had several sympathetic strings (up to eight) under the sounding board. These made the gudok's sound warm and...", "image": "http://img.freebase.com/api/trans/raw/m/02cmhjg", "name": "Gudok", "thumbnail": "http://indextank.com/_static/common/demo/02cmhjg.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gudok", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/clavichord", "text": "The clavichord is a European stringed keyboard instrument known from the late Medieval, through the Renaissance, Baroque and Classical eras. Historically, it was widely used as a practice instrument and as an aid to composition, not being loud enough for larger performances. The clavichord produces sound by striking brass or iron strings with small metal blades called tangents. Vibrations are transmitted through the bridge(s) to the soundboard. The name is derived from the Latin word clavis,...", "image": "http://img.freebase.com/api/trans/raw/m/041bht2", "name": "Clavichord", "thumbnail": "http://indextank.com/_static/common/demo/041bht2.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/clavichord", "categories": {"family": "Keyboard instrument"}} +{"fields": {"url": "http://freebase.com/view/en/double_bass", "text": "The double bass, also called the string bass, upright bass or contrabass, is the largest and lowest-pitched bowed string instrument in the modern symphony orchestra, with strings usually tuned to E1, A1, D2 and G2 (see standard tuning). The double bass is a standard member of the string section of the symphony orchestra and smaller string ensembles in Western classical music. In addition, it is used in other genres such as jazz, 1950s-style blues and rock and roll, rockabilly/psychobilly,...", "image": "http://img.freebase.com/api/trans/raw/m/03s_2wc", "name": "Double bass", "thumbnail": "http://indextank.com/_static/common/demo/03s_2wc.jpg"}, "variables": {"0": 454}, "docid": "http://freebase.com/view/en/double_bass", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/kanun", "text": "The Qanun (Azerbaijani\u00a0; Turkish: kanun; qan\u00fan or kanun (Arabic: \u0642\u0627\u0646\u0648\u0646 q\u0101n\u016bn, plural \u0642\u0648\u0627\u0646\u064a\u0646 qaw\u0101n\u012bn; Persian: \u0642\u0627\u0646\u0648\u0646 q\u0101n\u016bn;Armenian: \u0584\u0561\u0576\u0578\u0576 k\u2019anon; Greek: \u03ba\u03b1\u03bd\u03bf\u03bd\u03ac\u03ba\u03b9) is a string instrument found in the 10th century in Farab in Turkestan. The name derives from the Greek word \u03ba\u03b1\u03bd\u03ce\u03bd, \"kanon,\" which means rule, principle, and also \"mode.\" Its traditional music is based on Maqamat. It is essentially a zither with a narrow trapezoidal soundboard. Nylon or PVC strings are stretched over a single...", "image": "http://img.freebase.com/api/trans/raw/m/02cd8d4", "name": "Kanun", "thumbnail": "http://indextank.com/_static/common/demo/02cd8d4.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/kanun", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/tres", "text": "The tres is a 3-course, 6-string chordophone which was created in Cuba. \nIn Cuba, among the criollo class, the son was created as a song and a salon dance genre. Originally, a guitar, tiple or bandola, played rhythm and lead in the son, but ultimately these were replaced by a new native-born instrument which was a fusion of all three called the Cuban Tres.\nThe Cuban tres has three courses (groups) of two strings each for a total of six strings. From the low pitch to the highest, the...", "image": "http://img.freebase.com/api/trans/raw/m/0cmv5ww", "name": "Tres", "thumbnail": "http://indextank.com/_static/common/demo/0cmv5ww.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/tres", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/great_highland_bagpipe", "text": "The Great Highland Bagpipe (Scottish Gaelic: a' ph\u00ecob mh\u00f2r; often abbreviated GHB in English) is a type of bagpipe native to Scotland. It has achieved widespread recognition through its usage in the British military and in pipe bands throughout the world. It is closely related to the Great Irish Warpipes.\nThe bagpipe is first attested in Scotland around 1400 AD, having previously appeared in European artwork in Spain in the 13th century. The earliest references to bagpipes in Scotland are in...", "image": "http://img.freebase.com/api/trans/raw/m/02c9k5h", "name": "Great Highland Bagpipe", "thumbnail": "http://indextank.com/_static/common/demo/02c9k5h.jpg"}, "variables": {"0": 17}, "docid": "http://freebase.com/view/en/great_highland_bagpipe", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/holztrompete", "text": "A Holztrompete (Wooden Trumpet), an instrument somewhat resembling the Alpenhorn in tone-quality, designed by Richard Wagner for representing the natural pipe of the peasant in Tristan und Isolde. This instrument is not unlike the cor anglais in rough outline, being a conical tube of approximately the same length, terminating in a small globular bell, but having neither holes nor keys; it is blown through a cupshaped mouthpiece made of horn. The Holztrompete is in the key of C; the scale is...", "image": "http://img.freebase.com/api/trans/raw/m/062_x0j", "name": "Holztrompete", "thumbnail": "http://indextank.com/_static/common/demo/062_x0j.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/holztrompete", "categories": {"family": "Alphorn"}} +{"fields": {"url": "http://freebase.com/view/en/classical_guitar", "text": "The classical guitar \u2014 (also called the \"Spanish guitar\" or \"nylon string guitar\") \u2014 is a 6-stringed plucked string instrument from the family of instruments called chordophones. The classical guitar is well known for its comprehensive right hand technique, which allows the soloist to perform complex melodic and polyphonic material, in much the same manner as the piano.\nThe name classical guitar does not mean that only classical repertoire is performed on it, although classical music is a...", "image": "http://img.freebase.com/api/trans/raw/m/02ns86s", "name": "Classical guitar", "thumbnail": "http://indextank.com/_static/common/demo/02ns86s.jpg"}, "variables": {"0": 30}, "docid": "http://freebase.com/view/en/classical_guitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/contrabass_clarinet", "text": "The contrabass clarinet is the largest member of the clarinet family that has ever been in regular production or significant use. Modern contrabass clarinets are pitched in BB\u266d, sounding two octaves lower than the common B\u266d soprano clarinet and one octave lower than the B\u266d bass clarinet. Some contrabass clarinet models have a range extending down to low (written) E\u266d, while others can play down to low D or further to low C. Some early instruments were pitched in C; Arnold Schoenberg's F\u00fcnf...", "image": "http://img.freebase.com/api/trans/raw/m/029xqmr", "name": "Contrabass clarinet", "thumbnail": "http://indextank.com/_static/common/demo/029xqmr.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/contrabass_clarinet", "categories": {"family": "Clarinet"}} +{"fields": {"url": "http://freebase.com/view/en/dc_3", "text": "DC-3 guitars were manufactured by Danelectro. A small number of DC-3's were manufactured in the late 1990s. The DC-3's design is based on classical Danelectro models, such as the DC-59. The DC-3 has three pickups, whereas the DC-59, only two. The \"DC\" stands for 'double cutaway'.", "image": "http://img.freebase.com/api/trans/raw/m/029rynx", "name": "DC-3", "thumbnail": "http://indextank.com/_static/common/demo/029rynx.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/dc_3", "categories": {"family": "Electric guitar"}} +{"fields": {"url": "http://freebase.com/view/en/krar", "text": "The krar is a five- or six-stringed bowl-shaped lyre from Eritrea and Ethiopia. The instrument is tuned to a pentatonic scale. A modern krar may be amplified, much in the same way as an electric guitar or violin.\nThe krar, a chordophone, is usually decorated with wood, cloth, and beads. Its five or six strings determine the available pitches. The instrument's tone depends on the musician's playing technique: bowing, strumming or plucking. If plucked, the instrument will produce a soft tone....", "image": "http://img.freebase.com/api/trans/raw/m/029gh0r", "name": "Krar", "thumbnail": "http://indextank.com/_static/common/demo/029gh0r.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/krar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/harp", "text": "The harp is a multi-stringed instrument which has the plane of its strings positioned perpendicularly to the soundboard. Organologically, it falls in the general category of chordophones (stringed instruments) and occupies its own sub category (the harps). All harps have a neck, resonator, and strings. Some, known as frame harps, also have a pillar; those lacking the pillar are referred to as open harps. Depending on its size (which varies considerably), a harp may be played while held in...", "image": "http://img.freebase.com/api/trans/raw/m/02d48ns", "name": "Harp", "thumbnail": "http://indextank.com/_static/common/demo/02d48ns.jpg"}, "variables": {"0": 48}, "docid": "http://freebase.com/view/en/harp", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/dutar", "text": "The dutar ( Persian: \u062f\u0648 \u062a\u0627\u0631 , Uzbek: dutor) (also dotar or doutar) is a traditional long-necked two-stringed lute found in Iran, Central Asia and South Asia. Its name comes from the Persian word for \"two strings\", \u062f\u0648 \u062a\u0627\u0631 dot\u0101r (", "image": "http://img.freebase.com/api/trans/raw/m/041g743", "name": "Dutar", "thumbnail": "http://indextank.com/_static/common/demo/041g743.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/dutar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/player_piano", "text": "A player piano (also known as pianola or autopiano) is a self-playing piano, containing a pneumatic or electro-mechanical mechanism that operates the piano action via pre-programmed music perforated paper, or in rare instances, metallic rolls. The rise of the player piano grew with the rise of the mass-produced piano for the home in the late 19th and early 20th century. Sales peaked in 1924, as the improvement in phonograph recordings due to electrical recording methods developed in the...", "image": "http://img.freebase.com/api/trans/raw/m/02bzl31", "name": "Player piano", "thumbnail": "http://indextank.com/_static/common/demo/02bzl31.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/player_piano", "categories": {"family": "Piano"}} +{"fields": {"url": "http://freebase.com/view/en/thon_rammana", "text": "The thon-rammana (Thai: \u0e42\u0e17\u0e19\u0e23\u0e33\u0e21\u0e30\u0e19\u0e32, pronounced\u00a0[t\u02b0o\u02d0n ram.ma.na\u02d0]) are hand drums played as a pair in Central Thai classical music and Cambodian classical music. It consists of two drums: the thon (\u0e42\u0e17\u0e19), a goblet drum with a ceramic or wooden body) and the rammana (\u0e23\u0e33\u0e21\u0e30\u0e19\u0e32), a small frame drum. They are used usually in the khruang sai ensemble. The thon gives a low pitch and the rammana gives a high pitch.\nEarlier in the 20th century, the thon and rammana were sometimes played separately.", "image": "http://img.freebase.com/api/trans/raw/m/05mhk3y", "name": "Thon-rammana", "thumbnail": "http://indextank.com/_static/common/demo/05mhk3y.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/thon_rammana", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/dombra", "text": "The dombura (in Turkish domb\u0131ra, in Uzbekistan and Tajikistan, also rendered dambura or danbura in northern Afghanistan, Tajikistan, and Uzbekistan, dumbura in Bashkir and Tatar, dombor in Mongolia, dombyra in Kazakhstan, Dombira or \u51ac\u4e0d\u62c9--Dongbula in Xinjiang, China) is a long-necked lute popular in Central Asian nations. The name dombura is the Turkic rendering of Persian tanbur. The instrument shares some of its characteristics with the Central Asian komuz and dutar.\nThe instrument differs...", "image": "http://img.freebase.com/api/trans/raw/m/044m6d5", "name": "Dombra", "thumbnail": "http://indextank.com/_static/common/demo/044m6d5.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/dombra", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/torban", "text": "The torban (also teorban or Ukrainian theorbo) is a Ukrainian musical instrument that combines the features of the Baroque Lute with those of the psaltery. The \u0422orban differs from the more common European Bass lute known as the Theorbo in that it had additional short treble strings (known as prystrunky) strung along the treble side of the soundboard. It appeared ca. 1700, probably influenced by the central European Theorbo and the Angelique which Cossack mercenaries would have encountered in...", "image": "http://img.freebase.com/api/trans/raw/m/02bxych", "name": "Torban", "thumbnail": "http://indextank.com/_static/common/demo/02bxych.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/torban", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/sampler", "text": "A sampler is an electronic musical instrument similar in some respects to a synthesizer but, instead of generating sounds, it uses recordings (or \"samples\") of sounds that are loaded or recorded into it by the user and then played back by means of a keyboard, sequencer or other triggering device to perform or compose music. Because these samples are now usually stored in digital memory the information can be quickly accessed. A single sample may often be pitch-shifted to produce musical...", "image": "http://img.freebase.com/api/trans/raw/m/02cgxsn", "name": "Sampler", "thumbnail": "http://indextank.com/_static/common/demo/02cgxsn.jpg"}, "variables": {"0": 105}, "docid": "http://freebase.com/view/en/sampler", "categories": {"family": " Electronic instruments"}} +{"fields": {"url": "http://freebase.com/view/en/reed_organ", "text": "A reed organ, also called a parlor (or parlour) organ, pump organ, cabinet organ, cottage organ, is an organ that generates its sounds using free metal reeds. Smaller, cheaper and more portable than pipe organs, reed organs were widely used in smaller churches and in private homes in the 19th century, but their volume and tonal range are limited, and they were generally confined to one or two manuals, with pedal-boards being extremely rare.\nIn the generation of its tones, a reed organ is...", "image": "http://img.freebase.com/api/trans/raw/m/05lt0sf", "name": "Reed organ", "thumbnail": "http://indextank.com/_static/common/demo/05lt0sf.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/reed_organ", "categories": {"family": "Organ"}} +{"fields": {"url": "http://freebase.com/view/en/shaker", "text": "The word shaker describes a large number of percussive musical instruments used for creating rhythm in music.\nThey are so called because the method of creating sound involves shaking them\u2014moving them back and forth rather than striking them. Most may also be struck for a greater accent on certain beats. Shakers are often used in rock and popular styles to jive the j ride pattern along with or substituting for the ride cymbal.\nA shaker may comprise a container, partially full of small loose...", "image": "http://img.freebase.com/api/trans/raw/m/02d3lbb", "name": "Shaker", "thumbnail": "http://indextank.com/_static/common/demo/02d3lbb.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/shaker", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/palendag", "text": "The palendag, also called Pulalu (Manabo and Mansaka), Palandag (Bagobo), Pulala (Bukidnon) and Lumundeg (Banuwaen) is a type of Philippine bamboo flute, the largest one used by the Maguindanaon, a smaller type of this instrument is called the Hulakteb (Bukidnon).. A lip-valley flute, it is considered the toughest of the three bamboo flutes (the others being the tumpong and the suling) to use because of the way one must shape one's lips against its tip to make a sound. The construction of...", "image": "http://img.freebase.com/api/trans/raw/m/03t2fqg", "name": "Palendag", "thumbnail": "http://indextank.com/_static/common/demo/03t2fqg.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/palendag", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/arpa_doppia", "text": "An Arpa Doppia is a Double Harp common throughout Europe between the 16th and 19th Centuries.\nIt was the lack of a full chromatic compass that the theorist Juan Bermudo identified as the main 'defect' of the harp in the mid-16th century. His 'remedies' included tunings with 8 or 9 notes to the octave (more than the 7 'white' notes, but less than the full 12-note chromatic scale) and tunings adapted to each mode, with different accidentals in each octave. But he noted that some players had...", "image": "http://img.freebase.com/api/trans/raw/m/03r8mkd", "name": "Arpa Doppia", "thumbnail": "http://indextank.com/_static/common/demo/03r8mkd.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/arpa_doppia", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/alto_saxophone", "text": "The alto saxophone is a member of the saxophone family of woodwind instruments invented by Belgian instrument designer Adolphe Sax in 1841. It is smaller than the tenor but larger than the soprano, and is the type most used in classical compositions. The alto and tenor are the most common types of saxophone.\nThe alto saxophone is an E\u266d transposing instrument and reads the treble clef. A written C-natural sounds as the concert E\u266d a major sixth lower.\nThe range of the alto saxophone is from...", "image": "http://img.freebase.com/api/trans/raw/m/0291p9v", "name": "Alto saxophone", "thumbnail": "http://indextank.com/_static/common/demo/0291p9v.jpg"}, "variables": {"0": 83}, "docid": "http://freebase.com/view/en/alto_saxophone", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/guitar", "text": "The guitar is a plucked string instrument, usually played with fingers or a pick. The guitar consists of a body with a rigid neck to which the strings, generally six in number, are attached. Guitars are traditionally constructed of various woods and strung with animal gut or, more recently, with either nylon or steel strings. Some modern guitars are made of polycarbonate materials. Guitars are made and repaired by luthiers. There are two primary families of guitars: acoustic and...", "image": "http://img.freebase.com/api/trans/raw/m/042392q", "name": "Guitar", "thumbnail": "http://indextank.com/_static/common/demo/042392q.jpg"}, "variables": {"0": 5531}, "docid": "http://freebase.com/view/en/guitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/bass_clarinet", "text": "The bass clarinet is a musical instrument of the clarinet family. Like the more common soprano B\u266d clarinet, it is usually pitched in B\u266d (meaning it is a transposing instrument on which a written C sounds as B\u266d), but it plays notes an octave below the soprano B\u266d clarinet. Bass clarinets in other keys, notably C and A, also exist, but are very rare (in contrast to the regular A clarinet, which is quite common in classical music). Bass clarinets regularly perform in symphony orchestras, wind...", "image": "http://img.freebase.com/api/trans/raw/m/04r2w06", "name": "Bass clarinet", "thumbnail": "http://indextank.com/_static/common/demo/04r2w06.jpg"}, "variables": {"0": 28}, "docid": "http://freebase.com/view/en/bass_clarinet", "categories": {"family": "Clarinet"}} +{"fields": {"url": "http://freebase.com/view/en/electric_sitar", "text": "An electric sitar is a kind of electric guitar designed to mimic the sound of the traditional Indian instrument, the sitar. Depending on the manufacturer and model, these instruments bear varying degrees of resemblance to the traditional sitar. Most resemble the electric guitar in the style of the body and headstock, though some have a body shaped to resemble that of the sitar (such as a model made by Danelectro).\nThe instrument was developed in the late 1960s at Danelectro, when many...", "image": "http://img.freebase.com/api/trans/raw/m/02f_0sv", "name": "Electric sitar", "thumbnail": "http://indextank.com/_static/common/demo/02f_0sv.jpg"}, "variables": {"0": 5}, "docid": "http://freebase.com/view/en/electric_sitar", "categories": {"family": "Guitar"}} +{"fields": {"url": "http://freebase.com/view/en/virginals", "text": "The virginals or virginal (the plural form does not necessarily denote more than one instrument) is a keyboard instrument of the harpsichord family. It was popular in northern Europe and Italy during the late Renaissance and early baroque periods.\nThe virginals is a smaller and simpler rectangular form of the harpsichord with only one string per note running more or less parallel to the keyboard on the long side of the case. Many, if not most, of the instruments were constructed without...", "image": "http://img.freebase.com/api/trans/raw/m/04s7dzx", "name": "Virginals", "thumbnail": "http://indextank.com/_static/common/demo/04s7dzx.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/virginals", "categories": {"family": "Keyboard instrument"}} +{"fields": {"url": "http://freebase.com/view/en/tanbur", "text": "The term tanb\u016br (Persian: \u062a\u0646\u0628\u0648\u0631) can refer to various long-necked, fretted lutes originating in the Middle East or Central Asia. According to the New Grove Dictionary of Music and Musicians, \"terminology presents a complicated situation. Nowadays the term tanbur (or tambur) is applied to a variety of distinct and related long-necked lutes used in art and folk traditions. Similar or identical instruments are also known by other terms.\"\nHowever one study has identified the name \"tanb\u016br\" as...", "image": "http://img.freebase.com/api/trans/raw/m/02cm15x", "name": "Tanbur", "thumbnail": "http://indextank.com/_static/common/demo/02cm15x.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/tanbur", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/bandora", "text": "The Bandora or Bandore is a large long-necked plucked string-instrument that can be regarded as a bass cittern though it does not have the \"re-entrant\" tuning typical of the cittern. Probably first built by John Rose in England around 1560, it remained popular for over a century. A somewhat smaller version was the orpharion.\nFrequently one of the two bass instruments in a broken consort as associated with the works of Thomas Morley it is also a solo instrument in its own right. Anthony...", "image": "http://img.freebase.com/api/trans/raw/m/05ppmlp", "name": "Bandora", "thumbnail": "http://indextank.com/_static/common/demo/05ppmlp.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/bandora", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/english_bagpipes", "text": "The English bagpipes are bagpipes played in England. Of these, the only continuous tradition is that of the Northumbrian smallpipes, which are used in the northeastern county of Northumberland.\nAlthough bagpipes had formerly been used in other parts of England dating back at least to the Middle Ages, all but the Northumbrian smallpipes died out. Their reconstruction is a contested issue, as several distinct types of \"extinct\" bagpipes have been claimed and \"reconstructed\" based upon...", "image": "http://img.freebase.com/api/trans/raw/m/02cyw01", "name": "English bagpipes", "thumbnail": "http://indextank.com/_static/common/demo/02cyw01.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/english_bagpipes", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/orpharion", "text": "The orpharion (pronounced /\u02cc\u0254\u02d0f\u0259\u02c8ra\u026a\u0259n/ or /\u0254\u02d0\u02c8f\u00e6r\u026a\u0259n/) or opherion (pronounced /\u0252\u02c8fi\u02d0r\u026a\u0259n/) is a plucked instrument from the Renaissance. It is part of the cittern family. Its construction is similar to the larger bandora. The metal strings are tuned like a lute and are plucked with the fingers. Therefore, the orpharion can be used instead of a lute. The nut and bridge of an orpharion are typically sloped, so that the string length increases from treble to bass. Due to the extremely...", "image": "http://img.freebase.com/api/trans/raw/m/02gk9lz", "name": "Orpharion", "thumbnail": "http://indextank.com/_static/common/demo/02gk9lz.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/orpharion", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/ashiko", "text": "An ashiko is a kind of drum shaped like a truncated cone and meant to be played with bare hands. The drum is played throughout sub-Saharan Africa and the Americas. The Ashiko has three primary tones, just like the Djembe. Ashiko is the male version of djembe.\nThe A\u1e63\u00edk\u00f2 (Ashiko) drum, played by a skilled musician, can make a variety of tones, similar to the dundun or talking drums of Nigeria. It is associated with a whole genre of music of the same name and adherents of the Christian...", "image": "http://img.freebase.com/api/trans/raw/m/02chnjj", "name": "Ashiko", "thumbnail": "http://indextank.com/_static/common/demo/02chnjj.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/ashiko", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/lap_steel_guitar", "text": "The lap steel guitar is a type of steel guitar, an instrument derived from and similar to the guitar. The player changes pitch by pressing a metal or glass bar against the strings instead of by pressing strings against the fingerboard.\nThere are three main types of lap steel guitar:\nLap slide and resonator guitars may also be fitted with pickups, but do not depend on electrical amplification to produce sound.\nA lap steel guitar's strings are raised at both the nut and bridge ends of the...", "image": "http://img.freebase.com/api/trans/raw/m/029z7m_", "name": "Lap steel guitar", "thumbnail": "http://indextank.com/_static/common/demo/029z7m_.jpg"}, "variables": {"0": 38}, "docid": "http://freebase.com/view/en/lap_steel_guitar", "categories": {"family": "Guitar"}} +{"fields": {"url": "http://freebase.com/view/en/pocket_trumpet", "text": "The pocket trumpet is a compact size B\u266d trumpet, with the same playing range as the regular trumpet. The tubing is wound more tightly than that of a standard trumpet in order to reduce its size while retaining the instrument's range. It is not a standardized instrument to be found in orchestral brass sections and is generally regarded as a novelty. It is used mostly by trumpet players as a practice instrument that can be packed in a suitcase and taken to places where carrying standard...", "image": "http://img.freebase.com/api/trans/raw/m/02c4xqc", "name": "Pocket trumpet", "thumbnail": "http://indextank.com/_static/common/demo/02c4xqc.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/pocket_trumpet", "categories": {"family": "Trumpet"}} +{"fields": {"url": "http://freebase.com/view/en/bass_violin", "text": "Bass violin is the generic modern term used to denote various 16th- and 17th-century forms of bass instruments of the violin (i.e. \"viola da braccio\") family. They were the direct ancestor of the modern cello. Bass violins were usually somewhat larger than the modern cello, but tuned the same or sometimes just one step lower than it. Contemporary names for these instruments include \"basso de viola da braccio,\" \"basso da braccio,\" or the generic term \"violone,\" which simply meant \"large...", "image": "http://img.freebase.com/api/trans/raw/m/03rv9xn", "name": "Bass violin", "thumbnail": "http://indextank.com/_static/common/demo/03rv9xn.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/bass_violin", "categories": {"family": "Violin"}} +{"fields": {"url": "http://freebase.com/view/en/buccina", "text": "A buccina (Latin: buccina) or bucina (Latin: b\u016bcina), anglicized buccin or bucine, is a brass instrument used in the ancient Roman army similar to the Cornu. An aeneator who blew a buccina was called a \"buccinator\" or \"bucinator\" (Latin: buccin\u0101tor, b\u016bcin\u0101tor).\nIt was originally designed as a tube measuring some 11 to 12 feet in length, of narrow cylindrical bore, and played by means of a cup-shaped mouthpiece. The tube is bent round upon itself from the mouthpiece to the bell in the shape...", "image": "http://img.freebase.com/api/trans/raw/m/02bbt6t", "name": "Buccina", "thumbnail": "http://indextank.com/_static/common/demo/02bbt6t.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/buccina", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/tenor_saxophone", "text": "The tenor saxophone is a medium-sized member of the saxophone family, a group of instruments invented by Adolphe Sax in the 1840s. The tenor, with the alto, are the two most common types of saxophones. The tenor is pitched in the key of B\u266d, and written as a transposing instrument in the treble clef, sounding an octave and a major second lower than the written pitch. Modern tenor saxophones which have a high F# key have a range from A\u266d2 to E5 (concert) and are therefore pitched one octave...", "image": "http://img.freebase.com/api/trans/raw/m/044l733", "name": "Tenor saxophone", "thumbnail": "http://indextank.com/_static/common/demo/044l733.jpg"}, "variables": {"0": 115}, "docid": "http://freebase.com/view/en/tenor_saxophone", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/seven-string_guitar", "text": "A seven-string guitar is a guitar with seven strings instead of the usual six. Some types of seven-string guitars are specific to certain cultures (i.e. the Russian and Brazilian guitars).\nSeven-string electric guitars are particularly used in certain styles of music, and have been popular over the past few decades in the heavy metal genre. Mainstream artists such as Steve Vai, Muse, Dream Theater, Trivium, Staind, Rush, and Metallica have all experimented with seven-string guitars over the...", "image": "http://img.freebase.com/api/trans/raw/m/02b06zv", "name": "Seven-string guitar", "thumbnail": "http://indextank.com/_static/common/demo/02b06zv.jpg"}, "variables": {"0": 10}, "docid": "http://freebase.com/view/en/seven-string_guitar", "categories": {"family": "Guitar"}} +{"fields": {"url": "http://freebase.com/view/en/trumpet", "text": "The trumpet is the musical instrument with the highest register in the brass family. Trumpets are among the oldest musical instruments, dating back to at least 1500 BCE. They are constructed of brass tubing bent twice into an oblong shape, and are played by blowing air through closed lips, producing a \"buzzing\" sound which starts a standing wave vibration in the air column inside the trumpet.\nThere are several types of trumpet; the most common is a transposing instrument pitched in B\u266d with a...", "image": "http://img.freebase.com/api/trans/raw/m/0291t0f", "name": "Trumpet", "thumbnail": "http://indextank.com/_static/common/demo/0291t0f.jpg"}, "variables": {"0": 345}, "docid": "http://freebase.com/view/en/trumpet", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/double_bell_euphonium", "text": "The double bell euphonium is an instrument based on the euphonium that has a second bell that emulates a sound such as a baritone horn or trombone that is mainly used for special effects, such as echoes.\nThe last valve on the horn (either the fourth or the fifth, depending upon the model) is used to switch the sound from the main bell to the secondary bell. Both bells cannot play at the same time because each bell usually has its own tuning slide loop, such that they can be matched...", "image": "http://img.freebase.com/api/trans/raw/m/02dv55k", "name": "Double bell euphonium", "thumbnail": "http://indextank.com/_static/common/demo/02dv55k.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/double_bell_euphonium", "categories": {"family": "Euphonium"}} +{"fields": {"url": "http://freebase.com/view/en/bassoon", "text": "The bassoon is a woodwind instrument in the double reed family that typically plays music written in the bass and tenor registers, and occasionally higher. Appearing in its modern form in the 19th century, the bassoon figures prominently in orchestral, concert band, and chamber music literature. The bassoon is a non-transposing instrument known for its distinctive tone color, wide range, variety of character, and agility. Listeners often compare its warm, dark, reedy timbre to that of a male...", "image": "http://img.freebase.com/api/trans/raw/m/042218j", "name": "Bassoon", "thumbnail": "http://indextank.com/_static/common/demo/042218j.jpg"}, "variables": {"0": 76}, "docid": "http://freebase.com/view/en/bassoon", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/xiao", "text": "The xiao (simplified Chinese: \u7bab; traditional Chinese: \u7c2b; pinyin: xi\u0101o; Wade\u2013Giles: hsiao) is a Chinese vertical end-blown flute. It is generally made of dark brown bamboo (called \"purple bamboo\" in Chinese). It is also sometimes (particularly in Taiwan) called d\u00f2ngxi\u0101o (simplified Chinese: \u6d1e\u7bab; traditional Chinese: \u6d1e\u7c2b), d\u00f2ng meaning \"hole.\" An ancient name for the xi\u0101o is sh\u00f9d\u00ed (\u8c4e\u7b1b, lit. \"vertical bamboo flute\") but the name xi\u0101o in ancient times also included the side-blown bamboo flute,...", "image": "http://img.freebase.com/api/trans/raw/m/02wlxpg", "name": "Xiao", "thumbnail": "http://indextank.com/_static/common/demo/02wlxpg.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/xiao", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/viola", "text": "The viola ( /vi\u02c8o\u028al\u0259/ or /va\u026a\u02c8o\u028al\u0259/) is a bowed string instrument. It is the middle voice of the violin family, between the violin and the cello.\nThe viola is similar in material and construction to the violin but is larger in size and more variable in its proportions. A \"full-size\" viola's body is between one and four inches longer than the body of a full-size violin (i.e., between 15 and 18 inches (38 and 46 cm)), with an average length of about 16\u00a0inches (41\u00a0cm). Small violas made for...", "image": "http://img.freebase.com/api/trans/raw/m/02bk5b0", "name": "Viola", "thumbnail": "http://indextank.com/_static/common/demo/02bk5b0.jpg"}, "variables": {"0": 182}, "docid": "http://freebase.com/view/en/viola", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/pandura", "text": "The pandura is an ancient string instrument from the Mediterranian basin.\nThe ancient Greek pandoura (or pandora) (Greek: \u03c0\u03b1\u03bd\u03b4\u03bf\u03cd\u03c1\u03b1) was a medium or long-necked lute with a small resonating chamber. It commonly had three strings: such an instrument was also known as the trichordon (McKinnon 1984:10). Its descendants still survive as Greek tambouras and bouzouki, North African Kuitras and Balkan tamburitsas. Renato Meucci (1996) suggests that the some Italian Renaissance descendants of Pandura...", "image": "http://img.freebase.com/api/trans/raw/m/04rxd7r", "name": "Pandura", "thumbnail": "http://indextank.com/_static/common/demo/04rxd7r.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/pandura", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/saxhorn", "text": "The saxhorn is a valved brass instrument with a conical bore and deep cup-shaped mouthpiece. The sound has a characteristic mellow quality, and blends well with other brass.\nThe saxhorns form a family of seven instruments (although at one point ten different sizes seem to have existed). Designed for band use, they are pitched alternately in E-flat and B-flat, like the saxophone group.\nThere is much confusion as to nomenclature of the various instruments in different languages. This has been...", "image": "http://img.freebase.com/api/trans/raw/m/0291pc6", "name": "Saxhorn", "thumbnail": "http://indextank.com/_static/common/demo/0291pc6.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/saxhorn", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/native_american_flute", "text": "The Native American flute has achieved some measure of fame for its distinctive sound, used in a variety of New Age and world music recordings. The instrument was originally very personal; its music was played without accompaniment in courtship, healing, meditation, and spiritual rituals. Now it is played solo, with other instruments or vocals, or with backing tracks. The flute has been used in Native American music, as well as other genres. There are two different types of Native American...", "image": "http://img.freebase.com/api/trans/raw/m/02fh3s0", "name": "Native American flute", "thumbnail": "http://indextank.com/_static/common/demo/02fh3s0.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/native_american_flute", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/gaohu", "text": "The gaohu (\u9ad8\u80e1; pinyin: g\u0101oh\u00fa; Cantonese: gou1 wu4; also called yuehu \u7ca4\u80e1) is a Chinese bowed string instrument developed from the erhu in the 1920s by the musician and composer L\u00fc Wencheng (1898\u20131981) and used in Cantonese music and Cantonese opera. It belongs to the huqin family of instruments, together with the zhonghu, erhu, banhu, jinghu, and sihu, its name means \"high pitched huqin\". It has two strings and its soundbox is covered on the front (playing) end with snakeskin (from a...", "image": "http://img.freebase.com/api/trans/raw/m/02fk9hb", "name": "Gaohu", "thumbnail": "http://indextank.com/_static/common/demo/02fk9hb.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gaohu", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/keyboard_bass", "text": "The keyboard bass is the use of a low-pitched keyboard or pedal keyboard to substitute for the bass guitar or double bass in popular music.\nThe earliest keyboard bass instrument was the 1960 Fender Rhodes piano bass, pictured above. The piano bass was an electric piano with the same pitch range as the electric bass (or the double bass), which could be used to perform basslines. It could be placed on top of a piano or organ, or mounted on a stand. As well, keyboard players such as The Doors'...", "image": "http://img.freebase.com/api/trans/raw/m/02f8j3d", "name": "Keyboard bass", "thumbnail": "http://indextank.com/_static/common/demo/02f8j3d.jpg"}, "variables": {"0": 4}, "docid": "http://freebase.com/view/en/keyboard_bass", "categories": {"family": "Keyboard instrument"}} +{"fields": {"url": "http://freebase.com/view/en/duduk", "text": "The duduk is a traditional woodwind instrument indigenous to Armenia. Variations of it are popular in the Caucasus, the Middle East and Central Asia. The English word is often used generically for a family of ethnic instruments including the doudouk or duduk (\u0564\u0578\u0582\u0564\u0578\u0582\u056f), also tsiranapogh \u056e\u056b\u0580\u0561\u0576\u0561\u0583\u0578\u0572, literally \"apricot horn\" in Armenian), the d\u00fcd\u00fck or mey in Turkey, the duduki in Georgia, the balaban (or d\u00fcd\u00fck) in Azerbaijan, the narmeh-ney in Iran, the duduka or dudka in Russia and Ukraine. The...", "image": "http://img.freebase.com/api/trans/raw/m/02dgq34", "name": "Duduk", "thumbnail": "http://indextank.com/_static/common/demo/02dgq34.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/duduk", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/wanamaker_organ", "text": "The Wanamaker Grand Court Organ, in Philadelphia, Pennsylvania, is the largest operational pipe organ in the world, located within a spacious 7-story court at Macy's Center City (formerly Wanamaker's department store). The largest organ is the Boardwalk Hall Auditorium Organ (which is barely functional). The Wanamaker organ is played twice a day, Monday through Saturday, and more frequently during the Christmas season. The organ is also featured at several special concerts held throughout...", "image": "http://img.freebase.com/api/trans/raw/m/02b828b", "name": "Wanamaker Organ", "thumbnail": "http://indextank.com/_static/common/demo/02b828b.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/wanamaker_organ", "categories": {"family": "Organ"}} +{"fields": {"url": "http://freebase.com/view/en/grafton_saxophone", "text": "The Grafton saxophone was an injection moulded, cream-coloured acrylic plastic alto saxophone with metal keys, manufactured in London, England by the Grafton company, and later by 'John E. Dallas & Sons Ltd'. Only Grafton altos were ever made, due to the challenges in making larger models (i.e. the tenor) with 1950s plastic technology. Production commenced in 1950 and ended after approximately ten years. However, a few last examples were assembled from residual parts circa 1967. All tools,...", "image": "http://img.freebase.com/api/trans/raw/m/07ktgsj", "name": "Grafton saxophone", "thumbnail": "http://indextank.com/_static/common/demo/07ktgsj.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/grafton_saxophone", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/irish_bouzouki", "text": "The Irish bouzouki is a unique development of the Greek bouzouki adapted for Irish traditional and folk music from the late 1960s onward.\nThe Greek bouzouki, in the newer tetrachordo (four course/eight string, or \u03c4\u03b5\u03c4\u03c1\u03ac\u03c7\u03bf\u03c1\u03b4\u03bf) version developed in the twentieth century, was introduced into Irish Traditional Music in the late 1960s by Johnny Moynihan of the popular folk group Sweeney\u2019s Men, and popularized by Andy Irvine and D\u00f3nal Lunny in the group Planxty. In a separate but parallel...", "image": "http://img.freebase.com/api/trans/raw/m/02csz05", "name": "Irish bouzouki", "thumbnail": "http://indextank.com/_static/common/demo/02csz05.jpg"}, "variables": {"0": 4}, "docid": "http://freebase.com/view/en/irish_bouzouki", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/vibraphone", "text": "The vibraphone, sometimes called the vibraharp or simply the vibes, is a musical instrument in the mallet subfamily of the percussion family.\nIt is similar in appearance to the xylophone, marimba, and glockenspiel although the vibraphone uses aluminum bars instead of the wooden bars of the first two instruments. Each bar is paired with a resonator tube having a motor-driven butterfly valve at its upper end, mounted on a common shaft, which produces a tremolo or vibrato effect while spinning....", "image": "http://img.freebase.com/api/trans/raw/m/0292dq5", "name": "Vibraphone", "thumbnail": "http://indextank.com/_static/common/demo/0292dq5.jpg"}, "variables": {"0": 45}, "docid": "http://freebase.com/view/en/vibraphone", "categories": {"family": "Classical percussion"}} +{"fields": {"url": "http://freebase.com/view/en/tonbak", "text": "The tonbak (also tombak, donbak, dombak; in Persian: \u062a\u0645\u0628\u06a9, or \u062a\u064f\u0645\u0628\u064e\u06a9 ,\u062a\u0646\u0628\u06a9 ,\u062f\u0645\u0628\u06a9 ,\u062f\u0646\u0628\u06a9) or zarb (\u0636\u064e\u0631\u0628 or \u0636\u0631\u0628) is a goblet drum from Persia (ancient Iran). It is considered the principal percussion instrument of Persian music. The tonbak is normally positioned diagonally across the torso while the player uses one or more fingers and/or the palm(s) of the hand(s) on the drumhead, often (for a ringing timbre) near the drumhead's edge. Sometimes tonbak players wear metal finger rings for an...", "image": "http://img.freebase.com/api/trans/raw/m/03s6mz8", "name": "Tonbak", "thumbnail": "http://indextank.com/_static/common/demo/03s6mz8.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/tonbak", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/saraswati_veena", "text": "The Saraswati veena (also spelled Saraswati vina) is an Indian plucked string instrument. It is named after the Hindu goddess Saraswati, who is usually depicted holding or playing the instrument.\nIt is one of the three other major types of veena popular today. The others include vichitra veena and rudra veena. Out of these the rudra and vichitra veenas are used in Hindustani music, while the Saraswati veena is used in the Carnatic music of South India. Some people play traditional music,...", "image": "http://img.freebase.com/api/trans/raw/m/02dz35y", "name": "Saraswati veena", "thumbnail": "http://indextank.com/_static/common/demo/02dz35y.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/saraswati_veena", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/toy_piano", "text": "The toy piano, also known as the kinderklavier (child's keyboard), is a small piano-like musical instrument. The present form of the toy piano was invented in Philadelphia by a 17-year-old German immigrant named Albert Schoenhut. He worked as a repairman at Wanamaker's department store, repairing broken glass sounding pieces in German toy pianos damaged in shipping. Schoenhut conceived of the toy piano as it is known today in 1872, when he substituted durable steel plates for the traditional...", "image": "http://img.freebase.com/api/trans/raw/m/02c15hd", "name": "Toy piano", "thumbnail": "http://indextank.com/_static/common/demo/02c15hd.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/toy_piano", "categories": {"family": "Piano"}} +{"fields": {"url": "http://freebase.com/view/en/cornett", "text": "The cornett, cornetto or zink is an early wind instrument, dating from the Medieval, Renaissance and Baroque periods. It was used in what are now called alta capellas or wind ensembles. It is not to be confused with the trumpet-like instrument cornet.\nThere are three basic types of treble cornett: curved, straight and mute. The curved (Ger. krummer Zink, schwarzer Zink; It. cornetto curvo, cornetto alto (i.e. \u2018loud\u2019), cornetto nero) is the most common type, with over 140 extant examples. It...", "image": "http://img.freebase.com/api/trans/raw/m/0292b3v", "name": "Cornett", "thumbnail": "http://indextank.com/_static/common/demo/0292b3v.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/cornett", "categories": {"family": "Wind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/gehu", "text": "The gehu (\u9769\u80e1; pinyin: g\u00e9h\u00fa) is a Chinese instrument developed in the 20th century by the Chinese musician Yang Yusen (\u6768\u96e8\u68ee, 1926-1980). It is a fusion of the Chinese huqin family and the cello. Its four strings are also tuned (from low to high) C-G-D-A, exactly like the cello's. Unlike most other musical instruments in the huqin family, the bridge does not contact the snakeskin, which faces to the side\nThere is also a contrabass gehu that functions as a Chinese double bass, known as the...", "image": "http://img.freebase.com/api/trans/raw/m/02gdwq6", "name": "Gehu", "thumbnail": "http://indextank.com/_static/common/demo/02gdwq6.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gehu", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/theatre_organ", "text": "A theatre organ (also known as a cinema organ) is a pipe organ originally designed specifically for imitation of an orchestra. New designs have tended to be around some of the sounds and blends unique to the instrument itself.\nTheatre organs took the place of the orchestra when installed in a movie theatre during the heyday of silent films. Most theatre organs were modelled after the style originally devised by Robert Hope-Jones, which he called a \"unit orchestra\".\nSuch instruments were...", "image": "http://img.freebase.com/api/trans/raw/m/02dwqz6", "name": "Theatre organ", "thumbnail": "http://indextank.com/_static/common/demo/02dwqz6.jpg"}, "variables": {"0": 4}, "docid": "http://freebase.com/view/en/theatre_organ", "categories": {"family": "Pipe organ"}} +{"fields": {"url": "http://freebase.com/view/en/zendrum", "text": "A Zendrum is a hand-crafted MIDI controller that is used as a percussion instrument. There are two Zendrum models that are well-suited for live performances, the ZX and the LT. The Zendrum ZX is worn like a guitar and consists of a triangular hardwood body with 24 touch-sensitive plastic pads which act as MIDI triggers. The Zendrum LT can also be worn with a guitar strap, but it has 25 MIDI triggers in a symmetrical layout, which provides an ambidextrous playing surface. The pads are played...", "image": "http://img.freebase.com/api/trans/raw/m/0dg07vb", "name": "Zendrum", "thumbnail": "http://indextank.com/_static/common/demo/0dg07vb.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/zendrum", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/sanxian", "text": "The sanxian (Chinese: \u4e09\u5f26 (\u7d43?), literally \"three strings\") is a Chinese lute \u2014 a three-stringed fretless plucked musical instrument. It has a long fingerboard, and the body is traditionally made from snakeskin stretched over a rounded rectangular resonator. It is made in several sizes for different purposes and in the late 20th century a four-stringed version was also developed. The northern sanxian is generally larger, at about 122 cm in length, while southern versions of the instrument are...", "image": "http://img.freebase.com/api/trans/raw/m/03s5n58", "name": "Sanxian", "thumbnail": "http://indextank.com/_static/common/demo/03s5n58.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/sanxian", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/bamboo_flute", "text": "Flutes made of bamboo are found in many musical traditions.\nSome bamboo flutes include:", "image": "http://img.freebase.com/api/trans/raw/m/02c3rxm", "name": "Bamboo flute", "thumbnail": "http://indextank.com/_static/common/demo/02c3rxm.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/bamboo_flute", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/jouhikko", "text": "The jouhikko is a traditional, 2 or 3 stringed bowed lyre, from Finland and Karelia. Its strings are traditionally of horsehair. The playing of this instrument died out in the early 20th century but has been revived and there are now a number of musicians playing it. \nThe Jouhikko is also called jouhikannel or jouhikantele, meaning a bowed kantele. In English, the usual modern designation is 'bowed Lyre' though the earlier preferred term 'bowed Harp' is also met with. There are many...", "image": "http://img.freebase.com/api/trans/raw/m/078v698", "name": "Jouhikko", "thumbnail": "http://indextank.com/_static/common/demo/078v698.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/jouhikko", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/m/04w695", "text": "The viola caipira (Portuguese for hillbilly guitar) is a ten-string, five-course guitar. Unlike most steel-string guitars, its strings are plucked with the fingers of the right hand similarly to the technique used for classical and flamenco guitars, rather than by the use of a plectrum.\nIt is a folk instrument commonly found in Brazil, where it is often simply called viola\nThe origins of the viola caipira are obscure, but folklorist Lu\u00eds da C\u00e2mara Cascudo believes it to be an archaic form of...", "image": "http://img.freebase.com/api/trans/raw/m/05lxtkk", "name": "Viola caipira", "thumbnail": "http://indextank.com/_static/common/demo/05lxtkk.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/m/04w695", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/trombone", "text": "The trombone (Ger. Posaune, Sp. tromb\u00f3n) is a musical instrument in the brass family. Like all brass instruments, sound is produced when the player\u2019s vibrating lips (embouchure) cause the air column inside the instrument to vibrate. The trombone is usually characterised by a telescopic slide with which the player varies the length of the tube to change pitches, although the valve trombone uses three valves like those on a trumpet.\nThe word trombone derives from Italian tromba (trumpet) and...", "image": "http://img.freebase.com/api/trans/raw/m/02bjmnh", "name": "Trombone", "thumbnail": "http://indextank.com/_static/common/demo/02bjmnh.jpg"}, "variables": {"0": 309}, "docid": "http://freebase.com/view/en/trombone", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/violin", "text": "The violin is a string instrument, usually with four strings tuned in perfect fifths. It is the smallest, highest-pitched member of the violin family of string instruments, which includes the viola and cello.\nThe violin is sometimes informally called a fiddle, regardless of the type of music played on it. The word violin comes from the Middle Latin word vitula, meaning stringed instrument; this word is also believed to be the source of the Germanic \"fiddle\". The violin, while it has ancient...", "image": "http://img.freebase.com/api/trans/raw/m/02bk3yy", "name": "Violin", "thumbnail": "http://indextank.com/_static/common/demo/02bk3yy.jpg"}, "variables": {"0": 663}, "docid": "http://freebase.com/view/en/violin", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/snare_drum", "text": "The snare drum is a drum with strands of snares made of curled metal wire, metal cable, plastic cable, or gut cords stretched across the drumhead, typically the bottom. Pipe and tabor and some military snare drums often have a second set of snares on the bottom (internal) side of the top (batter) head to make a \"brighter\" sound. Different types can be found, like Piccolo snares, that have a smaller depth for a higher pitch, rope-tuned snares (Maracatoo snare) and the Brazilian \"Tarol\", that...", "image": "http://img.freebase.com/api/trans/raw/m/02bdbkf", "name": "Snare drum", "thumbnail": "http://indextank.com/_static/common/demo/02bdbkf.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/snare_drum", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/bass_saxophone", "text": "The bass saxophone is the second largest existing member of the saxophone family (not counting the subcontrabass tubax). It is similar in design to a baritone saxophone, but it is larger, with a longer loop near the mouthpiece. Unlike the baritone, the bass saxophone is not commonly used. While some composers did write parts for the instrument through the early twentieth century (such as Percy Grainger in Lincolnshire Posy), the bass sax part in today's wind bands is usually handled by the...", "image": "http://img.freebase.com/api/trans/raw/m/02f_0qz", "name": "Bass saxophone", "thumbnail": "http://indextank.com/_static/common/demo/02f_0qz.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/bass_saxophone", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/splash_cymbal", "text": "A splash cymbal is a small cymbal used for an accent in a drum kit. Splash cymbals and china cymbals are the main types of effects cymbals. The cymbal is also known as a multi-crash cymbal or crescent cymbal.\nMost splash cymbals range in size from 6\" to 12\" in diameter, though some splash cymbals go as low as 4\". Some makers have produced cymbals described as splash cymbals up to 22\" in diameter but these would be better described as medium thin crash cymbals. The most common size is 10\",...", "image": "http://img.freebase.com/api/trans/raw/m/0290zn9", "name": "Splash cymbal", "thumbnail": "http://indextank.com/_static/common/demo/0290zn9.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/splash_cymbal", "categories": {"family": "Cymbal"}} +{"fields": {"url": "http://freebase.com/view/en/timpani", "text": "Timpani, or kettledrums, are musical instruments in the percussion family. A type of drum, they consist of a skin called a head stretched over a large bowl traditionally made of copper. They are played by striking the head with a specialized drum stick called a timpani stick or timpani mallet. Unlike most drums, they are capable of producing an actual pitch when struck, and can be tuned, often with the use of a pedal mechanism to control each drum's range of notes. Timpani evolved from...", "image": "http://img.freebase.com/api/trans/raw/m/02byz95", "name": "Timpani", "thumbnail": "http://indextank.com/_static/common/demo/02byz95.jpg"}, "variables": {"0": 6}, "docid": "http://freebase.com/view/en/timpani", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/lyre", "text": "The lyre is a stringed musical instrument well known for its use in Greek classical antiquity and later. The word comes from the Greek \"\u03bb\u03cd\u03c1\u03b1\" (lyra) and the earliest reference to the word is the Mycenaean Greek ru-ra-ta-e, meaning \"lyrists\", written in Linear B syllabic script. The earliest picture of a lyre with seven strings appears in the famous sarcophagus of Hagia Triada (a Minoan settlement in Crete). The sarcophagus was used during the Mycenaean occupation of Crete (1400 BC). The...", "image": "http://img.freebase.com/api/trans/raw/m/02dvvxt", "name": "Lyre", "thumbnail": "http://indextank.com/_static/common/demo/02dvvxt.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/lyre", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/theremin", "text": "The theremin (/\u02c8\u03b8\u025br\u0259m\u026an/), originally known as the aetherphone/etherophone, thereminophone or termenvox/thereminvox is an early electronic musical instrument controlled without discernible physical contact from the player. It is named after its Russian inventor, Professor L\u00e9on Theremin, who patented the device in 1928. The controlling section usually consists of two metal antennas which sense the position of the player's hands and control oscillators for frequency with one hand, and...", "image": "http://img.freebase.com/api/trans/raw/m/02bgfft", "name": "Theremin", "thumbnail": "http://indextank.com/_static/common/demo/02bgfft.jpg"}, "variables": {"0": 24}, "docid": "http://freebase.com/view/en/theremin", "categories": {"family": "Electronic keyboard"}} +{"fields": {"url": "http://freebase.com/view/en/harpsichord", "text": "A harpsichord is a musical instrument played by means of a keyboard. It produces sound by plucking a string when a key is pressed.\nIn the narrow sense, \"harpsichord\" designates only the large wing-shaped instruments in which the strings are perpendicular to the keyboard. In a broader sense, \"harpsichord\" designates the whole family of similar plucked keyboard instruments, including the smaller virginals, muselar, and spinet.\nThe harpsichord was widely used in Renaissance and Baroque music....", "image": "http://img.freebase.com/api/trans/raw/m/05khvdd", "name": "Harpsichord", "thumbnail": "http://indextank.com/_static/common/demo/05khvdd.jpg"}, "variables": {"0": 106}, "docid": "http://freebase.com/view/en/harpsichord", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/pedal_harp", "text": "The pedal harp (also known as the concert harp) is a large and technically modern harp, designed primarily for classical music and played either solo, as part of chamber ensembles, as soloist with or as a section or member in an orchestra. The pedal harp is a descendant of ancient harps.\nA pedal harp typically has six and a half octaves (46 or 47 strings), weighs about 80\u00a0lb (36\u00a0kg), is approximately 6\u00a0ft (1.8 m) high, has a depth of 4\u00a0ft (1.2 m), and is 21.5 in (55\u00a0cm) wide at the bass end...", "image": "http://img.freebase.com/api/trans/raw/m/03t9bsc", "name": "Pedal harp", "thumbnail": "http://indextank.com/_static/common/demo/03t9bsc.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/pedal_harp", "categories": {"family": "Harp"}} +{"fields": {"url": "http://freebase.com/view/en/piccolo_trumpet", "text": "The smallest of the trumpet family is the piccolo trumpet, pitched one octave higher than the standard B\u266d trumpet. Most piccolo trumpets are built to play in either B\u266d or A, using a separate leadpipe for each key. The tubing in the B\u266d piccolo trumpet is one-half the length of that in a standard B\u266d trumpet. Piccolo trumpets in G, F, and even high C are also manufactured, but are rarer.\nThe soprano trumpet in D is also known as the Bach trumpet and was invented in about 1890 by the Belgian...", "image": "http://img.freebase.com/api/trans/raw/m/03trfhw", "name": "Piccolo trumpet", "thumbnail": "http://indextank.com/_static/common/demo/03trfhw.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/piccolo_trumpet", "categories": {"family": "Trumpet"}} +{"fields": {"url": "http://freebase.com/view/en/octave_mandolin", "text": "The octave mandolin is a fretted string instrument with four pairs of strings tuned in 5ths, G, D, A, E (low to high), an octave below a mandolin. It has a 20 to 23 inch scale length and its construction is similar to other instruments in the mandolin family. Usually the courses are all unison pairs but the lower two may sometimes be strung as octave pairs with the higher pitched octave string on top so that it is hit before the thicker lower pitched string.\nThe names of the mandolin family...", "image": "http://img.freebase.com/api/trans/raw/m/03sywnm", "name": "Octave mandolin", "thumbnail": "http://indextank.com/_static/common/demo/03sywnm.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/octave_mandolin", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/valiha", "text": "The valiha is a tube zither from Madagascar made from a species of local bamboo. It is played by plucking the strings, which may be made of metal or (originally) the bamboo skin which is pried up in long strands and propped up by small bridges made of pieces of dried gourd. The valiha is considered the national instrument of Madagascar.\nThe strings of the modern valiha are generally made of bicycle brake cable. The cables are unstrung into individual strands and each string of the instrument...", "image": "http://img.freebase.com/api/trans/raw/m/0dhf4b7", "name": "Valiha", "thumbnail": "http://indextank.com/_static/common/demo/0dhf4b7.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/valiha", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/bajo_sexto", "text": "A bajo sexto (Spanish: \"sixth bass\") is a musical instrument with 12 strings in 6 double courses, used in Mexican music. It is used primarily in norte\u00f1o music of northern Mexico and across the border in the music of south Texas known as \"Tex-Mex,\" \"conjunto,\" or \"m\u00fasica mexicana-tejana\".\nA similar instrument with five courses is the bajo quinto. The manufacture of bajo quinto achieved high quality in the 19th century, in the states of Aguascalientes, Morelos, Puebla, Oaxaca, Tlaxcala and...", "image": "http://img.freebase.com/api/trans/raw/m/05lclxv", "name": "Bajo sexto", "thumbnail": "http://indextank.com/_static/common/demo/05lclxv.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/bajo_sexto", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/gaita_transmontana", "text": "The gaita transmontana (or gaita-de-fole transmontana, gaita mirandesa) is a type of bagpipe native to the Tr\u00e1s-os-Montes region of Portugal.", "image": "http://img.freebase.com/api/trans/raw/m/0636_zv", "name": "Gaita transmontana", "thumbnail": "http://indextank.com/_static/common/demo/0636_zv.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/gaita_transmontana", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/baroque_trumpet", "text": "The baroque trumpet is a musical instrument in the brass family. It is most associated with music from the 16th to 18th centuries. Often synonymous with 'natural trumpet', the term 'baroque trumpet' is also often used to differentiate a modern replica that has added vent holes, with an original or replica natural trumpet which does not.\nSee natural trumpet.\nNowadays, the term \"baroque trumpet\" has come to mean a reproduction or copy of an original natural trumpet. These are the instruments...", "image": "http://img.freebase.com/api/trans/raw/m/03qxzkp", "name": "Baroque trumpet", "thumbnail": "http://indextank.com/_static/common/demo/03qxzkp.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/baroque_trumpet", "categories": {"family": "Trumpet"}} +{"fields": {"url": "http://freebase.com/view/en/begena", "text": "The begena (or b\u00e8gu\u00e8na, as in French) is an Ethiopian and Eritrean string instrument that resembles a large lyre. According to Ethiopian tradition, Menelik I brought the instrument to Ethiopia from Israel, where David had used the begena to soothe King Saul's nerves and heal him of insomnia. Its actual origin remains in doubt, though Ethiopian manuscripts depict the instrument at the beginning of the 15th century (Kimberlin 1978: 13).\nKnown as the instrument of noblemen, monks, and the upper...", "image": "http://img.freebase.com/api/trans/raw/m/02d06m7", "name": "Begena", "thumbnail": "http://indextank.com/_static/common/demo/02d06m7.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/begena", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/cimbasso", "text": "The Cimbasso is a brass instrument in the trombone family, with a sound ranging from warm and mellow to bright and menacing. It has three to five piston or rotary valves, a highly cylindrical bore, and is usually pitched in F or Bb. It is in the same range as a tuba or a contrabass trombone.\nThe modern instrument can be played by a tubist or a bass trombonist.\nThe modern cimbasso is most commonly used in opera scores by Verdi from Oberto to Aida and Puccini in Le Villi only, though the word...", "image": "http://img.freebase.com/api/trans/raw/m/02bj9wv", "name": "Cimbasso", "thumbnail": "http://indextank.com/_static/common/demo/02bj9wv.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/cimbasso", "categories": {"family": "Trombone"}} +{"fields": {"url": "http://freebase.com/view/en/sarrusophone", "text": "The sarrusophone is a family of transposing musical instruments patented and placed into production by Pierre-Louis Gautrot in 1856. It was named after the French bandmaster Pierre-Auguste Sarrus (1813\u20131876) who is credited with the concept of the instrument (it is not clear if Sarrus benefited financially from this association). The instrument was intended to serve as a replacement in wind bands for the oboe and bassoon which, at that time, lacked the carrying power required for outdoor...", "image": "http://img.freebase.com/api/trans/raw/m/029947_", "name": "Sarrusophone", "thumbnail": "http://indextank.com/_static/common/demo/029947_.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/sarrusophone", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/rhodes_piano", "text": "The Rhodes piano is an electro-mechanical piano, invented by Harold Rhodes during the fifties and later manufactured in a number of models, first in collaboration with Fender and after 1965 by CBS.\nAs a member of the electrophone sub-group of percussion instruments, it employs a piano-like keyboard with hammers that hit small metal tines, amplified by electromagnetic pickups. A 2001 New York Times article described the instrument as \"a pianistic counterpart to the electric guitar\" having a...", "image": "http://img.freebase.com/api/trans/raw/m/02923_q", "name": "Rhodes piano", "thumbnail": "http://indextank.com/_static/common/demo/02923_q.jpg"}, "variables": {"0": 13}, "docid": "http://freebase.com/view/en/rhodes_piano", "categories": {"family": "Electric piano"}} +{"fields": {"url": "http://freebase.com/view/en/woodwind_instrument", "text": "A woodwind instrument is a musical instrument which produces sound when the player blows air against a sharp edge or through a reed, causing the air within its resonator (usually a column of air) to vibrate. Most of these instruments are made of wood but can be made of other materials, such as metals or plastics.\nWoodwind instruments can further be divided into 2 groups: flutes and reed instruments.\nThe modern symphony orchestra's woodwinds section typically includes: 3 flutes, 1 piccolo, 3...", "image": "http://img.freebase.com/api/trans/raw/m/02bp_1j", "name": "Woodwind instrument", "thumbnail": "http://indextank.com/_static/common/demo/02bp_1j.jpg"}, "variables": {"0": 7}, "docid": "http://freebase.com/view/en/woodwind_instrument", "categories": {"family": "Wind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/drum", "text": "The drum is a member of the percussion group of musical instruments, technically classified as the membranous. Drums consist of at least one membrane, called a drumhead or drum skin, that is stretched over a shell and struck, either directly with the player's hands, or with a drumstick, to produce sound. There is usually a \"resonance head\" on the underside of the drum. Other techniques have been used to cause drums to make sound, such as the thumb roll. Drums are the world's oldest and most...", "image": "http://img.freebase.com/api/trans/raw/m/03s3c52", "name": "Drum", "thumbnail": "http://indextank.com/_static/common/demo/03s3c52.jpg"}, "variables": {"0": 1217}, "docid": "http://freebase.com/view/en/drum", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/tro", "text": "Tro is the generic name for traditional bowed string instruments in Cambodia.\nInstruments in this family include the two-stringed tro u, tro sau toch, tro sau thom, and tro che, as well as the three-stringed tro Khmer spike fiddle.", "image": "http://img.freebase.com/api/trans/raw/m/041m7qn", "name": "Tro", "thumbnail": "http://indextank.com/_static/common/demo/041m7qn.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/tro", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/kit_violin", "text": "The kit violin, or kit (Tanzmeistergeige in German), is a stringed musical instrument. It is essentially a very small violin, designed to fit in a pocket \u2014 hence its other common name, the pochette. It was used by dance masters in royal courts and other places of nobility, as well as by street musicians up until around the 18th century. Occasionally, the rebec was used in the same way. Several are called for (as violini piccoli alla francese - small French violins) in Monteverdi's 1607...", "image": "http://img.freebase.com/api/trans/raw/m/029mhmx", "name": "Kit violin", "thumbnail": "http://indextank.com/_static/common/demo/029mhmx.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/kit_violin", "categories": {"family": "Violin"}} +{"fields": {"url": "http://freebase.com/view/en/volynka", "text": "The volynka (Ukrainian: \u0432\u043e\u043b\u0438\u043d\u043a\u0430, Russian: \u0432\u043e\u043b\u044b\u043d\u043a\u0430, Crimean Tatar: tulup zurna \u2013 see also duda, koza, and kobza) is a Slavic bagpipe. Its etymology comes from the region Volyn, Ukraine, where it was borrowed from Romania.\nThe volynka is constructed around a goat skin air reservoir into which air is blown through a pipe with a valve to stop air escaping. (Modern concert instruments often have a reservoir made from a basketball bladder}. A number of playing pipes [two to four] extend from the...", "image": "http://img.freebase.com/api/trans/raw/m/03rlj3v", "name": "Volynka", "thumbnail": "http://indextank.com/_static/common/demo/03rlj3v.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/volynka", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/berimbau", "text": "The berimbau (English pronounced /b\u0259r\u026am\u02c8ba\u028a/, Brazilian Portuguese [be\u027e\u0129\u02c8baw]) is a single-string percussion instrument, a musical bow, from Brazil. The berimbau's origins are not entirely clear, but there is not much doubt on its African origin, as no Indigenous Brazilian or European people use musical bows, and very similar instruments are played in the southern parts of Africa. The berimbau was eventually incorporated into the practice of the Afro-Brazilian martial art capoeira, where it...", "image": "http://img.freebase.com/api/trans/raw/m/029wcz0", "name": "Berimbau", "thumbnail": "http://indextank.com/_static/common/demo/029wcz0.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/berimbau", "categories": {"family": "Struck string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/domra", "text": "The domra (Russian: \u0434\u043e\u043c\u0440\u0430) is a long-necked Russian string instrument of the lute family with a round body and three or four metal strings.\nIn 1896, a student of Vassily Vassilievich Andreyev found a broken instrument in a stable in rural Russia. It was thought that this instrument may have been an example of a domra, although no illustrations or examples of the traditional domra were known to exist in Russian chronicles. A three-stringed version of this instrument was later redesigned in...", "image": "http://img.freebase.com/api/trans/raw/m/02bs20q", "name": "Domra", "thumbnail": "http://indextank.com/_static/common/demo/02bs20q.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/domra", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/contrabass_saxophone", "text": "The contrabass saxophone is the lowest-pitched extant member of the saxophone family proper. It is extremely large (twice the length of tubing of the baritone saxophone, with a bore twice as wide, standing 1.9 meters tall, or 6 feet four inches) and heavy (approximately 20 kilograms, or 45 pounds), and is pitched in the key of E\u266d, one octave below the baritone.\nThe contrabass saxophone was part of the original saxophone family as conceived by Adolphe Sax, and is included in his saxophone...", "image": "http://img.freebase.com/api/trans/raw/m/02fx9nl", "name": "Contrabass saxophone", "thumbnail": "http://indextank.com/_static/common/demo/02fx9nl.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/contrabass_saxophone", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/nyckelharpa", "text": "A nyckelharpa (literally \"key harp\", plural nyckelharpor or sometimes keyed fiddle) is a traditional Swedish musical instrument. It is a string instrument or chordophone. Its keys are attached to tangents which, when a key is depressed, serve as frets to change the pitch of the string.\nThe nyckelharpa is similar in appearance to a fiddle or the bowed Byzantine lira. Structurally, it is more closely related to the hurdy gurdy, both employing key-actuated tangents to change the pitch.\nA...", "image": "http://img.freebase.com/api/trans/raw/m/029qc_t", "name": "Nyckelharpa", "thumbnail": "http://indextank.com/_static/common/demo/029qc_t.jpg"}, "variables": {"0": 5}, "docid": "http://freebase.com/view/en/nyckelharpa", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/bandurria", "text": "The bandurria is a plectrum plucked chordophone from Spain, similar to the cittern and the mandolin, primarily used in Spanish folk music.\nPrior to the 18th century, the bandurria had with a round back, similar or related to the mandore. It had become a flat-backed instrument by the 18th century, with five double courses of strings, tuned in fourths. The original bandurrias of the Medieval period had three strings. During the Renaissance they gained a fourth string. During the Baroque period...", "image": "http://img.freebase.com/api/trans/raw/m/04pm0zr", "name": "Bandurria", "thumbnail": "http://indextank.com/_static/common/demo/04pm0zr.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/bandurria", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/steelpan", "text": "Steelpans (also known as steel drums or pans, and sometimes, collectively with musicians, as a steel band) is a musical instrument originating from Trinidad and Tobago. Steel pan musicians are called pannists.\nThe pan is a chromatically pitched percussion instrument (although some toy or novelty steelpans are tuned diatonically), made from 55 gallon drums that usually store oil. In fact, drum refers to the steel drum containers from which the pans are made; the steeldrum is correctly called...", "image": "http://img.freebase.com/api/trans/raw/m/03r1dm4", "name": "Steelpan", "thumbnail": "http://indextank.com/_static/common/demo/03r1dm4.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/steelpan", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/zither", "text": "The zither is a musical string instrument, most commonly found in Slovenia, Austria, Hungary citera, northwestern Croatia, the southern regions of Germany, alpine Europe and East Asian cultures, including China. The term \"citre\" is also used more broadly, to describe the entire family of stringed instruments in which the strings do not extend beyond the sounding box, including the hammered dulcimer, psaltery, Appalachian dulcimer, guqin, guzheng (Chinese zither), koto, gusli, kantele,...", "image": "http://img.freebase.com/api/trans/raw/m/02c4cl9", "name": "Zither", "thumbnail": "http://indextank.com/_static/common/demo/02c4cl9.jpg"}, "variables": {"0": 6}, "docid": "http://freebase.com/view/en/zither", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/basset_clarinet", "text": "The basset clarinet is a clarinet, similar to the usual soprano clarinet but longer and with additional keys to enable playing several additional lower notes. Typically a basset clarinet has keywork going to a low (written) C, as opposed to the standard clarinet's E or E\u266d (both written), and is most commonly a transposing instrument in A, although basset clarinets in C and B\u266d also exist, and Stephen Fox makes a \"G basset clarinet/basset horn\". The similarly named basset horn is also a...", "image": "http://img.freebase.com/api/trans/raw/m/029tknz", "name": "Basset clarinet", "thumbnail": "http://indextank.com/_static/common/demo/029tknz.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/basset_clarinet", "categories": {"family": "Clarinet"}} +{"fields": {"url": "http://freebase.com/view/en/rattlesnake", "text": "Rattlesnakes are a group of venomous snakes, genera Crotalus and Sistrurus. They belong to the subfamily of venomous snakes known as Crotalinae (pit vipers).\nThere are approximately thirty species of rattlesnake, with numerous subspecies. They receive their name for the rattle located at the end of their tails. The rattle is used as a warning device when they are threatened, taking the place of a loud hiss as with other snakes. The scientific name Crotalus derives from the Greek, \u03ba\u03c1\u03cc\u03c4\u03b1\u03bb\u03bf\u03bd,...", "image": "http://img.freebase.com/api/trans/raw/m/02cttmf", "name": "Rattlesnake", "thumbnail": "http://indextank.com/_static/common/demo/02cttmf.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/rattlesnake", "categories": {"family": "Rattle"}} +{"fields": {"url": "http://freebase.com/view/en/sackbut", "text": "The sackbut (var. \"sacbutt\"; \"sackbutt\"; \"sagbutt\") is a trombone from the Renaissance and Baroque eras, i.e., a musical instrument in the brass family similar to the trumpet except characterised by a telescopic slide with which the player varies the length of the tube to change pitches, thus allowing them to obtain chromaticism, as well as easy and accurate doubling of voices. More delicately constructed than their modern counterparts, and featuring a softer, more flexible sound, they...", "image": "http://img.freebase.com/api/trans/raw/m/0291pbj", "name": "Sackbut", "thumbnail": "http://indextank.com/_static/common/demo/0291pbj.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/sackbut", "categories": {"family": "Trombone"}} +{"fields": {"url": "http://freebase.com/view/en/twelve_string_guitar", "text": "The twelve-string guitar is an acoustic or electric guitar with 12 strings in 6 courses, which produces a richer, more ringing tone than a standard six-string guitar. Essentially, it is a type of guitar with a natural chorus effect due to the subtle differences in the frequencies produced by each of the two strings on each course.\nThe strings are placed in courses of two strings each that are usually played together. The two strings in each bass course are normally tuned an octave apart,...", "image": "http://img.freebase.com/api/trans/raw/m/03shp7_", "name": "Twelve string guitar", "thumbnail": "http://indextank.com/_static/common/demo/03shp7_.jpg"}, "variables": {"0": 20}, "docid": "http://freebase.com/view/en/twelve_string_guitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/clash_cymbals", "text": "Clash cymbals or hand cymbals are cymbals played in identical pairs by holding one cymbal in each hand and striking the two together.\nThe technical term clash cymbal is rarely used. In musical scores, clash cymbals are normally indicated as cymbals, crash cymbals, or sometimes simply C.C. If another type of cymbal, for example a suspended cymbal, is required in an orchestral score, then for historical reasons this is often indicated cymbals. Some composers and arrangers use the plural...", "image": "http://img.freebase.com/api/trans/raw/m/04pj4dz", "name": "Clash cymbals", "thumbnail": "http://indextank.com/_static/common/demo/04pj4dz.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/clash_cymbals", "categories": {"family": "Cymbal"}} +{"fields": {"url": "http://freebase.com/view/en/liuqin", "text": "The liuqin (\u67f3\u7434; pinyin: li\u01d4q\u00edn) is a four-stringed Chinese mandolin with a pear-shaped body. It is small in size, almost a miniature copy of another Chinese plucked musical instrument, the pipa. The range of its voice is much higher than the pipa, and it has its own special place in Chinese music, whether in orchestral music or in solo pieces. This has been the result of a modernization in its usage in recent years, leading to a gradual elevation in status of the liuqin from an accompaniment...", "image": "http://img.freebase.com/api/trans/raw/m/02d91l2", "name": "Liuqin", "thumbnail": "http://indextank.com/_static/common/demo/02d91l2.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/liuqin", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/tambourine", "text": "The tambourine or marine (commonly called tambo) is a musical instrument of the percussion family consisting of a frame, often of wood or plastic, with pairs of small metal jingles, called \"zils\". Classically the term tambourine denotes an instrument with a drumhead, though some variants may not have a head at all. Tambourines are often used with regular percussion sets. They can be mounted, but position is largely down to preference.\nTambourines come in many different shapes with the most...", "image": "http://img.freebase.com/api/trans/raw/m/0291sfp", "name": "Tambourine", "thumbnail": "http://indextank.com/_static/common/demo/0291sfp.jpg"}, "variables": {"0": 33}, "docid": "http://freebase.com/view/en/tambourine", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/baglama", "text": "The ba\u011flama (Turkish: ba\u011flama, from ba\u011flamak, \"to tie\", pronounced\u00a0[ba\u02d0\u026ba\u02c8ma]) is a stringed musical instrument shared by various cultures in the Eastern Mediterranean, Near East, and Central Asia.\nIt is sometimes referred to as the saz (from the Persian \u0633\u0627\u0632\u200e, meaning a kit or set), although the term \"saz\" actually refers to a family of plucked string instruments, long-necked lutes used in Ottoman classical music, Turkish folk music, Azeri music, Kurdish music, Persian music, Assyrian music,...", "image": "http://img.freebase.com/api/trans/raw/m/02c5x6v", "name": "Ba\u011flama", "thumbnail": "http://indextank.com/_static/common/demo/02c5x6v.jpg"}, "variables": {"0": 7}, "docid": "http://freebase.com/view/en/baglama", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/soprano_saxophone", "text": "The soprano saxophone is a variety of the saxophone, a woodwind instrument, invented in 1840. The soprano is the third smallest member of the saxophone family, which consists (from smallest to largest) of the soprillo, sopranino, soprano, alto, tenor, baritone, bass, contrabass and tubax.\nA transposing instrument pitched in the key of B\u266d, modern soprano saxophones with a high F# key have a range from A\u266d3 to E6 and are therefore pitched one octave above the tenor saxophone. Some saxophones...", "image": "http://img.freebase.com/api/trans/raw/m/02dl_1s", "name": "Soprano saxophone", "thumbnail": "http://indextank.com/_static/common/demo/02dl_1s.jpg"}, "variables": {"0": 51}, "docid": "http://freebase.com/view/en/soprano_saxophone", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/sarod", "text": "The sarod is a stringed musical instrument, used mainly in Indian classical music. Along with the sitar, it is the most popular and prominent instrument in the classical music of Hindustan (northern India, Bangladesh and Pakistan). The sarod is known for a deep, weighty, introspective sound, in contrast with the sweet, overtone-rich texture of the sitar, with sympathetic strings that give it a resonant, reverberant quality. It is a fretless instrument able to produce the continuous slides...", "image": "http://img.freebase.com/api/trans/raw/m/044r6x1", "name": "Sarod", "thumbnail": "http://indextank.com/_static/common/demo/044r6x1.jpg"}, "variables": {"0": 13}, "docid": "http://freebase.com/view/en/sarod", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/tabor", "text": "Tabor, or tabret, (\"Tabwrdd\" = Welsh) refers to a portable snare drum played with one hand. The word \"tabor\" is simply an English variant of a Latin-derived word meaning \"drum\" - cf. tambour (Fr.), tamburo (It.). It has been used in the military as a marching instrument, and has been used as accompaniment in parades and processions.\nA tabor has a cylindrical wood shell, two skin heads tightened by rope tension, a leather strap, and an adjustable gut snare. Each tabor has a pitch range of...", "image": "http://img.freebase.com/api/trans/raw/m/02g57g5", "name": "Tabor", "thumbnail": "http://indextank.com/_static/common/demo/02g57g5.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/tabor", "categories": {"family": "Snare drum"}} +{"fields": {"url": "http://freebase.com/view/en/koto", "text": "The koto (\u7b8f) is a traditional Japanese stringed musical instrument, similar to the Chinese guzheng. The koto is the national instrument of Japan. Koto are about 180\u00a0centimetres (71\u00a0in) width, and made from kiri wood (Paulownia tomentosa). They have 13 strings that are strung over 13 movable bridges along the width of the instrument. Players can adjust the string pitches by moving these bridges before playing, and use three finger picks (on thumb, index finger, and middle finger) to pluck the...", "image": "http://img.freebase.com/api/trans/raw/m/02bg10g", "name": "Koto", "thumbnail": "http://indextank.com/_static/common/demo/02bg10g.jpg"}, "variables": {"0": 5}, "docid": "http://freebase.com/view/en/koto", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/haegeum", "text": "The haegeum is a traditional Korean string instrument, resembling a fiddle. It has a rodlike neck, a hollow wooden soundbox, and two silk strings, and is held vertically on the knee of the performer and played with a bow.\nThe haegeum is related to similar Chinese instruments in the huqin family of instruments, such as the erhu. Of these, it is most closely related to the ancient xiqin, as well as the erxian used in nanguan and Cantonese music.\nThe sohaegeum (\uc18c\ud574\uae08) is a modernized fiddle with...", "image": "http://img.freebase.com/api/trans/raw/m/03sql3x", "name": "Haegeum", "thumbnail": "http://indextank.com/_static/common/demo/03sql3x.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/haegeum", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/violone", "text": "The term violone (literally \"large viol\" in Italian, \"-one\" being the augmentative suffix) can refer to several distinct large, bowed musical instruments which belong to either the viol or violin family. The violone is sometimes a fretted instrument, and may have six, five, four, or even only three strings. The violone is also not always a contrabass instrument. In modern parlance, one usually tries to clarify the 'type' of violone by adding a qualifier based on the tuning (such as \"G...", "image": "http://img.freebase.com/api/trans/raw/m/02g1bpy", "name": "Violone", "thumbnail": "http://indextank.com/_static/common/demo/02g1bpy.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/violone", "categories": {"family": "Viol"}} +{"fields": {"url": "http://freebase.com/view/en/biniou", "text": "Binio\u00f9 means bagpipe in the Breton language.\nThere are two bagpipes called binio\u00f9 in Brittany: the traditional binio\u00f9 kozh (kozh means \"old\" in Breton) and the binio\u00f9 bras (bras means \"big\"), which was brought into Brittany from Scotland in the late 19th century. The oldest native bagpipe in Brittany is the veuze, from which the binio\u00f9 kozh is thought to be derived.\nThe binio\u00f9 bras is essentially the same as the Scottish Great Highland Bagpipe; sets are manufactured by Breton makers or...", "image": "http://img.freebase.com/api/trans/raw/m/05m2syg", "name": "Biniou", "thumbnail": "http://indextank.com/_static/common/demo/05m2syg.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/biniou", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/mexican_vihuela", "text": "Vihuela is the name of two different guitar-like string instruments: the historical vihuela (proper) of 16th century Spain, usually with 12 paired strings, and the Mexican vihuela from 19th century Mexico with five strings and typically played in mariachi groups.\nWhile the Mexican vihuela shares the same name as the historic Spanish plucked string instrument, the two have little to do with each other, and they are not closely related. The Mexican vihuela has more in common with the Timple...", "image": "http://img.freebase.com/api/trans/raw/m/09h1b5x", "name": "Mexican vihuela", "thumbnail": "http://indextank.com/_static/common/demo/09h1b5x.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/mexican_vihuela", "categories": {"family": "Vihuela"}} +{"fields": {"url": "http://freebase.com/view/en/acoustic_bass_guitar", "text": "The acoustic bass guitar (also called ABG or acoustic bass) is a bass instrument with a hollow wooden body similar to, though usually somewhat larger than a steel-string acoustic guitar. Like the traditional electric bass guitar and the double bass, the acoustic bass guitar commonly has four strings, which are normally tuned E-A-D-G, an octave below the lowest four strings of the 6-string guitar, which is the same tuning pitch as an electric bass guitar.\nBecause it can be difficult to hear...", "image": "http://img.freebase.com/api/trans/raw/m/02dxvg3", "name": "Acoustic bass guitar", "thumbnail": "http://indextank.com/_static/common/demo/02dxvg3.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/acoustic_bass_guitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/tom-tom_drum", "text": "A tom-tom drum (not to be confused with a tam-tam) is a cylindrical drum with no snare.\nAlthough \"tom-tom\" is the British term for a child's toy drum, the name came originally from the Anglo-Indian and Sinhala; the tom-tom itself comes from Asian or Native American cultures. The tom-tom drum is also a traditional means of communication. The tom-tom drum was added to the drum kit in the early part of the 20th century.\nThe first drum kit tom-toms had no rims; the heads were tacked to the...", "image": "http://img.freebase.com/api/trans/raw/m/02bdbkf", "name": "Tom-tom drum", "thumbnail": "http://indextank.com/_static/common/demo/02bdbkf.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/tom-tom_drum", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/ibanez_jem", "text": "Ibanez JEM is an electric guitar manufactured by Ibanez and first produced in 1987. The guitar's most notable user is its co-designer, Steve Vai. As of 2010, there have been five sub-models of the JEM: the JEM7, JEM77, JEM777, JEM555 and the JEM333. Although the Ibanez JEM series is a signature series guitar, Ibanez mass-produces several of the guitar's sub-models.\nThe Ibanez JEM series is heavily influenced by the superstrat to model name or bodyshape is called a soloist concept, a more...", "image": "http://img.freebase.com/api/trans/raw/m/02bdlbh", "name": "Ibanez JEM", "thumbnail": "http://indextank.com/_static/common/demo/02bdlbh.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/ibanez_jem", "categories": {"family": "Electric guitar"}} +{"fields": {"url": "http://freebase.com/view/en/gibson_les_paul", "text": "The Gibson Les Paul is a solid body electric guitar that was first sold in 1952. The Les Paul was designed by Ted McCarty in collaboration with popular guitarist Les Paul, whom Gibson enlisted to endorse the new model. It is one of the most well-known electric guitar types in the world, along with the Fender Stratocaster and Telecaster.\nThe Les Paul model was the result of a design collaboration between Gibson Guitar Corporation and the late jazz guitarist and electronics inventor Les Paul....", "image": "http://img.freebase.com/api/trans/raw/m/029xg0x", "name": "Gibson Les Paul", "thumbnail": "http://indextank.com/_static/common/demo/029xg0x.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/gibson_les_paul", "categories": {"family": "Electric guitar"}} +{"fields": {"url": "http://freebase.com/view/m/09_4nd", "text": "The tulum (guda (\u10d2\u10e3\u10d3\u10d0) in Laz) is a musical instrument, a form of bagpipe from Turkey. It is droneless with two parallel chanters, usually played by the Laz, Hamsheni people, and Pontic Greeks (particularly Chaldians). It is a prominent instrument in the music of Pazar, Hem\u015fin, \u00c7aml\u0131hem\u015fin, Arde\u015fen, F\u0131nd\u0131kl\u0131, Arhavi, Hopa, partly in other districts of Artvin and in the villages of the Tatos range (the watershed between the provinces of Rize and Trabzon) of \u0130spir. Tulum is the instrument of...", "image": "http://img.freebase.com/api/trans/raw/m/02fkf23", "name": "Tulum", "thumbnail": "http://indextank.com/_static/common/demo/02fkf23.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/09_4nd", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/ukulele", "text": "The ukulele, ( /\u02ccju\u02d0k\u0259\u02c8le\u026ali\u02d0/ EW-k\u0259-LAY-lee; from Hawaiian: \u02bbukulele [\u02c8\u0294uku\u02c8l\u025bl\u025b]; variantly spelled ukelele in the UK), sometimes abbreviated to uke, is a chordophone classified as a plucked lute; it is a subset of the guitar family of instruments, generally with four nylon or gut strings or four courses of strings.\nThe ukulele originated in the 19th century as a Hawaiian interpretation of the cavaquinho or braguinha and the raj\u00e3o, small guitar-like instruments taken to Hawai\u02bbi by...", "image": "http://img.freebase.com/api/trans/raw/m/02bk34k", "name": "Ukulele", "thumbnail": "http://indextank.com/_static/common/demo/02bk34k.jpg"}, "variables": {"0": 95}, "docid": "http://freebase.com/view/en/ukulele", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/messiah_stradivarius", "text": "The Messiah-Salabue Stradivarius of 1716 is a violin made by Italian luthier Antonio Stradivari of Cremona.\nThe Messiah, sobriquet Le Messie, remained in the Stradivarius workshop until his death in 1737. It was then sold by his son Paolo to Count Cozio di Salabue in 1775, and for a time, the violin bore the name Salabue. The instrument was then purchased by Luigi Tarisio in 1827. Upon Tarisio\u2019s death, in 1854, French luthier Jean Baptiste Vuillaume of Paris purchased The Messiah along with...", "image": "http://img.freebase.com/api/trans/raw/m/05m3gjg", "name": "Messiah Stradivarius", "thumbnail": "http://indextank.com/_static/common/demo/05m3gjg.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/messiah_stradivarius", "categories": {"family": "Violin"}} +{"fields": {"url": "http://freebase.com/view/en/brass_instrument", "text": "A brass instrument is a musical instrument whose sound is produced by sympathetic vibration of air in a tubular resonator in sympathy with the vibration of the player's lips. Brass instruments are also called labrosones, literally meaning \"lip-vibrated instruments\".\nThere are several factors involved in producing different pitches on a brass instrument: One is alteration of the player's lip tension (or \"embouchure\"), and another is air flow. Also, slides (or valves) are used to change the...", "image": "http://img.freebase.com/api/trans/raw/m/02ds_pg", "name": "Brass instrument", "thumbnail": "http://indextank.com/_static/common/demo/02ds_pg.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/brass_instrument", "categories": {"family": "Wind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/bowed_psaltery", "text": "A bowed psaltery is a psaltery that is played with a bow.\nIn 1925 a German patent was issued to the Clemens Neuber Company for a bowed psaltery which also included a set of strings arranged in chords, so that one could play the melody on the bowed psaltery strings, and strum accompaniment with the other hand. These are usually called violin zithers.\nSimilar instruments were being produced by American companies of the same time period, often with Hawaiian-inspired names, such as Hawaiian Art...", "image": "http://img.freebase.com/api/trans/raw/m/02g96xy", "name": "Bowed psaltery", "thumbnail": "http://indextank.com/_static/common/demo/02g96xy.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/bowed_psaltery", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/ocarina", "text": "The ocarina ( /\u0252k\u0259\u02c8ri\u02d0n\u0259/) is an ancient flute-like wind instrument. Variations do exist, but a typical ocarina is an enclosed space with four to twelve finger holes and a mouthpiece that projects from the body. It is often ceramic, but other materials may also be used, such as plastic, wood, glass, clay, and metal.\nThe ocarina belongs to a very old family of instruments, believed to date back to over 12,000 years. Ocarina-type instruments have been of particular importance in Chinese and...", "image": "http://img.freebase.com/api/trans/raw/m/02bzf1c", "name": "Ocarina", "thumbnail": "http://indextank.com/_static/common/demo/02bzf1c.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/ocarina", "categories": {"family": "Duct flutes"}} +{"fields": {"url": "http://freebase.com/view/en/orchestrion", "text": "An orchestrion is a generic name for a machine that plays music and is designed to sound like an orchestra or band. Orchestrions may be operated by means of a large pinned cylinder or by a music roll and less commonly book music. The sound is usually produced by pipes, though they will be voiced differently to those found in a pipe organ, as well as percussion instruments. Many orchestrions contain a piano as well.\nThe fist known automatic playing orchestrion was the panharmonicon a musical...", "image": "http://img.freebase.com/api/trans/raw/m/02fkp50", "name": "Orchestrion", "thumbnail": "http://indextank.com/_static/common/demo/02fkp50.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/orchestrion", "categories": {"family": "Mechanical organ"}} +{"fields": {"url": "http://freebase.com/view/en/didgeridoo", "text": "The didgeridoo (also known as a didjeridu or didge) is a wind instrument developed by Indigenous Australians of northern Australia at least 1,500 years ago and is still in widespread usage today both in Australia and around the world. It is sometimes described as a natural wooden trumpet or \"drone pipe\". Musicologists classify it as a brass aerophone.\nThere are no reliable sources stating the didgeridoo's exact age. Archaeological studies of rock art in Northern Australia suggest that the...", "image": "http://img.freebase.com/api/trans/raw/m/02912jy", "name": "Didgeridoo", "thumbnail": "http://indextank.com/_static/common/demo/02912jy.jpg"}, "variables": {"0": 34}, "docid": "http://freebase.com/view/en/didgeridoo", "categories": {"family": "Natural brass instruments"}} +{"fields": {"url": "http://freebase.com/view/en/rackett", "text": "The rackett is a Renaissance-era double reed wind instrument related to the bassoon.\nThere are several sizes of rackett, in a family ranging from soprano to great bass. Relative to their pitch, racketts are quite small (the tenor rackett is only 4\u00bd inches long, yet its lowest note is F, two octaves below middle C). This is achieved through its ingenious construction. The body consists of a wooden chamber into which nine parallel cylinders are drilled. These are connected alternately at the...", "image": "http://img.freebase.com/api/trans/raw/m/03r57_w", "name": "Rackett", "thumbnail": "http://indextank.com/_static/common/demo/03r57_w.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/rackett", "categories": {"family": "Wind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/soprano_cornet", "text": "The soprano cornet is a brass instrument that is very similar to the standard B\u266d cornet. It is a transposing instrument in E\u266d, pitched higher than the standard B\u266d cornet.\nOne soprano cornet is usually seen in brass bands and silver bands and can often be found playing lead or descant parts in ensembles.", "image": "http://img.freebase.com/api/trans/raw/m/05lxf0b", "name": "Soprano cornet", "thumbnail": "http://indextank.com/_static/common/demo/05lxf0b.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/soprano_cornet", "categories": {"family": "Cornet"}} +{"fields": {"url": "http://freebase.com/view/en/mezzo-soprano_saxophone", "text": "The mezzo-soprano saxophone, sometimes called the F alto saxophone, is an instrument in the saxophone family. It is in the key of F, pitched a whole step above the alto saxophone. Its size and the sound are similar to the E\u266d alto, although the upper register sounds more like a B\u266d soprano. Very few mezzo-sopranos exist \u2014 they were only produced in 1928 and 1929 by the C. G. Conn company. They were not popular and did not sell widely, as their production coincided with the Wall Street Crash of...", "image": "http://img.freebase.com/api/trans/raw/m/02gsfgs", "name": "Mezzo-soprano saxophone", "thumbnail": "http://indextank.com/_static/common/demo/02gsfgs.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/mezzo-soprano_saxophone", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/tumpong", "text": "The tumpong (also inci by Maranao) is a type of Philippine bamboo flute used by the Maguindanaon, half the size of the largest bamboo flute, the palendag. A lip-valley flute like the palendag, the tumpong makes a sound when players blow through \u0130NC\u0130 GELD\u0130 a bamboo reed placed on top of the instrument and the air stream produced is passed over an airhole atop the instrument. This masculine instrument is usually played during family gatherings in the evening and is presently the most common...", "image": "http://img.freebase.com/api/trans/raw/m/03t2fx_", "name": "Tumpong", "thumbnail": "http://indextank.com/_static/common/demo/03t2fx_.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/tumpong", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/zhonghu", "text": "The zhonghu (\u4e2d\u80e1, pinyin: zh\u014dngh\u00fa) is a low-pitched Chinese bowed string instrument. Together with the erhu and gaohu, it is a member of the huqin family, and was developed in the 20th century as the alto member of the huqin family (similar to how the European viola is used in traditional Chinese orchestras).\nThe zhonghu is analogous with the erhu, but is slightly larger and lower pitched. Its body is covered on the playing end with snakeskin. The instrument has two strings, which are...", "image": "http://img.freebase.com/api/trans/raw/m/05mnrh7", "name": "Zhonghu", "thumbnail": "http://indextank.com/_static/common/demo/05mnrh7.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/zhonghu", "categories": {"family": "Bowed string instruments"}} +{"fields": {"url": "http://freebase.com/view/en/tahitian_ukulele", "text": "The Tahitian ukulele (also known as the Tahitian banjo) is a short-necked fretted lute with eight nylon strings in four doubled courses, native to Tahiti. This variant of the older Hawaiian Ukulele is noted by a higher and thinner sound and is often strummed much faster.\nThe Tahitian ukulele is significantly different from other ukuleles in that it does not have a hollow soundbox. The body (including the head and neck) is usually carved from a single piece of wood, with a wide conical hole...", "image": "http://img.freebase.com/api/trans/raw/m/04rxjcm", "name": "Tahitian ukulele", "thumbnail": "http://indextank.com/_static/common/demo/04rxjcm.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/tahitian_ukulele", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/fender_stratocaster", "text": "The Fender Stratocaster, often referred to as \"Strat\", is a model of electric guitar designed by Leo Fender, George Fullerton, and Freddie Tavares in 1954, and manufactured continuously by the Fender Musical Instruments Corporation to the present. It is a double-cutaway guitar, with an extended top horn for balance while standing. The Stratocaster has been used by many leading guitarists and can be heard on many historic recordings. Along with the Gibson Les Paul, the Gibson SG and the...", "image": "http://img.freebase.com/api/trans/raw/m/044rrlv", "name": "Fender Stratocaster", "thumbnail": "http://indextank.com/_static/common/demo/044rrlv.jpg"}, "variables": {"0": 40}, "docid": "http://freebase.com/view/en/fender_stratocaster", "categories": {"family": "Guitar"}} +{"fields": {"url": "http://freebase.com/view/en/crotales", "text": "Crotales (pronounced \"kro-tah'-les\", IPA: [kro\u02c8t\u03b1les]), sometimes called antique cymbals, are percussion instruments consisting of small, tuned bronze or brass disks. Each is about 4\u00a0inches in diameter with a flat top surface and a nipple on the base. They are commonly played by being struck with hard mallets. However, they may also be played by striking two disks together in the same manner as finger cymbals, or by bowing. Their sound is rather like a small tuned bell, only with a much...", "image": "http://img.freebase.com/api/trans/raw/m/029y2l0", "name": "Crotales", "thumbnail": "http://indextank.com/_static/common/demo/029y2l0.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/crotales", "categories": {"family": "Cymbal"}} +{"fields": {"url": "http://freebase.com/view/en/drumitar", "text": "The Synthaxe Drumitar is an instrument created by Roy \"Future Man\" Wooten of B\u00e9la Fleck and the Flecktones. The Drumitar comprises piezo elements mounted in a guitar shaped body, connected by cable to assorted MIDI devices including samplers and drum machines. The instrument is one of a kind, and it was originally modified from a SynthAxe previously owned by jazz musician Lee Ritenour.\nThe Zendrum is a similar instrument that is commercially available.", "image": "http://img.freebase.com/api/trans/raw/m/03s6xrn", "name": "Drumitar", "thumbnail": "http://indextank.com/_static/common/demo/03s6xrn.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/drumitar", "categories": {"family": "Zendrum"}} +{"fields": {"url": "http://freebase.com/view/en/shawm", "text": "The shawm was a medieval and Renaissance musical instrument of the woodwind family made in Europe from the 12th century (at the latest) until the 17th century. It was developed from the oriental zurna and is the predecessor of the modern oboe. The body of the shawm was usually turned from a single piece of wood, and terminated in a flared bell somewhat like that of a trumpet. Beginning in the 16th century, shawms were made in several sizes, from sopranino to great bass, and four and...", "image": "http://img.freebase.com/api/trans/raw/m/02c9mfg", "name": "Shawm", "thumbnail": "http://indextank.com/_static/common/demo/02c9mfg.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/shawm", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/duda", "text": "The Magyar duda\u2014Hungarian duda\u2014(also known as t\u00f6ml\u00f6s\u00edp and b\u00f6rduda) is the traditional bagpipe of Hungary. It is an example of a group of bagpipes called Medio-Carparthian bagpipes. In common with most bagpipes in the area east of an imaginary line running from the Baltics to the Istrian Coast, the duda\u2019s chanters use single reeds much like Western drone reeds.\nDudmaisis or duda are made of sheep, ox, goat or dogskin or of sheep\u2019s stomach. A blowing tube is attached to the top. On one side...", "image": "http://img.freebase.com/api/trans/raw/m/02g521y", "name": "Duda", "thumbnail": "http://indextank.com/_static/common/demo/02g521y.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/duda", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/lute", "text": "Lute can refer generally to any plucked string instrument with a neck (either fretted or unfretted) and a deep round back, or more specifically to an instrument from the family of European lutes.\nThe European lute and the modern Near-Eastern oud both descend from a common ancestor via diverging evolutionary paths. The lute is used in a great variety of instrumental music from the early Renaissance to the late Baroque eras. It is also an accompanying instrument, especially in vocal works,...", "image": "http://img.freebase.com/api/trans/raw/m/03s3ynh", "name": "Lute", "thumbnail": "http://indextank.com/_static/common/demo/03s3ynh.jpg"}, "variables": {"0": 23}, "docid": "http://freebase.com/view/en/lute", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/c_melody_saxophone", "text": "The C melody saxophone is a saxophone pitched in the key of C, one whole step above the tenor saxophone. In the UK it is sometimes referred to as a \"C tenor\", and in France as a \"tenor en ut\". The C melody was part of the series of saxophones pitched in C and F, intended by the instrument's inventor, Adolphe Sax, for orchestral use. Since 1930, only saxophones in the key of B\u266d and E\u266d (originally intended by Sax for use in military bands and wind ensembles) have been produced on a large...", "image": "http://img.freebase.com/api/trans/raw/m/02bvy35", "name": "C melody saxophone", "thumbnail": "http://indextank.com/_static/common/demo/02bvy35.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/c_melody_saxophone", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/degerpipes", "text": "The electronic bagpipes are an electronic instrument emulating the tone and/or playing style of the bagpipes. Most electronic bagpipe emulators feature a simulated chanter, which is used to play the melody. Some models also produce a harmonizing drone(s). Some variants employ a simulated bag, wherein the player's pressure on the bag activates a switch maintaining a constant tone.\nElectronic bagpipes are produced to replicate various types of bagpipes from around the world, including the...", "image": "http://img.freebase.com/api/trans/raw/m/03thxc1", "name": "Electronic bagpipes", "thumbnail": "http://indextank.com/_static/common/demo/03thxc1.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/degerpipes", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/m/042v_gx", "text": "An acoustic guitar is a guitar that uses only acoustic methods to project the sound produced by its strings. The term is a retronym, coined after the advent of electric guitars, which rely on electronic amplification to make their sound audible.\nIn all types of guitars the sound is produced by the vibration of the strings. However, because the string can only displace a small amount of air, the volume of the sound needs to be increased in order to be heard. In an acoustic guitar, this is...", "image": "http://img.freebase.com/api/trans/raw/m/0290vfk", "name": "Acoustic guitar", "thumbnail": "http://indextank.com/_static/common/demo/0290vfk.jpg"}, "variables": {"0": 276}, "docid": "http://freebase.com/view/m/042v_gx", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/concertina", "text": "A concertina is a free-reed musical instrument, like the various accordions and the harmonica. It has a bellows and buttons typically on both ends of it. When pressed, the buttons travel in the same direction as the bellows, unlike accordion buttons which travel perpendicularly to it. Also, each button produces one note, while accordions typically can produce chords with a single button.\nThe concertina was developed in England and Germany, most likely independently. The English version was...", "image": "http://img.freebase.com/api/trans/raw/m/0292mbm", "name": "Concertina", "thumbnail": "http://indextank.com/_static/common/demo/0292mbm.jpg"}, "variables": {"0": 10}, "docid": "http://freebase.com/view/en/concertina", "categories": {"family": "Accordion"}} +{"fields": {"url": "http://freebase.com/view/en/an_tranh", "text": "The \u0111\u00e0n tranh (\u5f48\u7b8f) is a plucked zither of Vietnam. It has a wooden body and steel strings, each of which is supported by a bridge in the shape of an inverted \"V.\"\nThe \u0111\u00e0n tranh can be used either as a solo instrument, or as one of many to accompany singer/s. The \u0111\u00e0n tranh originally had 16 strings but it was renovated by Master Nguy\u1ec5n V\u0129nh B\u1ea3o (b. 1918) of South Vietnam in the mid 1950s. Since then, the 17-stringed \u0111\u00e0n tranh has gained massive popularity and become the most preferred form of...", "image": "http://img.freebase.com/api/trans/raw/m/03srw1w", "name": "\u0110\u00e0n tranh", "thumbnail": "http://indextank.com/_static/common/demo/03srw1w.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/an_tranh", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/boha", "text": "The boha (prounouced bou-heu, also known as the Cornemuse Landaise or bohaossac) is a type of bagpipe native to the Landes and Gascony regions of southwestern France.\nThis bagpipe is notable in that it bears a greater resemblance to Eastern European bagpipes, particularly the contra-chanter bagpipes of the Pannonian Plain (e.g., the Hungarian duda), than to other Western European pipes. It features both a chanter and a drone bored into a common rectangular body. Both chanter and drone...", "image": "http://img.freebase.com/api/trans/raw/m/078qd7p", "name": "Boha", "thumbnail": "http://indextank.com/_static/common/demo/078qd7p.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/boha", "categories": {"family": "Bagpipes"}} +{"fields": {"url": "http://freebase.com/view/en/balalaika", "text": "The balalaika (Russian: \u0431\u0430\u043b\u0430\u043b\u0430\u0301\u0439\u043a\u0430, Russian pronunciation:\u00a0[b\u0250l\u0250\u02c8lajk\u0259]) is a stringed musical instrument of Russian origin, with a characteristic triangular body and three strings.\nThe balalaika family of instruments includes, from the highest-pitched to the lowest, the prima balalaika, secunda balalaika, alto balalaika, bass balalaika and contrabass balalaika. All have three-sided bodies, spruce or fir tops, backs made of 3-9 wooden sections, and usually three strings. The prima balalaika...", "image": "http://img.freebase.com/api/trans/raw/m/02bcg1_", "name": "Balalaika", "thumbnail": "http://indextank.com/_static/common/demo/02bcg1_.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/balalaika", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/bell", "text": "A bell is a simple sound-making device. The bell is a percussion instrument and an idiophone. Its form is usually a hollow, cup-shaped object, which resonates upon being struck. The striking implement can be a tongue suspended within the bell, known as a clapper, a small, free sphere enclosed within the body of the bell or a separate mallet or hammer.\nBells are usually made of cast metal, but small bells can also be made from ceramic or glass. Bells can be of all sizes: from tiny dress...", "image": "http://img.freebase.com/api/trans/raw/m/044wp6c", "name": "Bell", "thumbnail": "http://indextank.com/_static/common/demo/044wp6c.jpg"}, "variables": {"0": 3}, "docid": "http://freebase.com/view/en/bell", "categories": {"family": "Percussion"}} +{"fields": {"url": "http://freebase.com/view/en/piccolo", "text": "The piccolo (Italian for small) is a half-size flute, and a member of the woodwind family of musical instruments. The piccolo has the same fingerings as its larger sibling, the standard transverse flute, but the sound it produces is an octave higher than written. This gave rise to the name \"ottavino,\" the name by which the instrument is referred to in the scores of Italian composers.\nPiccolos are now only manufactured in the key of C; however, they were once also available in D\u266d. It was for...", "image": "http://img.freebase.com/api/trans/raw/m/02bt06d", "name": "Piccolo", "thumbnail": "http://indextank.com/_static/common/demo/02bt06d.jpg"}, "variables": {"0": 8}, "docid": "http://freebase.com/view/en/piccolo", "categories": {"family": "Flute (transverse)"}} +{"fields": {"url": "http://freebase.com/view/en/geomungo", "text": "The geomungo (also spelled komungo or k\u014fmun'go) or hyeongeum (literally \"black zither\", also spelled hyongum or hy\u014fn'g\u016dm) is a traditional Korean stringed musical instrument of the zither family of instruments with both bridges and frets. Scholars believe that the name refers to Goguryeo and translates to \"Goguryeo zither\" or that it refers to the colour and translates to \"black crane zither\".\nThe instrument originated circa the fourth century (see Anak Tomb No.3 infra) through the 7th...", "image": "http://img.freebase.com/api/trans/raw/m/041hxhx", "name": "Geomungo", "thumbnail": "http://indextank.com/_static/common/demo/041hxhx.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/geomungo", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/euphonium", "text": "The euphonium is a conical-bore, tenor-voiced brass instrument. It derives its name from the Greek word euphonos, meaning \"well-sounding\" or \"sweet-voiced\" (eu means \"well\" or \"good\" and phonos means \"of sound\", so \"of good sound\"). The euphonium is a valved instrument; nearly all current models are piston valved, though rotary valved models do exist.\nA person who plays the euphonium is sometimes called a euphoniumist, euphophonist, or a euphonist, while British players often colloquially...", "image": "http://img.freebase.com/api/trans/raw/m/02bdncm", "name": "Euphonium", "thumbnail": "http://indextank.com/_static/common/demo/02bdncm.jpg"}, "variables": {"0": 7}, "docid": "http://freebase.com/view/en/euphonium", "categories": {"family": "Brass instrument"}} +{"fields": {"url": "http://freebase.com/view/en/fender_jazz_bass", "text": "The Jazz Bass (or J Bass) was the second model of electric bass created by Leo Fender. The bass is distinct from the Precision Bass in that its tone is brighter and richer in the midrange and treble with less emphasis on the fundamental harmonic. Because of this, many bass players who want to be more \"forward\" in the mix (including smaller bands such as power trios) prefer the Jazz Bass. The sound of the Fender Jazz Bass has been fundamental in the development of signature sounds in certain...", "image": "http://img.freebase.com/api/trans/raw/m/02f6r8s", "name": "Fender Jazz Bass", "thumbnail": "http://indextank.com/_static/common/demo/02f6r8s.jpg"}, "variables": {"0": 5}, "docid": "http://freebase.com/view/en/fender_jazz_bass", "categories": {"family": "Bass guitar"}} +{"fields": {"url": "http://freebase.com/view/en/resonator_guitar", "text": "A resonator guitar or resophonic guitar is an acoustic guitar whose sound is produced by one or more spun metal cones (resonators) instead of the wooden sound board (guitar top/face). Resonator guitars were originally designed to be louder than conventional acoustic guitars which were overwhelmed by horns and percussion instruments in dance orchestras. They became prized for their distinctive sound, however, and found life with several musical styles (most notably bluegrass and also blues)...", "image": "http://img.freebase.com/api/trans/raw/m/02f27b3", "name": "Resonator guitar", "thumbnail": "http://indextank.com/_static/common/demo/02f27b3.jpg"}, "variables": {"0": 8}, "docid": "http://freebase.com/view/en/resonator_guitar", "categories": {"family": "Guitar"}} +{"fields": {"url": "http://freebase.com/view/m/0dkc489", "text": "Asaph Music has a vision to see the name of the Lord exalted through worship and people encouraged to trust in Christ. Its motto is \"O God, my heart is fixed; I will sing and give praise, even with my glory,\" founded upon Psalm 108:1. It is our earnest desired to see Christians inspired in their Christian walk through our musical presentations. Asaph Music works in conjunction with Kingdom Builders Productions in producing musical works.", "image": "http://img.freebase.com/api/trans/raw/m/0dkc4d5", "name": "Asaph Music", "thumbnail": "http://indextank.com/_static/common/demo/0dkc4d5.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/0dkc489", "categories": {"family": "Acoustic guitar"}} +{"fields": {"url": "http://freebase.com/view/en/ems_vcs_3", "text": "The VCS 3 (an initialism for Voltage Controlled Studio with 3 oscillators) is a portable analog synthesiser with a flexible semi-modular voice architecture.\nIt was created in 1969 by Peter Zinovieff's EMS company. The electronics were largely designed by David Cockerell and the machine's distinctive visual appearance was the work of electronic composer Tristram Cary. The VCS 3 was more or less the first portable commercially available synthesiser\u2014portable in the sense that the VCS 3 was...", "image": "http://img.freebase.com/api/trans/raw/m/03t04ld", "name": "EMS VCS 3", "thumbnail": "http://indextank.com/_static/common/demo/03t04ld.jpg"}, "variables": {"0": 21}, "docid": "http://freebase.com/view/en/ems_vcs_3", "categories": {"family": "Synthesizer"}} +{"fields": {"url": "http://freebase.com/view/en/mandolin", "text": "A mandolin (Italian: mandolino) is a musical instrument in the lute family (plucked, or strummed). It descends from the mandore, a soprano member of the lute family. The mandolin soundboard (the top) comes in many shapes\u2014but generally round or teardrop-shaped, sometimes with scrolls or other projections. A mandolin may have f-holes, or a single round or oval sound hole. A round or oval sound hole may be bordered with decorative rosettes or purfling, but usually doesn't feature an intricately...", "image": "http://img.freebase.com/api/trans/raw/m/03s49h2", "name": "Mandolin", "thumbnail": "http://indextank.com/_static/common/demo/03s49h2.jpg"}, "variables": {"0": 266}, "docid": "http://freebase.com/view/en/mandolin", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/pedal_steel_guitar", "text": "The pedal steel guitar is a type of electric guitar that uses a metal bar to \"fret\" or shorten the length of the strings, rather than fingers on strings as with a conventional guitar. Unlike other types of steel guitar, it also uses foot pedals and knee levers to affect the pitch, hence the name \"pedal\" steel guitar. The word \"steel\" in the name comes from the metal tone bar, which is called a \"steel\", and which acts as a moveable fret, shortening the effective length of the string or...", "image": "http://img.freebase.com/api/trans/raw/m/029z7lx", "name": "Pedal steel guitar", "thumbnail": "http://indextank.com/_static/common/demo/029z7lx.jpg"}, "variables": {"0": 39}, "docid": "http://freebase.com/view/en/pedal_steel_guitar", "categories": {"family": "Electric guitar"}} +{"fields": {"url": "http://freebase.com/view/en/ruan", "text": "The ruan (\u962e, pinyin: ru\u01cen) is a Chinese plucked string instrument. It is a lute with a fretted neck, a circular body, and four strings. Its strings were formerly made of silk but since the 20th century they have been made of steel (flatwound for the lower strings). The modern ruan has 24 frets with 12 semitones on each string, which has greatly expanded its range from a previous 13 frets. The frets are commonly made of ivory. Or in recent times, metal mounted on wood. The metal frets produce...", "image": "http://img.freebase.com/api/trans/raw/m/044vyw1", "name": "Ruan", "thumbnail": "http://indextank.com/_static/common/demo/044vyw1.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/ruan", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/banjo", "text": "The banjo is a stringed instrument with, typically, four or five strings, which vibrate a membrane of plastic material or animal hide stretched over a circular frame. Simpler forms of the instrument were fashioned by enslaved Africans in Colonial America, adapted from several African instruments of the same basic design.\nThe banjo is usually associated with country, folk, classical music, Irish traditional music and bluegrass music. Historically, the banjo occupied a central place in African...", "image": "http://img.freebase.com/api/trans/raw/m/0290x6g", "name": "Banjo", "thumbnail": "http://indextank.com/_static/common/demo/0290x6g.jpg"}, "variables": {"0": 357}, "docid": "http://freebase.com/view/en/banjo", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/m/0cc8zh", "text": "The bombard, also known as talabard or ar vombard in the Breton language or bombarde in French, is a popular contemporary conical bore double reed instrument widely used to play traditional Breton music. The bombard is a woodwind instrument; the reed is held between the lips. The bombard is a member of the oboe family. Describing it as an oboe, however, can be misleading since it has a broader and very powerful sound, vaguely resembling a trumpet. It is played as other oboes are played, with...", "image": "http://img.freebase.com/api/trans/raw/m/05l29l2", "name": "Bombard", "thumbnail": "http://indextank.com/_static/common/demo/05l29l2.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/m/0cc8zh", "categories": {"family": "Woodwind instrument"}} +{"fields": {"url": "http://freebase.com/view/en/charango", "text": "The charango is a small South American stringed instrument of the lute family, about 66 cm long, traditionally made with the shell of the back of an armadillo. Many contemporary charangos are now made with different types of wood. It typically has 10 strings in five courses of 2 strings each, though other variations exist.\nThe instrument was invented in the early 18th century in the Viceroyalty of Peru (nowadays Per\u00fa and Bolivia).\nWhen the Spanish conquistadores came to South America, they...", "image": "http://img.freebase.com/api/trans/raw/m/02btn26", "name": "Charango", "thumbnail": "http://indextank.com/_static/common/demo/02btn26.jpg"}, "variables": {"0": 8}, "docid": "http://freebase.com/view/en/charango", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/wurlitzer_electric_piano", "text": "The Wurlitzer electric piano was one of a series of electromechanical stringless pianos manufactured and marketed by the Rudolph Wurlitzer Company, Corinth, Mississippi, U.S. and Tonawanda, New York. The Wurlitzer company itself never called the instrument an \"electric piano\", instead inventing the phrase \"Electronic Piano\" and using this as a trademark throughout the production of the instrument. See however electronic piano, the generally accepted term for a completely different type of...", "image": "http://img.freebase.com/api/trans/raw/m/041w6pq", "name": "Wurlitzer electric piano", "thumbnail": "http://indextank.com/_static/common/demo/041w6pq.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/wurlitzer_electric_piano", "categories": {"family": "Electric piano"}} +{"fields": {"url": "http://freebase.com/view/en/jazz_guitar", "text": "The term jazz guitar may refer to either a type of guitar or to the variety of guitar playing styles used in the various genres which are commonly termed \"jazz\". The jazz-type guitar was born as a result of using electric amplification to increase the volume of conventional acoustic guitars.\nConceived in the early 1930s, the electric guitar became a necessity as jazz musicians sought to amplify their sound. Arguably, no other musical instrument had greater influence on how music evolved...", "image": "http://img.freebase.com/api/trans/raw/m/05l99wd", "name": "Jazz guitar", "thumbnail": "http://indextank.com/_static/common/demo/05l99wd.jpg"}, "variables": {"0": 2}, "docid": "http://freebase.com/view/en/jazz_guitar", "categories": {"family": "Guitar"}} +{"fields": {"url": "http://freebase.com/view/en/wagner_tuba", "text": "The Wagner tuba is a comparatively rare brass instrument that combines elements of both the French horn and the tuba. Also referred to as the \"Bayreuth Tuba\", it was originally created for Richard Wagner's operatic cycle Der Ring des Nibelungen. Since then, other composers have written for it, most notably Anton Bruckner, in whose Symphony No. 7 a quartet of them is first heard in the slow movement in memory of Wagner. The euphonium is sometimes used as a substitute when a Wagner tuba cannot...", "image": "http://img.freebase.com/api/trans/raw/m/02c8561", "name": "Wagner tuba", "thumbnail": "http://indextank.com/_static/common/demo/02c8561.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/wagner_tuba", "categories": {"family": "Tuba"}} +{"fields": {"url": "http://freebase.com/view/en/sopranino_saxophone", "text": "The sopranino saxophone is one of the smallest members of the saxophone family. A sopranino saxophone is tuned in the key of E\u266d, and sounds an octave above the alto saxophone. This saxophone has a sweet sound and although the sopranino is one of the least common of the saxophones in regular use today, it is still being produced by several of the major musical manufacturing companies. Due to their small size, sopraninos are not usually curved like other saxes. Orsi, however, does make curved...", "image": "http://img.freebase.com/api/trans/raw/m/02dl_1s", "name": "Sopranino saxophone", "thumbnail": "http://indextank.com/_static/common/demo/02dl_1s.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/sopranino_saxophone", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/baroque_guitar", "text": "The Baroque guitar is a guitar from the baroque era (c. 1600\u20131750), an ancestor of the modern classical guitar. The term is also used for modern instruments made in the same style.\nThe instrument was smaller than a modern guitar, of lighter construction, and had gut strings. The frets were also usually made of gut, and tied around the neck. A typical instrument had five courses, each consisting of two separate strings although the first (highest sounding) course was often a single string,...", "image": "http://img.freebase.com/api/trans/raw/m/03t4sks", "name": "Baroque guitar", "thumbnail": "http://indextank.com/_static/common/demo/03t4sks.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/baroque_guitar", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/contra-alto_clarinet", "text": "The contra-alto clarinet is a large, low-sounding musical instrument of the clarinet family. The modern contra-alto clarinet is pitched in the key of EEb and is sometimes incorrectly referred to as the EEb contrabass clarinet. The unhyphenated form \"contra alto clarinet\" is also sometimes used, as is \"contralto clarinet\", but the latter is confusing since the instrument's range is much lower than the contralto vocal range; the more correct term \"contra-alto\" is meant to convey, by analogy...", "image": "http://img.freebase.com/api/trans/raw/m/02b9drl", "name": "Contra-alto clarinet", "thumbnail": "http://indextank.com/_static/common/demo/02b9drl.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/contra-alto_clarinet", "categories": {"family": "Clarinet"}} +{"fields": {"url": "http://freebase.com/view/en/vihuela", "text": "Vihuela is a name given to two different guitar-like string instruments: one from 15th and 16th century Spain, usually with 12 paired strings, and the other, the Mexican vihuela, from the 19th century Mexico with five strings and typically played in Mariachi bands.\nThe vihuela, as it was known in Spain, was called the viola da mano in Italy and Portugal. The two names are functionally synonymous and interchangeable. In its most developed form, the vihuela was a guitar-like instrument with...", "image": "http://img.freebase.com/api/trans/raw/m/02csx2d", "name": "Vihuela", "thumbnail": "http://indextank.com/_static/common/demo/02csx2d.jpg"}, "variables": {"0": 7}, "docid": "http://freebase.com/view/en/vihuela", "categories": {"family": "Plucked string instrument"}} +{"fields": {"url": "http://freebase.com/view/en/baritone_saxophone", "text": "The baritone saxophone, often called \"bari sax\" (to avoid confusion with the baritone horn, which is often referred to simply as \"baritone\"), is one of the larger and lower pitched members of the saxophone family. It was invented by Adolphe Sax. The baritone is distinguished from smaller sizes of saxophone by the extra loop near its mouthpiece; this helps to keep the instrument at a practical height (the rarer bass saxophone has a similar, but larger loop). It is the lowest pitched saxophone...", "image": "http://img.freebase.com/api/trans/raw/m/044mtf9", "name": "Baritone saxophone", "thumbnail": "http://indextank.com/_static/common/demo/044mtf9.jpg"}, "variables": {"0": 29}, "docid": "http://freebase.com/view/en/baritone_saxophone", "categories": {"family": "Saxophone"}} +{"fields": {"url": "http://freebase.com/view/en/portative_organ", "text": "A portative organ (portatif organ, portativ organ, or simply portative, portatif, or portativ) (from the Latin verb portare, \"to carry\") is a small pipe organ that consists of one rank of flue pipes and played while strapped to the performer at a right angle. The performer manipulates the bellows with one hand and fingers the keys with the other. The portative organ lacks a reservoir to retain a supply of wind, thus it will only produce sound while the bellows are being operated. The...", "image": "http://img.freebase.com/api/trans/raw/m/042tms2", "name": "Portative organ", "thumbnail": "http://indextank.com/_static/common/demo/042tms2.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/portative_organ", "categories": {"family": "Pipe organ"}} +{"fields": {"url": "http://freebase.com/view/en/electric_mandolin", "text": "The electric mandolin is an instrument tuned and played as the mandolin and amplified in similar fashion to an electric guitar. As with electric guitars, electric mandolins take many forms:\nElectric mandolins were built in the United States as early as the late 1920s. Among the first companies to produce them were Stromberg-Voisinet, Electro (which later became Rickenbacker), ViViTone, and National Reso-Phonic. Gibson and Vega introduced their electric mandolins in 1936.\nIn the United...", "image": "http://img.freebase.com/api/trans/raw/m/03sgsdn", "name": "Electric mandolin", "thumbnail": "http://indextank.com/_static/common/demo/03sgsdn.jpg"}, "variables": {"0": 1}, "docid": "http://freebase.com/view/en/electric_mandolin", "categories": {"family": "Mandolin"}} +{"fields": {"url": "http://freebase.com/view/en/yazheng", "text": "The yazheng (simplified: \u8f67\u7b5d; traditional: \u8ecb\u7b8f; pinyin: y\u00e0zh\u0113ng; also spelled ya zheng or ya cheng) is a Chinese string instrument. It is a long zither similar to the guzheng but bowed by scraping with a sorghum stem dusted with resin, a bamboo stick, or a piece of forsythia wood. The musical instrument was popular in the Tang Dynasty, but is today little used except in the folk music of some parts of northern China, where it is called yaqin (simplified: \u8f67\u7434; traditional: \u8ecb\u7434).\nThe Korean ajaeng...", "image": "http://img.freebase.com/api/trans/raw/m/03spb3n", "name": "Yazheng", "thumbnail": "http://indextank.com/_static/common/demo/03spb3n.jpg"}, "variables": {"0": 0}, "docid": "http://freebase.com/view/en/yazheng", "categories": {"family": "Bowed string instruments"}} diff --git a/nebu/lib/__init__.py b/nebu/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nebu/lib/authorizenet.py b/nebu/lib/authorizenet.py new file mode 100644 index 0000000..00bcd6c --- /dev/null +++ b/nebu/lib/authorizenet.py @@ -0,0 +1,153 @@ +from xml.dom.minidom import Document, parseString +import httplib +import urlparse + + +class AuthorizeNet: + """ + Basic client for Authorize.net's Automated Recurring Billing (ARB) service + """ + + def __init__(self): + from django.conf import settings + f = open("authorize.settings.prod") if not settings.DEBUG else open("authorize.settings.debug") + for line in f: + line = line.strip() + if len(line) > 0 and not line.startswith('#'): + parts = line.split('=',1) + var = parts[0].strip() + val = parts[1].strip() + if var in ['host_url','api_login_id','transaction_key']: + cmd = 'self.%s = %s' % (var,val) + exec(cmd) + + def subscription_create(self, refId, name, length, unit, startDate, totalOccurrences, trialOccurrences, + amount, trialAmount, cardNumber, expirationDate, firstName, lastName, company, + address, city, state, zip, country): + doc,root = self._new_doc("ARBCreateSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + subscription = self._add_node(doc, root, 'subscription') + self._add_text_node(doc, subscription, 'name', name) + paymentSchedule = self._add_node(doc, subscription, 'paymentSchedule') + interval = self._add_node(doc, paymentSchedule, 'interval') + self._add_text_node(doc, interval, 'length', length) + self._add_text_node(doc, interval, 'unit', unit) + self._add_text_node(doc, paymentSchedule, 'startDate', startDate) + self._add_text_node(doc, paymentSchedule, 'totalOccurrences', totalOccurrences) + self._add_text_node(doc, paymentSchedule, 'trialOccurrences', trialOccurrences) + self._add_text_node(doc, subscription, 'amount', amount) + self._add_text_node(doc, subscription, 'trialAmount', trialAmount) + payment = self._add_node(doc, subscription, 'payment') + creditcard = self._add_node(doc, payment, 'creditCard') + self._add_text_node(doc, creditcard, 'cardNumber', cardNumber) + self._add_text_node(doc, creditcard, 'expirationDate', expirationDate) + billto = self._add_node(doc, subscription, 'billTo') + self._add_text_node(doc, billto, 'firstName', firstName) + self._add_text_node(doc, billto, 'lastName', lastName) + self._add_text_node(doc, billto, 'company', company) + self._add_text_node(doc, billto, 'address', address) + self._add_text_node(doc, billto, 'city', city) + self._add_text_node(doc, billto, 'state', state) + self._add_text_node(doc, billto, 'zip', zip) + self._add_text_node(doc, billto, 'country', country) + res = self._send_xml(doc.toxml()) + subscriptionId = res.getElementsByTagName('subscriptionId')[0].childNodes[0].nodeValue + return subscriptionId + + + def subscription_update(self, refId, subscriptionId, name, startDate, totalOccurrences, trialOccurrences, + amount, trialAmount, cardNumber, expirationDate, firstName, lastName, company, + address, city, state, zip, country): + doc,root = self._new_doc("ARBUpdateSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + self._add_text_node(doc, root, 'subscriptionId', subscriptionId) + subscription = self._add_node(doc, root, 'subscription') + if name: + self._add_text_node(doc, subscription, 'name', name) + if startDate or totalOccurrences or trialOccurrences: + paymentSchedule = self._add_node(doc, subscription, 'paymentSchedule') + if startDate: + self._add_text_node(doc, paymentSchedule, 'startDate', startDate) + if totalOccurrences: + self._add_text_node(doc, paymentSchedule, 'totalOccurrences', totalOccurrences) + if trialOccurrences: + self._add_text_node(doc, paymentSchedule, 'trialOccurrences', trialOccurrences) + if amount: + self._add_text_node(doc, subscription, 'amount', amount) + if trialAmount: + self._add_text_node(doc, subscription, 'trialAmount', trialAmount) + if cardNumber and expirationDate: + payment = self._add_node(doc, subscription, 'payment') + creditcard = self._add_node(doc, payment, 'creditCard') + self._add_text_node(doc, creditcard, 'cardNumber', cardNumber) + self._add_text_node(doc, creditcard, 'expirationDate', expirationDate) + if firstName and lastName: + billto = self._add_node(doc, subscription, 'billTo') + self._add_text_node(doc, billto, 'firstName', firstName) + self._add_text_node(doc, billto, 'lastName', lastName) + if company: + self._add_text_node(doc, billto, 'company', company) + if address and city and state and zip and country: + self._add_text_node(doc, billto, 'address', address) + self._add_text_node(doc, billto, 'city', city) + self._add_text_node(doc, billto, 'state', state) + self._add_text_node(doc, billto, 'zip', zip) + self._add_text_node(doc, billto, 'country', country) + self._send_xml(doc.toxml()) + + + def subscription_cancel(self, refId, subscriptionId): + doc,root = self._new_doc("ARBCancelSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + self._add_text_node(doc, root, 'subscriptionId', subscriptionId) + self._send_xml(doc.toxml()) + + + def _add_node(self, doc, node, name): + elem = doc.createElement(name) + node.appendChild(elem) + return elem + + def _add_text_node(self, doc, node, name, text): + elem = self._add_node(doc, node, name) + text_node = doc.createTextNode(text) + elem.appendChild(text_node) + return elem + + def _new_doc(self, operation): + doc = Document() + root = doc.createElement(operation) + root.setAttribute('xmlns','AnetApi/xml/v1/schema/AnetApiSchema.xsd') + doc.appendChild(root) + auth = self._add_node(doc, root, 'merchantAuthentication') + self._add_text_node(doc, auth, 'name', self.api_login_id) + self._add_text_node(doc, auth, 'transactionKey', self.transaction_key) + return doc, root + + def _send_xml(self, xml): + splits = urlparse.urlsplit(self.host_url) + print "connection.request('POST', "+self.host_url+", xml, {'Content-Type':'text/xml'})" + print "xml: "+xml + connection = httplib.HTTPSConnection(splits.hostname) + connection.request('POST', self.host_url, xml, {'Content-Type':'text/xml'}) + response = connection.getresponse() + response.body = response.read() + connection.close() + print "resp: "+response.body + res = parseString(response.body) + ok = res.getElementsByTagName('resultCode')[0].childNodes[0].nodeValue == "Ok" + if not ok: + code = res.getElementsByTagName('message')[0].childNodes[0].childNodes[0].nodeValue + msg = res.getElementsByTagName('message')[0].childNodes[1].childNodes[0].nodeValue + " (%s)"%code + raise BillingException(msg,code) + return res + + +class BillingException(Exception): + def __init__(self, msg, code): + self.msg = msg + self.code = code + def __str__(self): + return repr(self.msg) + + diff --git a/nebu/lib/encoder.py b/nebu/lib/encoder.py new file mode 100644 index 0000000..f6bb4dd --- /dev/null +++ b/nebu/lib/encoder.py @@ -0,0 +1,74 @@ +# Short URL Generator + +#DEFAULT_ALPHABET = 'JedR8LNFY2j6MrhkBSADUyfP5amuH9xQCX4VqbgpsGtnW7vc3TwKE' +DEFAULT_ALPHABET = 'ed82j6rhkyf5amu9x4qbgpstn7vc3w1ioz' +DEFAULT_BLOCK_SIZE = 22 + +class Encoder(object): + def __init__(self, alphabet=DEFAULT_ALPHABET, block_size=DEFAULT_BLOCK_SIZE): + self.alphabet = alphabet + self.block_size = block_size + self.mask = (1 << block_size) - 1 + self.mapping = range(block_size) + self.mapping.reverse() + def encode_url(self, n, min_length=0): + return self.enbase(self.encode(n), min_length) + def decode_url(self, n): + return self.decode(self.debase(n)) + def encode(self, n): + return (n & ~self.mask) | self._encode(n & self.mask) + def _encode(self, n): + result = 0 + for i, b in enumerate(self.mapping): + if n & (1 << i): + result |= (1 << b) + return result + def decode(self, n): + return (n & ~self.mask) | self._decode(n & self.mask) + def _decode(self, n): + result = 0 + for i, b in enumerate(self.mapping): + if n & (1 << b): + result |= (1 << i) + return result + def enbase(self, x, min_length=0): + result = self._enbase(x) + padding = self.alphabet[0] * (min_length - len(result)) + return '%s%s' % (padding, result) + def _enbase(self, x): + n = len(self.alphabet) + if x < n: + return self.alphabet[x] + return self.enbase(x/n) + self.alphabet[x%n] + def debase(self, x): + n = len(self.alphabet) + result = 0 + for i, c in enumerate(reversed(x)): + result += self.alphabet.index(c) * (n**i) + return result + +DEFAULT_ENCODER = Encoder() + +def encode(n): + return DEFAULT_ENCODER.encode(n) + +def decode(n): + return DEFAULT_ENCODER.decode(n) + +def enbase(n, min_length=0): + return DEFAULT_ENCODER.enbase(n, min_length) + +def debase(n): + return DEFAULT_ENCODER.debase(n) + +def encode_url(n, min_length=0): + return DEFAULT_ENCODER.encode_url(n, min_length) + +def decode_url(n): + return DEFAULT_ENCODER.decode_url(n) + +def to_key(n): + return enbase(encode(n)) + +def from_key(n): + return decode(debase(n)) diff --git a/nebu/lib/error_logging.py b/nebu/lib/error_logging.py new file mode 100644 index 0000000..dbf927b --- /dev/null +++ b/nebu/lib/error_logging.py @@ -0,0 +1,13 @@ +import traceback +from lib import flaptor_logging +from django.http import HttpResponse + +logger = flaptor_logging.get_logger('error_logging') + +class ViewErrorLoggingMiddleware: + + def process_view(self, request, view_func, view_args, view_kwargs): + self.view_name = view_func.__name__ + def process_exception(self, request, exception): + logger.error('UNEXPECTED EXCEPTION in view "%s". Exception is: %s', self.view_name, repr(traceback.print_exc())) + return HttpResponse('{"status":"ERROR", "message":"Unexpected error."}') diff --git a/nebu/lib/exceptions.py b/nebu/lib/exceptions.py new file mode 100644 index 0000000..c6ff0a7 --- /dev/null +++ b/nebu/lib/exceptions.py @@ -0,0 +1,8 @@ + + +class CloudException(Exception): + pass + +class NoIndexerException(CloudException): + pass + diff --git a/nebu/lib/flaptor_logging.py b/nebu/lib/flaptor_logging.py new file mode 100644 index 0000000..1af893c --- /dev/null +++ b/nebu/lib/flaptor_logging.py @@ -0,0 +1,100 @@ +import logging as pylogging +from logging import config +import os + +usingNativeLogger = True + +__loggers = {} + + + +def get_logger(name, force_new=False): + '''Get the Logger instance for a given name''' + global __loggers + if __loggers is None: + __loggers = {} + if force_new: + return pylogging.getLogger(name) + if not __loggers.has_key(name): + __loggers[name] = pylogging.getLogger(name) + return __loggers[name] + +class SpecialFormatter(pylogging.Formatter): + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) + RESET_SEQ = "\033[0m" + COLOR_SEQ = "\033[37;4%dm" + PIDCOLOR_SEQ = "\033[1;3%dm" + BOLD_SEQ = "\033[1m" + COLORS = { + 'WARN': YELLOW, + 'INFO': GREEN, + 'DEBU': BLUE, + 'CRIT': RED, + 'ERRO': RED + } + + def __init__(self, *args, **kwargs): + pylogging.Formatter.__init__(self, *args, **kwargs) + def format(self, record): + if not hasattr(record, 'prefix'): record.prefix = '' + if not hasattr(record, 'suffix'): record.suffix = '' + if not hasattr(record, 'compname'): record.compname = '' + record.pid = os.getpid() + + record.levelname = record.levelname[:4] + + r = pylogging.Formatter.format(self, record) + if record.levelname in SpecialFormatter.COLORS: + levelcolor = SpecialFormatter.COLOR_SEQ % (SpecialFormatter.COLORS[record.levelname]) + r = r.replace('$LEVELCOLOR', levelcolor) + r = r.replace('$RESET', SpecialFormatter.RESET_SEQ) + else: + r = r.replace('$COLOR', '') + r = r.replace('$RESET', '') + pidcolor = SpecialFormatter.COLOR_SEQ % (1 + (record.pid % 5)) + r = r.replace('$PIDCOLOR', pidcolor) + r = r.replace('$BOLD', SpecialFormatter.BOLD_SEQ) + return r + +pylogging.SpecialFormatter = SpecialFormatter + +if usingNativeLogger: + try: + config.fileConfig('logging.conf') + except Exception, e: + print e + +#class NativePythonLogger: +# def __init__(self, name): +# '''Creates a new Logger for the given name. +# Do not call this method directly, instead use +# get_logger(name) to get the appropriate instance''' +# self.name = name +# self.__logger = pylogging.getLogger(name) +# #self.updateLevel(5) +# +# def updateLevel(self, level): +# self.__level = level +# if level == 1: +# self.__logger.setLevel(pylogging.CRITICAL) +# elif level == 2: +# self.__logger.setLevel(pylogging.INFO) +# elif level == 3: +# self.__logger.setLevel(pylogging.WARNING) +# elif level == 4: +# self.__logger.setLevel(pylogging.INFO) +# elif level == 5: +# self.__logger.setLevel(pylogging.DEBUG) +# +# def debug(self, format_str, *values): +# self.__logger.debug(format_str, *values) +# def info(self, format_str, *values): +# self.__logger.info(format_str, *values) +# def warn(self, format_str, *values): +# self.__logger.warn(format_str, *values) +# def error(self, format_str, *values): +# self.__logger.error(format_str, *values) +# def exception(self, format_str, *values): +# self.__logger.exception(format_str, *values) +# def fatal(self, format_str, *values): +# self.__logger.critical(format_str, *values) diff --git a/nebu/lib/mail.py b/nebu/lib/mail.py new file mode 100644 index 0000000..b1d3a08 --- /dev/null +++ b/nebu/lib/mail.py @@ -0,0 +1,101 @@ +from django.core.mail import send_mail + +try: + ENV=open('/data/env.name').readline().strip() +except Exception: + ENV='PROD+EXC' + +def _no_fail(method, *args, **kwargs): + def decorated(*args, **kwargs): + try: + return method(*args, **kwargs) + except Exception, e: + print e + return + return decorated + + + +@_no_fail +def report_payment_data(account): + activity_report = 'An Account has entered payment data\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + account.package.name + '\n' + activity_report += 'Email: ' + account.user.email + '\n' + activity_report += 'API KEY: ' + account.apikey + '' + + report_activity('Payment Data for ' + account.package.name + ' Account (' + account.user.email + ')', activity_report) + +@_no_fail +def report_new_account(account): + activity_report = 'A new Account was created\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + account.package.name + '\n' + activity_report += 'Email: ' + account.user.email + '\n' + activity_report += 'API KEY: ' + account.apikey + '' + + report_activity('New ' + account.package.name + ' Account (' + account.user.email + ')', activity_report) + +@_no_fail +def report_new_index(index): + activity_report = 'A new Index was created\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + index.account.package.name + '\n' + activity_report += 'User Email: ' + index.account.user.email + '\n' + activity_report += 'Index Name: ' + index.name + '' + + report_activity('Index activity (' + index.code + ')', activity_report, 'l') + +@_no_fail +def report_new_deploy(deploy): + activity_report = 'A new Deploy is now controllable\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + deploy.index.account.package.name + '\n' + activity_report += 'User Email: ' + deploy.index.account.user.email + '\n' + activity_report += 'Index Name: ' + deploy.index.name + '\n' + activity_report += 'Worker: #' + str(deploy.worker.id) + '\n' + activity_report += ('Deploy: %r' % deploy) + '\n' + activity_report += ('Container Index: %r' % deploy.index) + '\n' + + report_activity('Index activity (' + deploy.index.code + ')', activity_report, 'l') + +@_no_fail +def report_delete_index(index): + activity_report = 'An Index has been deleted\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + index.account.package.name + '\n' + activity_report += 'User Email: ' + index.account.user.email + '\n' + activity_report += 'Index Name: ' + index.name + '\n' + + report_activity('Index activity (' + index.code + ')', activity_report, 'l') + +@_no_fail +def report_new_worker(worker): + activity_report = 'A new Worker was created\n' + activity_report += '---------------------------\n' + activity_report += repr(worker) + + report_activity('New Worker (%d)' % (worker.pk), activity_report, 't') + +@_no_fail +def report_automatic_redeploy(deploy, initial_xmx, new_xmx): + activity_report = 'Automatic redeploy.\n' + activity_report += '---------------------------\n' + activity_report += 'initial xmx value: %d\n' % (initial_xmx) + activity_report += 'new xmx value: %d\n' % (new_xmx) + activity_report += repr(deploy) + + report_activity('Automatic redeploy', activity_report, 't') + +@_no_fail +def report_activity(subject, body, type='b'): + if type == 'b': + mail_to = 'activity@indextank.com' + elif type == 't': + mail_to = 'activitytech@indextank.com' + elif type == 'l': + mail_to = 'lowactivity@indextank.com' + else: + raise Exception('Wrong report type') + + send_mail(ENV + ' - ' + subject, body, 'IndexTank Activity ', [mail_to], fail_silently=False) diff --git a/nebu/lib/monitor.py b/nebu/lib/monitor.py new file mode 100644 index 0000000..3fe9818 --- /dev/null +++ b/nebu/lib/monitor.py @@ -0,0 +1,148 @@ + +from threading import Thread +from traceback import format_tb +import time, datetime +import sys +import shelve + +from django.core.mail import send_mail + +from lib import flaptor_logging + +try: + ENV=open('/data/env.name').readline().strip() +except Exception: + ENV='PROD+EXC' + +#helper functions +def is_prod(): + return ENV == 'PROD' or ENV == 'QoS_Monitor' + +def env_name(): + if ENV == 'PROD': + return 'PRODUCTION' + elif ENV == 'QoS_Monitor': + return 'QoS_Monitor' + else: + return ENV + +class Monitor(Thread): + def __init__(self, pagerduty_email='api-monitor@flaptor.pagerduty.com'): + super(Monitor, self).__init__() + self.name = self.__class__.__name__ + self.statuses = shelve.open('/data/monitor-%s.shelf' % self.name) + self.logger = flaptor_logging.get_logger(self.name) + self.failure_threshold = 1 + self.fatal_failure_threshold = 0 + self.severity = 'WARNING' + self.title_template = '%s::%s: [%s] %s' + self.pagerduty_email = pagerduty_email + + def iterable(self): + return [None] + + def run(self): + self.step = 1 + while True: + starttime = int(time.time()) + try: + self.logger.info("running cycle %d", self.step) + for object in self.iterable(): + self._monitor(object) + self.report_ok("unexpected error in monitor cycle") + self.clean() + except Exception: + self.logger.exception("Unexpected error while executing cycle") + self.report_bad("unexpected error in monitor cycle", 1, 0, 'UNEXPECTED ERROR IN THE CYCLE OF %s\n\n%s' % (self.name, self.describe_error())) + self.step += 1 + self.statuses.sync() + time.sleep(max(0, self.period - (int(time.time()) - starttime))) + + def clean(self): + for title, status in self.statuses.items(): + if not status['working']: + if status['last_update'] != self.step: + self.report_ok(title) + else: + del self.statuses[title] + + + def _monitor(self, object): + try: + if self.monitor(object): + self.report_ok(str(self.alert_title(object))) + else: + self.report_bad(str(self.alert_title(object)), self.failure_threshold, self.fatal_failure_threshold, self.alert_msg(object)) + self.report_ok("unexpected error in monitor") + except Exception, e: + self.logger.exception("Unexpected error while executing monitor. Exception is: %s" % (e)) + message = 'UNEXPECTED ERROR IN THE MONITORING OF %s FOR TITLE: %s\n\n%s' % (self.name, self.alert_title(object), self.describe_error()) + self.report_bad("unexpected error in monitor", 1, 'WARNING', message) + + def describe_error(self): + exc_type, exc_value, exc_traceback = sys.exc_info() + return 'EXCEPTION: %s : %s\ntraceback:\n%s' % (exc_type, exc_value, ''.join(format_tb(exc_traceback))) + + def update_status(self, key, **kwargs): + self.statuses[key] = kwargs + + def send_alert(self, title, message, severity): + try: + if is_prod(): + if severity == 'FATAL': + name = 'FATAL ALERT (%s)' % env_name() + else: + name = 'ALERT (%s)' % env_name() + else: + name = '%s test alert' % ENV + + title = self.title_template % (ENV, self.name, severity, title) + message += '\n\n--------SENT AT ' + str(datetime.datetime.now()) + to = ['alerts@indextank.com'] + if severity == 'FATAL' and is_prod(): + to.append('alerts+fatal@indextank.com') + to.append(self.pagerduty_email) + send_mail(title, message, '"%s" ' % name, to, fail_silently=False) + self.logger.info('Sending alert for title: %s\n============\n%s', title, message) + except Exception, e: + self.logger.exception("Unexpected error while sending alerts. Exception is: %s" % (e)) + + def report_ok(self, title): + if title in self.statuses and not self.statuses[title]['working'] and (self.statuses[title]['alerted'] or self.statuses[title]['alerted_fatal']): + # it has just been resolved + self.send_alert(title, 'The problem is no longer reported. The last message was:\n %s' % (self.statuses[title]['message']), self.severity) + if title in self.statuses: + del self.statuses[title] + + def report_bad(self, title, threshold, fatal_threshold, message): + if title in self.statuses and not self.statuses[title]['working']: + # this object had already failed, let's grab the first step in which it failed + first_failure = self.statuses[title]['first_failure'] + has_alerted = self.statuses[title]['alerted'] + has_alerted_fatal = self.statuses[title]['alerted_fatal'] + else: + # this object was fine, first failure is now + first_failure = self.step + has_alerted = False + has_alerted_fatal = False + + + should_alert = self.step - first_failure + 1 >= threshold + should_alert_fatal = fatal_threshold > 0 and self.step - first_failure + 1 >= fatal_threshold + + if should_alert_fatal: + if not has_alerted_fatal: + has_alerted_fatal = True + if is_prod(): + self.send_alert(title, 'A new problem on IndexTank has been detected:\n %s' % (message), 'FATAL') + else: + self.logger.info('Fatal error was found but alert has already been sent') + elif should_alert: + if not has_alerted: + has_alerted = True + self.send_alert(title, 'A new problem on IndexTank has been detected:\n %s' % (message), self.severity) + else: + self.logger.info('Error was found but alert has already been sent') + + # save current state of the object (is_failed, message, first_failure, last_update) + self.update_status(title, working=False, last_update=self.step, message=message, first_failure=first_failure, alerted=has_alerted, alerted_fatal=has_alerted_fatal) diff --git a/nebu/log4j.properties b/nebu/log4j.properties new file mode 100644 index 0000000..491a9cd --- /dev/null +++ b/nebu/log4j.properties @@ -0,0 +1,16 @@ +log4j.rootLogger=INFO, console +log4j.logger.org.apache = ERROR + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%-5p [%t] %c - [%m] %d{ISO8601}%n + +# BEGIN APPENDER: ROLLING FILE APPENDER (rolling) +log4j.appender.rolling=org.apache.log4j.RollingFileAppender +log4j.appender.rolling.File=logs/indexengine.log +log4j.appender.rolling.MaxFileSize=100MB +log4j.appender.rolling.MaxBackupIndex=1 +log4j.appender.rolling.layout=org.apache.log4j.PatternLayout +log4j.appender.rolling.layout.ConversionPattern=%-5p [%t] %c - [%m] %d{ISO8601}%n +# END APPENDER: ROLLING FILE APPENDER (rolling) + diff --git a/nebu/logging.conf b/nebu/logging.conf new file mode 100644 index 0000000..c09a3b7 --- /dev/null +++ b/nebu/logging.conf @@ -0,0 +1,35 @@ +[loggers] +keys=root,rpc,boto + +[handlers] +keys=consoleHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler + +[logger_rpc] +level=INFO +handlers=consoleHandler +propagate=0 +qualname=RPC + +[logger_boto] +level=INFO +handlers=consoleHandler +propagate=0 +qualname=boto + +[handler_consoleHandler] +class=StreamHandler +formatter=simpleFormatter +args=(sys.stdout,) + +[formatter_simpleFormatter] +format=%(pid)+5s %(asctime)s %(name)+8.8s:%(levelname)s%(prefix)s %(message)-90s %(suffix)s@%(filename)s:%(lineno)s +datefmt=%d/%m-%H.%M.%S +class=logging.SpecialFormatter + diff --git a/nebu/manage.py b/nebu/manage.py new file mode 100644 index 0000000..5e78ea9 --- /dev/null +++ b/nebu/manage.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +from django.core.management import execute_manager +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) + sys.exit(1) + +if __name__ == "__main__": + execute_manager(settings) diff --git a/nebu/models.py b/nebu/models.py new file mode 100644 index 0000000..ad66945 --- /dev/null +++ b/nebu/models.py @@ -0,0 +1 @@ +from api_linked_models import * #@UnusedWildImport diff --git a/nebu/populator.py b/nebu/populator.py new file mode 100644 index 0000000..85c036b --- /dev/null +++ b/nebu/populator.py @@ -0,0 +1,108 @@ +from lib.monitor import Monitor +from nebu.models import Index, IndexPopulation +from lib.indextank.client import ApiClient +from django.utils import simplejson as json + +from lib import flaptor_logging + +batch_size = 1000 +dataset_files_path = './' + +logger = flaptor_logging.get_logger('Populator') + +class IndexPopulator(Monitor): + + def __init__(self): + super(IndexPopulator, self).__init__() + self.failure_threshold = 5 + self.fatal_failure_threshold = 20 + self.period = 30 + + def iterable(self): + return IndexPopulation.objects.exclude(status=IndexPopulation.Statuses.finished) + + def monitor(self, population): + client = ApiClient(population.index.account.get_private_apiurl()) + index = client.get_index(population.index.name) + + if index.has_started(): + if population.status == IndexPopulation.Statuses.created: + logger.info('Populating index ' + population.index.code + ' with dataset "' + population.dataset.name + '"') + + population.status = IndexPopulation.Statuses.populating + population.populated_size = 0 + population.save() + + eof, documents_added = self._populate_batch(population.index, population.dataset, population.populated_size, batch_size) + population.populated_size = population.populated_size + documents_added + + if eof: + population.status = IndexPopulation.Statuses.finished + + population.save() + + return True + + ''' + line_from: zero based + ''' + def _populate_batch(self, index, dataset, line_from, lines): + logger.info('Adding batch from line ' + str(line_from)) + + client = ApiClient(index.account.get_private_apiurl()) + index = client.get_index(index.name) + + try: + dataset_file = open(dataset_files_path + dataset.filename, 'r') + for i in range(line_from): + dataset_file.readline() + except Exception, e: + logger.error('Failed processing dataset file: %s', e) + return False, 0 + + + added_docs = 0 + eof = False + + for i in xrange(lines): + try: + line = dataset_file.readline() + except Exception, e: + logger.error('Failed processing dataset file: %s', e) + return False, added_docs + + if not line: + break + + try: + document = json.loads(line) + id = document['docid'] + del document['docid'] + + fields = document['fields'] + variables = document.get('variables', {}) + + added_docs += 1 + index.add_document(id, fields) + + categories = document.get('categories') + if categories: + index.update_categories(id, categories) + except Exception, e: + logger.error('Failed processing dataset line: %s', e) + return False, added_docs + + eof = not dataset_file.readline() + + logger.info('Added ' + str(added_docs) + ' lines') + + return eof, added_docs + + def alert_title(self, deploy): + return '' + + def alert_msg(self, deploy): + return '' + +if __name__ == '__main__': + IndexPopulator().start() diff --git a/nebu/rpc.py b/nebu/rpc.py new file mode 100644 index 0000000..e03a661 --- /dev/null +++ b/nebu/rpc.py @@ -0,0 +1 @@ +from api_linked_rpc import * #@UnusedWildImport diff --git a/nebu/run_controller.sh b/nebu/run_controller.sh new file mode 100755 index 0000000..f89eea0 --- /dev/null +++ b/nebu/run_controller.sh @@ -0,0 +1,5 @@ +echo "WARNING: THIS SCRIPT IS FOR TESTING ONLY" +export DJANGO_SETTINGS_MODULE=settings +export PYTHONPATH=../:. + +python controller.py diff --git a/nebu/run_deploy_manager.sh b/nebu/run_deploy_manager.sh new file mode 100755 index 0000000..02705d5 --- /dev/null +++ b/nebu/run_deploy_manager.sh @@ -0,0 +1,5 @@ +echo "WARNING: THIS SCRIPT IS FOR TESTING ONLY" +export DJANGO_SETTINGS_MODULE=settings +export PYTHONPATH=../:. + +python deploy_manager.py diff --git a/nebu/run_populator.sh b/nebu/run_populator.sh new file mode 100755 index 0000000..c818d07 --- /dev/null +++ b/nebu/run_populator.sh @@ -0,0 +1,5 @@ +echo "WARNING: THIS SCRIPT IS FOR TESTING ONLY" +export DJANGO_SETTINGS_MODULE=settings +export PYTHONPATH=../:. + +python populator.py diff --git a/nebu/run_supervisor.sh b/nebu/run_supervisor.sh new file mode 100755 index 0000000..c9fc35e --- /dev/null +++ b/nebu/run_supervisor.sh @@ -0,0 +1,5 @@ +echo "WARNING: THIS SCRIPT IS FOR TESTING ONLY" +export DJANGO_SETTINGS_MODULE=settings +export PYTHONPATH=../:. + +python supervisor.py diff --git a/nebu/settings.py b/nebu/settings.py new file mode 100644 index 0000000..07db415 --- /dev/null +++ b/nebu/settings.py @@ -0,0 +1,74 @@ +# Django settings for burbio project. + +DEBUG = False +TEMPLATE_DEBUG = DEBUG + +ADMINS = ( + # ('Your Name', 'your_email@domain.com'), +) + +MANAGERS = ADMINS + +DATABASE_ENGINE = 'mysql' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'ado_mssql'. +DATABASE_NAME = 'indextank' # Or path to database file if using sqlite3. +DATABASE_USER = '****' # Not used with sqlite3. +DATABASE_PASSWORD = '****' # Not used with sqlite3. +DATABASE_HOST = 'database' # Set to empty string for localhost. Not used with sqlite3. +DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'Etc/GMT+0' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.load_template_source', + 'django.template.loaders.app_directories.load_template_source', +# 'django.template.loaders.eggs.load_template_source', +) + +MIDDLEWARE_CLASSES = ( + 'api.error_logging.ViewErrorLoggingMiddleware', + 'django.middleware.gzip.GZipMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.locale.LocaleMiddleware', +) + +ROOT_URLCONF = 'api.urls' + +TEMPLATE_DIRS = ( + 'templates' + # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +INSTALLED_APPS = ( + 'nebu', +) + + + +STATIC_URLS = [ '/_static' ] +ALLOWED_INCLUDE_ROOTS = ('static') + +USER_COOKIE_NAME = "pf_user" +COMMON_DOMAIN = 'localhost' +#SESSION_COOKIE_DOMAIN = COMMON_DOMAIN +FORCE_SCRIPT_NAME = '' +USE_MULTITHREADED_SERVER = True +LOGGER_CONFIG_FILE='logging.conf' + +EMAIL_HOST='localhost' +EMAIL_PORT=25 +EMAIL_HOST_USER='user%localhost' +EMAIL_HOST_PASSWORD='****' diff --git a/nebu/startIndex.py b/nebu/startIndex.py new file mode 100755 index 0000000..bdd60b2 --- /dev/null +++ b/nebu/startIndex.py @@ -0,0 +1,135 @@ +#!/usr/bin/python +import os +import sys +import subprocess +import simplejson as json + + +# prevent everyone from importing +if __name__ != "__main__": + print >> sys.stderr, 'this script is not meant to be imported. Exiting' + sys.exit(1) + + +# ok, good to go +# parse the environment +if os.path.exists('/data/env.name'): + env = open('/data/env.name').readline().rstrip("\n") +else: + env = 'PROD' + print >>sys.stderr, 'WARNING, the instance does not have a /data/env.name. USING PROD AS ENV' + + + +# parse json +config_file = sys.argv[1] +config = json.load(open(config_file)) + +# setup logging files +log_dir = 'logs' +log_file = "%s/indextank.log" % log_dir +gc_log_file = "%s/gc.log" % log_dir +os.makedirs(log_dir) +pid_file = open('pid', 'w') + +# setup index directory +index_dir = 'index' +os.makedirs(index_dir) + +# first, generate vm args +vm_args = config.get('vmargs',[]) +vm_args.append('-cp') +vm_args.append('.:lib/indextank-trunk.jar:lib/indextank-trunk-deps.jar') +vm_args.append('-verbose:gc') +vm_args.append('-Xloggc:%s'%gc_log_file) +vm_args.append('-XX:+PrintGCTimeStamps') +vm_args.append("-XX:+UseConcMarkSweepGC") +vm_args.append("-XX:+UseParNewGC") +vm_args.append("-Dorg.apache.lucene.FSDirectory.class=org.apache.lucene.store.MMapDirectory") +vm_args.append("-Dapp=INDEX-ENGINE-%s"%config['index_code']) +vm_args.append("-Xmx%sM"%config['xmx']) + + +# a list of possible IndexEngine params +app_params_mapping = {} +app_params_mapping['base_port'] = "--port" +app_params_mapping['index_code'] = "--index-code" +app_params_mapping['allow_snippets'] = "--snippets" # we may remove this line +app_params_mapping['allows_snippets'] = "--snippets" +app_params_mapping['snippets'] = "--snippets" +app_params_mapping['allow_facets'] = "--facets" # we may remove this line +app_params_mapping['allows_facets'] = "--facets" +app_params_mapping['max_variables'] = "--boosts" +app_params_mapping['autocomplete'] = "--suggest" +app_params_mapping['didyoumean'] = "--didyoumean" +app_params_mapping['rti_size'] = "--rti-size" +app_params_mapping['functions'] = "--functions" +app_params_mapping['storage'] = "--storage" +app_params_mapping['bdb_cache'] = "--bdb-cache" + + +def adapt_config_kv(key, value, json_config): + if key == '--suggest': + if value: + value = json_config.get('autocomplete_type', 'documents') + else: + key, value = '', '' + if key == '--facets': + if value: + value = '' + else: + key, value = '', '' + if key == '--snippets': + if value: + value = '' + else: + key, value = '', '' + if key == '--didyoumean' and value == True: + value = "" # didyoumean does not take arguments + if key == '--functions': + value = "|".join("%s:%s"%(k,v) for k,v in value.iteritems()) + return key, value + + +# now, app params +app_params = {} + +app_params['--conf-file'] = config_file + +app_params.update(adapt_config_kv(app_params_mapping.get(k),v, config) for k,v in config.iteritems() if k in app_params_mapping) + + +# dependency checking +# --didyoumean needs --suggest documents +if '--didyoumean' in app_params: + app_params['--suggest'] == 'documents' + +# just make sure to use index dir +app_params['--dir'] = index_dir +app_params['--environment-prefix'] = config.get('environment',env) # let configuration override env. +app_params['--recover'] = '' # recover does not take arguments + + +# We want to use everything 64 bits because mmap doesn't like things it can't map to 31 bits. +java = '/opt/java/64/jre/bin/java' +os.putenv('LD_LIBRARY_PATH','lib/berkeleydb_libs_64/') + + +full_args = ['nohup', java] +full_args.extend(vm_args) +full_args.append('com.flaptor.indextank.index.IndexEngine') +for k,v in app_params.iteritems(): + full_args.extend((k,str(v))) +log_file = open(log_file, 'w') + + +print >>log_file, "" +print >>log_file, "" +print >>log_file, "="*80 +print >>log_file, "%s STARTING JVM %s" % ( "="*33, "="*33) +print >>log_file, "="*80 +print >>log_file, "== %s ==" % full_args +print >>log_file, "="*80 +pid = subprocess.Popen(full_args, stderr=log_file, stdout=log_file, close_fds=True) +print >>pid_file, pid.pid + diff --git a/nebu/supervisor.py b/nebu/supervisor.py new file mode 100644 index 0000000..62a246a --- /dev/null +++ b/nebu/supervisor.py @@ -0,0 +1,272 @@ +#!/usr/bin/python + +import systemutils + +import rpc +from lib import flaptor_logging +from lib.monitor import Monitor + +from nebu.models import Index, Worker, Deploy +from flaptor.indextank.rpc.ttypes import IndexerStatus, NebuException +import datetime + + +class DeployPingMonitor(Monitor): + def __init__(self): + super(DeployPingMonitor, self).__init__(pagerduty_email='index-monitor@flaptor.pagerduty.com') + self.failure_threshold = 5 + self.fatal_failure_threshold = 20 + self.period = 30 + + def iterable(self): + return (d for d in Deploy.objects.all().select_related('index') if d.is_writable()) + + def monitor(self, deploy): + deploy.index # so that it fails if the index foreign key is broken + try: + client = rpc.getThriftIndexerClient(deploy.worker.lan_dns, int(deploy.base_port), 5000) + client.ping() + return True + except Exception: + self.logger.exception("Failed to ping deploy %s for index %s", deploy.id, deploy.index.code) + self.err_msg = self.describe_error() + return False + #'A writable deploy [%d] is failing to answer to ping.\n\n%s\n\nEXCEPTION: %s : %s\ntraceback:\n%s' % (deploy.id, index.get_debug_info(), exc_type, exc_value, ''.join(format_tb(exc_traceback))) + + def alert_title(self, deploy): + return 'Unable to ping index %s deploy id %d' % (deploy.index.code, deploy.id) + + def alert_msg(self, deploy): + return 'A writable deploy [%d] is failing to answer to ping.\n\n%s\n\n%s' % (deploy.id, deploy.index.get_debug_info(), self.err_msg) + +class IndexSizeMonitor(Monitor): + def __init__(self): + super(IndexSizeMonitor, self).__init__(pagerduty_email='index-monitor@flaptor.pagerduty.com') + self.failure_threshold = 2 + self.fatal_failure_threshold = 5 + self.period = 120 + + def iterable(self): + return (i for i in Index.objects.all() if i.is_ready() and not i.deleted) + + def monitor(self, index): + try: + self.logger.debug("Fetching size for index %s" , index.code) + searcher = rpc.get_searcher_client(index, 10000) + current_size = searcher.size() + Index.objects.filter(id=index.id).update(current_docs_number=current_size) + self.logger.info("Updated size for index %s: %d" , index.code, index.current_docs_number) + return True + except Exception: + self.logger.exception("Failed to update size for index %s" , index.code) + self.err_msg = self.describe_error() + return False + + def alert_title(self, index): + return "Failed to fetch size for index %s" % index.code + + def alert_msg(self, index): + return 'An IndexEngine is failing when attempting to query its size via thrift.\n\n%s\n\n%s' % (index.get_debug_info(), self.err_msg) + +class ServiceDeploys(Monitor): + def __init__(self): + super(ServiceDeploys, self).__init__(pagerduty_email='index-monitor@flaptor.pagerduty.com') + self.failure_threshold = 1 + self.period = 2 + + def monitor(self, object): + try: + rpc.get_deploy_manager().service_deploys() + return True + except NebuException, e: + self.nebu_e = e + return False + + def alert_title(self, object): + return "Nebu exception" + + def alert_msg(self, object): + return self.nebu_e.message + +class ServiceWorkers(Monitor): + def __init__(self): + super(ServiceWorkers, self).__init__(pagerduty_email='index-monitor@flaptor.pagerduty.com') + self.failure_threshold = 1 + self.period = 30 + + def iterable(self): + return (w for w in Worker.objects.all() if not w.is_ready()) + + def monitor(self, worker): + try: + rpc.getThriftWorkerManagerClient('workermanager').update_status(worker.instance_name) + return True + except NebuException, e: + self.nebu_e = e + return False + + def alert_title(self, worker): + return "Nebu exception for worker id %d" % worker.id + + def alert_msg(self, worker): + return "INFO ABOUT THE WORKER\ninstance id: %s\nwan_dns: %s\nlan_dns: %s\n\nError message: " % (worker.instance_name, worker.wan_dns, worker.lan_dns, self.nebu_e.message) + +class WorkerFreeDiskMonitor(Monitor): + def __init__(self, threshold): + super(WorkerFreeDiskMonitor, self).__init__(pagerduty_email='index-monitor@flaptor.pagerduty.com') + self.failure_threshold = 1 + self.period = 60 + self.threshold = threshold + + def get_fs_sizes(self, worker): + controller = rpc.get_worker_controller(worker) + worker_stats = controller.get_worker_mount_stats() + return worker_stats.fs_sizes.items() + + def iterable(self): + # generates a list of pairs worker,filesystem with each filesystem in each worker + return [(w,fs) for w in Worker.objects.all() for fs in self.get_fs_sizes(w) if w.is_ready()] + + def monitor(self, info): + worker, (fs, (used, available)) = info + self.logger.debug('Checking free space on %s for worker %s', fs, worker.wan_dns) + + ratio = float(available) / (available + used) + return ratio * 100 > self.threshold + + def alert_title(self, info): + worker, (fs, _) = info + return 'Filesystem %s free space below %d%% for worker id %d' % (fs, self.threshold, worker.id) + + def alert_msg(self, info): + worker, (fs, (used, available)) = info + ratio = float(available) / (available + used) + return 'Worker %d\nFilesystem mounted on %s has only %d%% of available space (%d free of %d)\n\nINFO ABOUT THE WORKER\ninstance id: %s\nwan_dns: %s\nlan_dns: %s' % (worker.id, fs, (ratio * 100), available, used, worker.instance_name, worker.wan_dns, worker.lan_dns) + +class FrontendFreeDiskMonitor(Monitor): + def __init__(self, threshold): + super(FrontendFreeDiskMonitor, self).__init__(pagerduty_email='index-monitor@flaptor.pagerduty.com') + self.failure_threshold = 1 + self.period = 60 + self.threshold = threshold + + + def iterable(self): + # generates a list of pairs worker,filesystem with each filesystem in each worker + return [fs for fs in systemutils.get_available_sizes().items()] + + def monitor(self, info): + fs, (used, available) = info + self.logger.debug('Checking free space on %s for the frontend', fs) + + ratio = float(available) / (available + used) + return ratio * 100 > self.threshold + + def alert_title(self, info): + fs, _ = info + return 'Filesystem %s free space below %d%% for FRONTEND machine' % (fs, self.threshold) + + def alert_msg(self, info): + fs, (used, available) = info + ratio = float(available) / (available + used) + return 'Frontend\nFilesystem mounted on %s has only %d%% of available space (%d free of %d)' % (fs, (ratio * 100), available, used) + +class IndexStartedMonitor(Monitor): + def __init__(self): + super(IndexStartedMonitor, self).__init__(pagerduty_email='index-monitor@flaptor.pagerduty.com') + self.failure_threshold = 1 + self.period = 60 + + def iterable(self): + return (i for i in Index.objects.all() if not i.is_ready() and not i.is_hibernated() and not i.deleted) + + def monitor(self, index): + return datetime.datetime.now() - index.creation_time < datetime.timedelta(minutes=5) + + def alert_title(self, index): + return 'Index %s hasn\'t started in at least 5 minutes' % (index.code) + + def alert_msg(self, index): + return 'The following index hasn\'t started in more than 5 minutes:\n\n%s' % (index.get_debug_info()) + +class MoveIncompleteMonitor(Monitor): + def __init__(self): + super(MoveIncompleteMonitor, self).__init__(pagerduty_email='index-monitor@flaptor.pagerduty.com') + self.failure_threshold = 1 + self.period = 360 + + def iterable(self): + return Deploy.objects.filter(status=Deploy.States.moving) + + def monitor(self, deploy): + return datetime.datetime.now() - deploy.timestamp < datetime.timedelta(hours=4) + + def alert_title(self, deploy): + return 'Index %s has been moving for over 4 hours' % (deploy.index.code) + + def alert_msg(self, deploy): + return 'The following index has been moving for more than 4 hours:\n\n%s' % (deploy.index.get_debug_info()) + + +class RecoveryErrorMonitor(Monitor): + ''' + Pings RECOVERING indexes, to find out about recovery errors (status=3). + ''' + + def __init__(self): + super(RecoveryErrorMonitor, self).__init__(pagerduty_email='index-monitor@flaptor.pagerduty.com') + self.failure_threshold = 1 + self.period = 360 + + def iterable(self): + return Deploy.objects.filter(status=Deploy.States.recovering) + + def monitor(self, deploy): + # get the current recovery status + indexer = rpc.getThriftIndexerClient(deploy.worker.lan_dns, int(deploy.base_port), 10000) + indexer_status = indexer.getStatus() + + # complain only if an error arised + return indexer_status != IndexerStatus.error + + + def alert_title(self, deploy): + return 'Recovery failed for index %s' % (deploy.index.code) + + def alert_msg(self, deploy): + return 'The following index has a recovering deploy that failed:\n\n%s' % (deploy.index.get_debug_info()) + +class DeployInitializedMonitor(Monitor): + def __init__(self): + super(DeployInitializedMonitor, self).__init__(pagerduty_email='index-monitor@flaptor.pagerduty.com') + self.failure_threshold = 1 + self.period = 20 + + def iterable(self): + return Deploy.objects.filter(status=Deploy.States.initializing) + + def monitor(self, deploy): + return datetime.datetime.now() - deploy.timestamp < datetime.timedelta(seconds=20) + + def alert_title(self, deploy): + return 'Deploy %d has been initializing for over 20 seconds' % (deploy.id) + + def alert_msg(self, deploy): + return 'A deploy has been started more than 20 seconds ago (i.e. startIndex.sh has been executed) and it\'s still not responding to its thrift interface.\n\nDeploy id: %d\n\nIndex info:\n%s' % (deploy.id, deploy.index.get_debug_info()) + +if __name__ == '__main__': + DeployPingMonitor().start() + IndexSizeMonitor().start() + ServiceDeploys().start() + ServiceWorkers().start() + WorkerFreeDiskMonitor(15).start() + WorkerFreeDiskMonitor(10).start() + WorkerFreeDiskMonitor(5).start() + FrontendFreeDiskMonitor(15).start() + FrontendFreeDiskMonitor(10).start() + FrontendFreeDiskMonitor(5).start() + IndexStartedMonitor().start() + MoveIncompleteMonitor().start() + RecoveryErrorMonitor().start() + DeployInitializedMonitor().start() + diff --git a/nebu/systemutils.py b/nebu/systemutils.py new file mode 100644 index 0000000..c5d0cd8 --- /dev/null +++ b/nebu/systemutils.py @@ -0,0 +1,58 @@ +import re +import commands +import subprocess + +def _get_df_lines(): + lines = re.split('\n', commands.getstatusoutput('df')[1]) + for i in range(len(lines)): + lines[i] = re.split('\s+', lines[i]) + return lines + +def get_available_sizes(filesystems=None): + df = _get_df_lines() + results = {} + for fs in df: + if not fs[0] == 'Filesystem' and (filesystems is None or fs[0] in filesystems): + results[fs[5]] = [int(fs[2]), int(fs[3])] + return results + +def get_load_averages(): + return [float(x) for x in commands.getstatusoutput('uptime')[1].split('load average: ')[1].split(', ')] + +def get_index_stats(index_code, base_port): + stats = dict(disk=0) + + ps = subprocess.Popen('ps -e -orss=,pcpu=,args= | grep %s | grep -v grep' % index_code, shell=True, stdout=subprocess.PIPE) + + psout, _ = ps.communicate() + #psout = "36049 0.0 java -Dapp=INDEX-ENGINE-DD9r -Xmx700M -Dorg.apache.lucene.FSDirectory.class=org.apache.lucene.store.MMapDirectory -cp ../../conf:lib/indextank-trunk.jar:lib/indextank-trunk-deps.jar com.flaptor.indextank.index.IndexEngine -dir index -port 20130 -index-code DD9r -rti-size 500 -r -snippets" + for line in psout.split('\n'): + if line: + mem, cpu, args = line.split(None, 2) + print args + if args != index_code: + + extract = lambda l: l[0] if l else None + parse = lambda s: extract(re.findall(s, args)) + + stats.update(mem=(float(mem) / 1024), + cpu=(float(cpu)), + args=args, + xmx=parse(r'-Xmx(\d+\w)'), + port=parse(r'-port (\d+)'), + rti=parse(r'-rti-size (\d+)'), + suggest=parse(r'(-suggest)'), + snippets=parse(r'(-snippets)'), + boosts=parse(r'-b (\d+)') or 1) + break + + du = subprocess.Popen('du -s /data/indexes/%s-%s/index/' % (index_code, base_port), shell=True, stdout=subprocess.PIPE) + duout, _ = du.communicate() + if duout: + disk = duout.split()[0] + if disk: + disk = (float(disk) / 1024) + stats.update(disk=disk) + + return stats + diff --git a/nebu/thrift/TSCons.py b/nebu/thrift/TSCons.py new file mode 100644 index 0000000..2404625 --- /dev/null +++ b/nebu/thrift/TSCons.py @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from os import path +from SCons.Builder import Builder + +def scons_env(env, add=''): + opath = path.dirname(path.abspath('$TARGET')) + lstr = 'thrift --gen cpp -o ' + opath + ' ' + add + ' $SOURCE' + cppbuild = Builder(action = lstr) + env.Append(BUILDERS = {'ThriftCpp' : cppbuild}) + +def gen_cpp(env, dir, file): + scons_env(env) + suffixes = ['_types.h', '_types.cpp'] + targets = map(lambda s: 'gen-cpp/' + file + s, suffixes) + return env.ThriftCpp(targets, dir+file+'.thrift') diff --git a/nebu/thrift/TSerialization.py b/nebu/thrift/TSerialization.py new file mode 100644 index 0000000..b19f98a --- /dev/null +++ b/nebu/thrift/TSerialization.py @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from protocol import TBinaryProtocol +from transport import TTransport + +def serialize(thrift_object, protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()): + transport = TTransport.TMemoryBuffer() + protocol = protocol_factory.getProtocol(transport) + thrift_object.write(protocol) + return transport.getvalue() + +def deserialize(base, buf, protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()): + transport = TTransport.TMemoryBuffer(buf) + protocol = protocol_factory.getProtocol(transport) + base.read(protocol) + return base + diff --git a/nebu/thrift/Thrift.py b/nebu/thrift/Thrift.py new file mode 100644 index 0000000..91728a7 --- /dev/null +++ b/nebu/thrift/Thrift.py @@ -0,0 +1,133 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys + +class TType: + STOP = 0 + VOID = 1 + BOOL = 2 + BYTE = 3 + I08 = 3 + DOUBLE = 4 + I16 = 6 + I32 = 8 + I64 = 10 + STRING = 11 + UTF7 = 11 + STRUCT = 12 + MAP = 13 + SET = 14 + LIST = 15 + UTF8 = 16 + UTF16 = 17 + +class TMessageType: + CALL = 1 + REPLY = 2 + EXCEPTION = 3 + ONEWAY = 4 + +class TProcessor: + + """Base class for procsessor, which works on two streams.""" + + def process(iprot, oprot): + pass + +class TException(Exception): + + """Base class for all thrift exceptions.""" + + # BaseException.message is deprecated in Python v[2.6,3.0) + if (2,6,0) <= sys.version_info < (3,0): + def _get_message(self): + return self._message + def _set_message(self, message): + self._message = message + message = property(_get_message, _set_message) + + def __init__(self, message=None): + Exception.__init__(self, message) + self.message = message + +class TApplicationException(TException): + + """Application level thrift exceptions.""" + + UNKNOWN = 0 + UNKNOWN_METHOD = 1 + INVALID_MESSAGE_TYPE = 2 + WRONG_METHOD_NAME = 3 + BAD_SEQUENCE_ID = 4 + MISSING_RESULT = 5 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + + def __str__(self): + if self.message: + return self.message + elif self.type == self.UNKNOWN_METHOD: + return 'Unknown method' + elif self.type == self.INVALID_MESSAGE_TYPE: + return 'Invalid message type' + elif self.type == self.WRONG_METHOD_NAME: + return 'Wrong method name' + elif self.type == self.BAD_SEQUENCE_ID: + return 'Bad sequence ID' + elif self.type == self.MISSING_RESULT: + return 'Missing result' + else: + return 'Default (unknown) TApplicationException' + + def read(self, iprot): + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.message = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.type = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + oprot.writeStructBegin('TApplicationException') + if self.message != None: + oprot.writeFieldBegin('message', TType.STRING, 1) + oprot.writeString(self.message) + oprot.writeFieldEnd() + if self.type != None: + oprot.writeFieldBegin('type', TType.I32, 2) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() diff --git a/nebu/thrift/__init__.py b/nebu/thrift/__init__.py new file mode 100644 index 0000000..48d659c --- /dev/null +++ b/nebu/thrift/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['Thrift', 'TSCons'] diff --git a/nebu/thrift/protocol/TBinaryProtocol.py b/nebu/thrift/protocol/TBinaryProtocol.py new file mode 100644 index 0000000..50c6aa8 --- /dev/null +++ b/nebu/thrift/protocol/TBinaryProtocol.py @@ -0,0 +1,259 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TProtocol import * +from struct import pack, unpack + +class TBinaryProtocol(TProtocolBase): + + """Binary implementation of the Thrift protocol driver.""" + + # NastyHaxx. Python 2.4+ on 32-bit machines forces hex constants to be + # positive, converting this into a long. If we hardcode the int value + # instead it'll stay in 32 bit-land. + + # VERSION_MASK = 0xffff0000 + VERSION_MASK = -65536 + + # VERSION_1 = 0x80010000 + VERSION_1 = -2147418112 + + TYPE_MASK = 0x000000ff + + def __init__(self, trans, strictRead=False, strictWrite=True): + TProtocolBase.__init__(self, trans) + self.strictRead = strictRead + self.strictWrite = strictWrite + + def writeMessageBegin(self, name, type, seqid): + if self.strictWrite: + self.writeI32(TBinaryProtocol.VERSION_1 | type) + self.writeString(name) + self.writeI32(seqid) + else: + self.writeString(name) + self.writeByte(type) + self.writeI32(seqid) + + def writeMessageEnd(self): + pass + + def writeStructBegin(self, name): + pass + + def writeStructEnd(self): + pass + + def writeFieldBegin(self, name, type, id): + self.writeByte(type) + self.writeI16(id) + + def writeFieldEnd(self): + pass + + def writeFieldStop(self): + self.writeByte(TType.STOP); + + def writeMapBegin(self, ktype, vtype, size): + self.writeByte(ktype) + self.writeByte(vtype) + self.writeI32(size) + + def writeMapEnd(self): + pass + + def writeListBegin(self, etype, size): + self.writeByte(etype) + self.writeI32(size) + + def writeListEnd(self): + pass + + def writeSetBegin(self, etype, size): + self.writeByte(etype) + self.writeI32(size) + + def writeSetEnd(self): + pass + + def writeBool(self, bool): + if bool: + self.writeByte(1) + else: + self.writeByte(0) + + def writeByte(self, byte): + buff = pack("!b", byte) + self.trans.write(buff) + + def writeI16(self, i16): + buff = pack("!h", i16) + self.trans.write(buff) + + def writeI32(self, i32): + buff = pack("!i", i32) + self.trans.write(buff) + + def writeI64(self, i64): + buff = pack("!q", i64) + self.trans.write(buff) + + def writeDouble(self, dub): + buff = pack("!d", dub) + self.trans.write(buff) + + def writeString(self, str): + self.writeI32(len(str)) + self.trans.write(str) + + def readMessageBegin(self): + sz = self.readI32() + if sz < 0: + version = sz & TBinaryProtocol.VERSION_MASK + if version != TBinaryProtocol.VERSION_1: + raise TProtocolException(type=TProtocolException.BAD_VERSION, message='Bad version in readMessageBegin: %d' % (sz)) + type = sz & TBinaryProtocol.TYPE_MASK + name = self.readString() + seqid = self.readI32() + else: + if self.strictRead: + raise TProtocolException(type=TProtocolException.BAD_VERSION, message='No protocol version header') + name = self.trans.readAll(sz) + type = self.readByte() + seqid = self.readI32() + return (name, type, seqid) + + def readMessageEnd(self): + pass + + def readStructBegin(self): + pass + + def readStructEnd(self): + pass + + def readFieldBegin(self): + type = self.readByte() + if type == TType.STOP: + return (None, type, 0) + id = self.readI16() + return (None, type, id) + + def readFieldEnd(self): + pass + + def readMapBegin(self): + ktype = self.readByte() + vtype = self.readByte() + size = self.readI32() + return (ktype, vtype, size) + + def readMapEnd(self): + pass + + def readListBegin(self): + etype = self.readByte() + size = self.readI32() + return (etype, size) + + def readListEnd(self): + pass + + def readSetBegin(self): + etype = self.readByte() + size = self.readI32() + return (etype, size) + + def readSetEnd(self): + pass + + def readBool(self): + byte = self.readByte() + if byte == 0: + return False + return True + + def readByte(self): + buff = self.trans.readAll(1) + val, = unpack('!b', buff) + return val + + def readI16(self): + buff = self.trans.readAll(2) + val, = unpack('!h', buff) + return val + + def readI32(self): + buff = self.trans.readAll(4) + val, = unpack('!i', buff) + return val + + def readI64(self): + buff = self.trans.readAll(8) + val, = unpack('!q', buff) + return val + + def readDouble(self): + buff = self.trans.readAll(8) + val, = unpack('!d', buff) + return val + + def readString(self): + len = self.readI32() + str = self.trans.readAll(len) + return str + + +class TBinaryProtocolFactory: + def __init__(self, strictRead=False, strictWrite=True): + self.strictRead = strictRead + self.strictWrite = strictWrite + + def getProtocol(self, trans): + prot = TBinaryProtocol(trans, self.strictRead, self.strictWrite) + return prot + + +class TBinaryProtocolAccelerated(TBinaryProtocol): + + """C-Accelerated version of TBinaryProtocol. + + This class does not override any of TBinaryProtocol's methods, + but the generated code recognizes it directly and will call into + our C module to do the encoding, bypassing this object entirely. + We inherit from TBinaryProtocol so that the normal TBinaryProtocol + encoding can happen if the fastbinary module doesn't work for some + reason. (TODO(dreiss): Make this happen sanely in more cases.) + + In order to take advantage of the C module, just use + TBinaryProtocolAccelerated instead of TBinaryProtocol. + + NOTE: This code was contributed by an external developer. + The internal Thrift team has reviewed and tested it, + but we cannot guarantee that it is production-ready. + Please feel free to report bugs and/or success stories + to the public mailing list. + """ + + pass + + +class TBinaryProtocolAcceleratedFactory: + def getProtocol(self, trans): + return TBinaryProtocolAccelerated(trans) diff --git a/nebu/thrift/protocol/TCompactProtocol.py b/nebu/thrift/protocol/TCompactProtocol.py new file mode 100644 index 0000000..fbc156a --- /dev/null +++ b/nebu/thrift/protocol/TCompactProtocol.py @@ -0,0 +1,368 @@ +from TProtocol import * +from struct import pack, unpack + +__all__ = ['TCompactProtocol', 'TCompactProtocolFactory'] + +CLEAR = 0 +FIELD_WRITE = 1 +VALUE_WRITE = 2 +CONTAINER_WRITE = 3 +BOOL_WRITE = 4 +FIELD_READ = 5 +CONTAINER_READ = 6 +VALUE_READ = 7 +BOOL_READ = 8 + +def make_helper(v_from, container): + def helper(func): + def nested(self, *args, **kwargs): + assert self.state in (v_from, container), (self.state, v_from, container) + return func(self, *args, **kwargs) + return nested + return helper +writer = make_helper(VALUE_WRITE, CONTAINER_WRITE) +reader = make_helper(VALUE_READ, CONTAINER_READ) + +def makeZigZag(n, bits): + return (n << 1) ^ (n >> (bits - 1)) + +def fromZigZag(n): + return (n >> 1) ^ -(n & 1) + +def writeVarint(trans, n): + out = [] + while True: + if n & ~0x7f == 0: + out.append(n) + break + else: + out.append((n & 0xff) | 0x80) + n = n >> 7 + trans.write(''.join(map(chr, out))) + +def readVarint(trans): + result = 0 + shift = 0 + while True: + x = trans.readAll(1) + byte = ord(x) + result |= (byte & 0x7f) << shift + if byte >> 7 == 0: + return result + shift += 7 + +class CompactType: + TRUE = 1 + FALSE = 2 + BYTE = 0x03 + I16 = 0x04 + I32 = 0x05 + I64 = 0x06 + DOUBLE = 0x07 + BINARY = 0x08 + LIST = 0x09 + SET = 0x0A + MAP = 0x0B + STRUCT = 0x0C + +CTYPES = {TType.BOOL: CompactType.TRUE, # used for collection + TType.BYTE: CompactType.BYTE, + TType.I16: CompactType.I16, + TType.I32: CompactType.I32, + TType.I64: CompactType.I64, + TType.DOUBLE: CompactType.DOUBLE, + TType.STRING: CompactType.BINARY, + TType.STRUCT: CompactType.STRUCT, + TType.LIST: CompactType.LIST, + TType.SET: CompactType.SET, + TType.MAP: CompactType.MAP, + } + +TTYPES = {} +for k, v in CTYPES.items(): + TTYPES[v] = k +TTYPES[CompactType.FALSE] = TType.BOOL +del k +del v + +class TCompactProtocol(TProtocolBase): + "Compact implementation of the Thrift protocol driver." + + PROTOCOL_ID = 0x82 + VERSION = 1 + VERSION_MASK = 0x1f + TYPE_MASK = 0xe0 + TYPE_SHIFT_AMOUNT = 5 + + def __init__(self, trans): + TProtocolBase.__init__(self, trans) + self.state = CLEAR + self.__last_fid = 0 + self.__bool_fid = None + self.__bool_value = None + self.__structs = [] + self.__containers = [] + + def __writeVarint(self, n): + writeVarint(self.trans, n) + + def writeMessageBegin(self, name, type, seqid): + assert self.state == CLEAR + self.__writeUByte(self.PROTOCOL_ID) + self.__writeUByte(self.VERSION | (type << self.TYPE_SHIFT_AMOUNT)) + self.__writeVarint(seqid) + self.__writeString(name) + self.state = VALUE_WRITE + + def writeMessageEnd(self): + assert self.state == VALUE_WRITE + self.state = CLEAR + + def writeStructBegin(self, name): + assert self.state in (CLEAR, CONTAINER_WRITE, VALUE_WRITE), self.state + self.__structs.append((self.state, self.__last_fid)) + self.state = FIELD_WRITE + self.__last_fid = 0 + + def writeStructEnd(self): + assert self.state == FIELD_WRITE + self.state, self.__last_fid = self.__structs.pop() + + def writeFieldStop(self): + self.__writeByte(0) + + def __writeFieldHeader(self, type, fid): + delta = fid - self.__last_fid + if 0 < delta <= 15: + self.__writeUByte(delta << 4 | type) + else: + self.__writeByte(type) + self.__writeI16(fid) + self.__last_fid = fid + + def writeFieldBegin(self, name, type, fid): + assert self.state == FIELD_WRITE, self.state + if type == TType.BOOL: + self.state = BOOL_WRITE + self.__bool_fid = fid + else: + self.state = VALUE_WRITE + self.__writeFieldHeader(CTYPES[type], fid) + + def writeFieldEnd(self): + assert self.state in (VALUE_WRITE, BOOL_WRITE), self.state + self.state = FIELD_WRITE + + def __writeUByte(self, byte): + self.trans.write(pack('!B', byte)) + + def __writeByte(self, byte): + self.trans.write(pack('!b', byte)) + + def __writeI16(self, i16): + self.__writeVarint(makeZigZag(i16, 16)) + + def __writeSize(self, i32): + self.__writeVarint(i32) + + def writeCollectionBegin(self, etype, size): + assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state + if size <= 14: + self.__writeUByte(size << 4 | CTYPES[etype]) + else: + self.__writeUByte(0xf0 | CTYPES[etype]) + self.__writeSize(size) + self.__containers.append(self.state) + self.state = CONTAINER_WRITE + writeSetBegin = writeCollectionBegin + writeListBegin = writeCollectionBegin + + def writeMapBegin(self, ktype, vtype, size): + assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state + if size == 0: + self.__writeByte(0) + else: + self.__writeSize(size) + self.__writeUByte(CTYPES[ktype] << 4 | CTYPES[vtype]) + self.__containers.append(self.state) + self.state = CONTAINER_WRITE + + def writeCollectionEnd(self): + assert self.state == CONTAINER_WRITE, self.state + self.state = self.__containers.pop() + writeMapEnd = writeCollectionEnd + writeSetEnd = writeCollectionEnd + writeListEnd = writeCollectionEnd + + def writeBool(self, bool): + if self.state == BOOL_WRITE: + self.__writeFieldHeader(types[bool], self.__bool_fid) + elif self.state == CONTAINER_WRITE: + self.__writeByte(int(bool)) + else: + raise AssertetionError, "Invalid state in compact protocol" + + writeByte = writer(__writeByte) + writeI16 = writer(__writeI16) + + @writer + def writeI32(self, i32): + self.__writeVarint(makeZigZag(i32, 32)) + + @writer + def writeI64(self, i64): + self.__writeVarint(makeZigZag(i64, 64)) + + @writer + def writeDouble(self, dub): + self.trans.write(pack('!d', dub)) + + def __writeString(self, s): + self.__writeSize(len(s)) + self.trans.write(s) + writeString = writer(__writeString) + + def readFieldBegin(self): + assert self.state == FIELD_READ, self.state + type = self.__readUByte() + if type & 0x0f == TType.STOP: + return (None, 0, 0) + delta = type >> 4 + if delta == 0: + fid = self.__readI16() + else: + fid = self.__last_fid + delta + self.__last_fid = fid + type = type & 0x0f + if type == CompactType.TRUE: + self.state = BOOL_READ + self.__bool_value = True + elif type == CompactType.FALSE: + self.state = BOOL_READ + self.__bool_value = False + else: + self.state = VALUE_READ + return (None, self.__getTType(type), fid) + + def readFieldEnd(self): + assert self.state in (VALUE_READ, BOOL_READ), self.state + self.state = FIELD_READ + + def __readUByte(self): + result, = unpack('!B', self.trans.readAll(1)) + return result + + def __readByte(self): + result, = unpack('!b', self.trans.readAll(1)) + return result + + def __readVarint(self): + return readVarint(self.trans) + + def __readZigZag(self): + return fromZigZag(self.__readVarint()) + + def __readSize(self): + result = self.__readVarint() + if result < 0: + raise TException("Length < 0") + return result + + def readMessageBegin(self): + assert self.state == CLEAR + proto_id = self.__readUByte() + if proto_id != self.PROTOCOL_ID: + raise TProtocolException(TProtocolException.BAD_VERSION, + 'Bad protocol id in the message: %d' % proto_id) + ver_type = self.__readUByte() + type = (ver_type & self.TYPE_MASK) >> self.TYPE_SHIFT_AMOUNT + version = ver_type & self.VERSION_MASK + if version != self.VERSION: + raise TProtocolException(TProtocolException.BAD_VERSION, + 'Bad version: %d (expect %d)' % (version, self.VERSION)) + seqid = self.__readVarint() + name = self.__readString() + return (name, type, seqid) + + def readMessageEnd(self): + assert self.state == VALUE_READ + assert len(self.__structs) == 0 + self.state = CLEAR + + def readStructBegin(self): + assert self.state in (CLEAR, CONTAINER_READ, VALUE_READ), self.state + self.__structs.append((self.state, self.__last_fid)) + self.state = FIELD_READ + self.__last_fid = 0 + + def readStructEnd(self): + assert self.state == FIELD_READ + self.state, self.__last_fid = self.__structs.pop() + + def readCollectionBegin(self): + assert self.state in (VALUE_READ, CONTAINER_READ), self.state + size_type = self.__readUByte() + size = size_type >> 4 + type = self.__getTType(size_type) + if size == 15: + size = self.__readSize() + self.__containers.append(self.state) + self.state = CONTAINER_READ + return type, size + readSetBegin = readCollectionBegin + readListBegin = readCollectionBegin + + def readMapBegin(self): + assert self.state in (VALUE_READ, CONTAINER_READ), self.state + size = self.__readSize() + types = 0 + if size > 0: + types = self.__readUByte() + vtype = self.__getTType(types) + ktype = self.__getTType(types >> 4) + self.__containers.append(self.state) + self.state = CONTAINER_READ + return (ktype, vtype, size) + + def readCollectionEnd(self): + assert self.state == CONTAINER_READ, self.state + self.state = self.__containers.pop() + readSetEnd = readCollectionEnd + readListEnd = readCollectionEnd + readMapEnd = readCollectionEnd + + def readBool(self): + if self.state == BOOL_READ: + return self.__bool_value + elif self.state == CONTAINER_READ: + return bool(self.__readByte()) + else: + raise AssertionError, "Invalid state in compact protocol: %d" % self.state + + readByte = reader(__readByte) + __readI16 = __readZigZag + readI16 = reader(__readZigZag) + readI32 = reader(__readZigZag) + readI64 = reader(__readZigZag) + + @reader + def readDouble(self): + buff = self.trans.readAll(8) + val, = unpack('!d', buff) + return val + + def __readString(self): + len = self.__readSize() + return self.trans.readAll(len) + readString = reader(__readString) + + def __getTType(self, byte): + return TTYPES[byte & 0x0f] + + +class TCompactProtocolFactory: + def __init__(self): + pass + + def getProtocol(self, trans): + return TCompactProtocol(trans) diff --git a/nebu/thrift/protocol/TProtocol.py b/nebu/thrift/protocol/TProtocol.py new file mode 100644 index 0000000..be3cb14 --- /dev/null +++ b/nebu/thrift/protocol/TProtocol.py @@ -0,0 +1,205 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from thrift.Thrift import * + +class TProtocolException(TException): + + """Custom Protocol Exception class""" + + UNKNOWN = 0 + INVALID_DATA = 1 + NEGATIVE_SIZE = 2 + SIZE_LIMIT = 3 + BAD_VERSION = 4 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + +class TProtocolBase: + + """Base class for Thrift protocol driver.""" + + def __init__(self, trans): + self.trans = trans + + def writeMessageBegin(self, name, type, seqid): + pass + + def writeMessageEnd(self): + pass + + def writeStructBegin(self, name): + pass + + def writeStructEnd(self): + pass + + def writeFieldBegin(self, name, type, id): + pass + + def writeFieldEnd(self): + pass + + def writeFieldStop(self): + pass + + def writeMapBegin(self, ktype, vtype, size): + pass + + def writeMapEnd(self): + pass + + def writeListBegin(self, etype, size): + pass + + def writeListEnd(self): + pass + + def writeSetBegin(self, etype, size): + pass + + def writeSetEnd(self): + pass + + def writeBool(self, bool): + pass + + def writeByte(self, byte): + pass + + def writeI16(self, i16): + pass + + def writeI32(self, i32): + pass + + def writeI64(self, i64): + pass + + def writeDouble(self, dub): + pass + + def writeString(self, str): + pass + + def readMessageBegin(self): + pass + + def readMessageEnd(self): + pass + + def readStructBegin(self): + pass + + def readStructEnd(self): + pass + + def readFieldBegin(self): + pass + + def readFieldEnd(self): + pass + + def readMapBegin(self): + pass + + def readMapEnd(self): + pass + + def readListBegin(self): + pass + + def readListEnd(self): + pass + + def readSetBegin(self): + pass + + def readSetEnd(self): + pass + + def readBool(self): + pass + + def readByte(self): + pass + + def readI16(self): + pass + + def readI32(self): + pass + + def readI64(self): + pass + + def readDouble(self): + pass + + def readString(self): + pass + + def skip(self, type): + if type == TType.STOP: + return + elif type == TType.BOOL: + self.readBool() + elif type == TType.BYTE: + self.readByte() + elif type == TType.I16: + self.readI16() + elif type == TType.I32: + self.readI32() + elif type == TType.I64: + self.readI64() + elif type == TType.DOUBLE: + self.readDouble() + elif type == TType.STRING: + self.readString() + elif type == TType.STRUCT: + name = self.readStructBegin() + while True: + (name, type, id) = self.readFieldBegin() + if type == TType.STOP: + break + self.skip(type) + self.readFieldEnd() + self.readStructEnd() + elif type == TType.MAP: + (ktype, vtype, size) = self.readMapBegin() + for i in range(size): + self.skip(ktype) + self.skip(vtype) + self.readMapEnd() + elif type == TType.SET: + (etype, size) = self.readSetBegin() + for i in range(size): + self.skip(etype) + self.readSetEnd() + elif type == TType.LIST: + (etype, size) = self.readListBegin() + for i in range(size): + self.skip(etype) + self.readListEnd() + +class TProtocolFactory: + def getProtocol(self, trans): + pass diff --git a/nebu/thrift/protocol/__init__.py b/nebu/thrift/protocol/__init__.py new file mode 100644 index 0000000..01bfe18 --- /dev/null +++ b/nebu/thrift/protocol/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TProtocol', 'TBinaryProtocol', 'fastbinary'] diff --git a/nebu/thrift/protocol/fastbinary.c b/nebu/thrift/protocol/fastbinary.c new file mode 100644 index 0000000..67b215a --- /dev/null +++ b/nebu/thrift/protocol/fastbinary.c @@ -0,0 +1,1203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include "cStringIO.h" +#include +#include +#include + +/* Fix endianness issues on Solaris */ +#if defined (__SVR4) && defined (__sun) + #if defined(__i386) && !defined(__i386__) + #define __i386__ + #endif + + #ifndef BIG_ENDIAN + #define BIG_ENDIAN (4321) + #endif + #ifndef LITTLE_ENDIAN + #define LITTLE_ENDIAN (1234) + #endif + + /* I386 is LE, even on Solaris */ + #if !defined(BYTE_ORDER) && defined(__i386__) + #define BYTE_ORDER LITTLE_ENDIAN + #endif +#endif + +// TODO(dreiss): defval appears to be unused. Look into removing it. +// TODO(dreiss): Make parse_spec_args recursive, and cache the output +// permanently in the object. (Malloc and orphan.) +// TODO(dreiss): Why do we need cStringIO for reading, why not just char*? +// Can cStringIO let us work with a BufferedTransport? +// TODO(dreiss): Don't ignore the rv from cwrite (maybe). + +/* ====== BEGIN UTILITIES ====== */ + +#define INIT_OUTBUF_SIZE 128 + +// Stolen out of TProtocol.h. +// It would be a huge pain to have both get this from one place. +typedef enum TType { + T_STOP = 0, + T_VOID = 1, + T_BOOL = 2, + T_BYTE = 3, + T_I08 = 3, + T_I16 = 6, + T_I32 = 8, + T_U64 = 9, + T_I64 = 10, + T_DOUBLE = 4, + T_STRING = 11, + T_UTF7 = 11, + T_STRUCT = 12, + T_MAP = 13, + T_SET = 14, + T_LIST = 15, + T_UTF8 = 16, + T_UTF16 = 17 +} TType; + +#ifndef __BYTE_ORDER +# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN) +# define __BYTE_ORDER BYTE_ORDER +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __BIG_ENDIAN BIG_ENDIAN +# else +# error "Cannot determine endianness" +# endif +#endif + +// Same comment as the enum. Sorry. +#if __BYTE_ORDER == __BIG_ENDIAN +# define ntohll(n) (n) +# define htonll(n) (n) +#elif __BYTE_ORDER == __LITTLE_ENDIAN +# if defined(__GNUC__) && defined(__GLIBC__) +# include +# define ntohll(n) bswap_64(n) +# define htonll(n) bswap_64(n) +# else /* GNUC & GLIBC */ +# define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) ) +# define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) ) +# endif /* GNUC & GLIBC */ +#else /* __BYTE_ORDER */ +# error "Can't define htonll or ntohll!" +#endif + +// Doing a benchmark shows that interning actually makes a difference, amazingly. +#define INTERN_STRING(value) _intern_ ## value + +#define INT_CONV_ERROR_OCCURRED(v) ( ((v) == -1) && PyErr_Occurred() ) +#define CHECK_RANGE(v, min, max) ( ((v) <= (max)) && ((v) >= (min)) ) + +// Py_ssize_t was not defined before Python 2.5 +#if (PY_VERSION_HEX < 0x02050000) +typedef int Py_ssize_t; +#endif + +/** + * A cache of the spec_args for a set or list, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + TType element_type; + PyObject* typeargs; +} SetListTypeArgs; + +/** + * A cache of the spec_args for a map, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + TType ktag; + TType vtag; + PyObject* ktypeargs; + PyObject* vtypeargs; +} MapTypeArgs; + +/** + * A cache of the spec_args for a struct, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + PyObject* klass; + PyObject* spec; +} StructTypeArgs; + +/** + * A cache of the item spec from a struct specification, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + int tag; + TType type; + PyObject* attrname; + PyObject* typeargs; + PyObject* defval; +} StructItemSpec; + +/** + * A cache of the two key attributes of a CReadableTransport, + * so we don't have to keep calling PyObject_GetAttr. + */ +typedef struct { + PyObject* stringiobuf; + PyObject* refill_callable; +} DecodeBuffer; + +/** Pointer to interned string to speed up attribute lookup. */ +static PyObject* INTERN_STRING(cstringio_buf); +/** Pointer to interned string to speed up attribute lookup. */ +static PyObject* INTERN_STRING(cstringio_refill); + +static inline bool +check_ssize_t_32(Py_ssize_t len) { + // error from getting the int + if (INT_CONV_ERROR_OCCURRED(len)) { + return false; + } + if (!CHECK_RANGE(len, 0, INT32_MAX)) { + PyErr_SetString(PyExc_OverflowError, "string size out of range"); + return false; + } + return true; +} + +static inline bool +parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) { + long val = PyInt_AsLong(o); + + if (INT_CONV_ERROR_OCCURRED(val)) { + return false; + } + if (!CHECK_RANGE(val, min, max)) { + PyErr_SetString(PyExc_OverflowError, "int out of range"); + return false; + } + + *ret = (int32_t) val; + return true; +} + + +/* --- FUNCTIONS TO PARSE STRUCT SPECIFICATOINS --- */ + +static bool +parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 2) { + PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args"); + return false; + } + + dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->element_type)) { + return false; + } + + dest->typeargs = PyTuple_GET_ITEM(typeargs, 1); + + return true; +} + +static bool +parse_map_args(MapTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 4) { + PyErr_SetString(PyExc_TypeError, "expecting 4 arguments for typeargs to map"); + return false; + } + + dest->ktag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->ktag)) { + return false; + } + + dest->vtag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2)); + if (INT_CONV_ERROR_OCCURRED(dest->vtag)) { + return false; + } + + dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1); + dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3); + + return true; +} + +static bool +parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 2) { + PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for struct args"); + return false; + } + + dest->klass = PyTuple_GET_ITEM(typeargs, 0); + dest->spec = PyTuple_GET_ITEM(typeargs, 1); + + return true; +} + +static int +parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) { + + // i'd like to use ParseArgs here, but it seems to be a bottleneck. + if (PyTuple_Size(spec_tuple) != 5) { + PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for spec tuple"); + return false; + } + + dest->tag = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->tag)) { + return false; + } + + dest->type = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1)); + if (INT_CONV_ERROR_OCCURRED(dest->type)) { + return false; + } + + dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2); + dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3); + dest->defval = PyTuple_GET_ITEM(spec_tuple, 4); + return true; +} + +/* ====== END UTILITIES ====== */ + + +/* ====== BEGIN WRITING FUNCTIONS ====== */ + +/* --- LOW-LEVEL WRITING FUNCTIONS --- */ + +static void writeByte(PyObject* outbuf, int8_t val) { + int8_t net = val; + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int8_t)); +} + +static void writeI16(PyObject* outbuf, int16_t val) { + int16_t net = (int16_t)htons(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int16_t)); +} + +static void writeI32(PyObject* outbuf, int32_t val) { + int32_t net = (int32_t)htonl(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int32_t)); +} + +static void writeI64(PyObject* outbuf, int64_t val) { + int64_t net = (int64_t)htonll(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int64_t)); +} + +static void writeDouble(PyObject* outbuf, double dub) { + // Unfortunately, bitwise_cast doesn't work in C. Bad C! + union { + double f; + int64_t t; + } transfer; + transfer.f = dub; + writeI64(outbuf, transfer.t); +} + + +/* --- MAIN RECURSIVE OUTPUT FUCNTION -- */ + +static int +output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) { + /* + * Refcounting Strategy: + * + * We assume that elements of the thrift_spec tuple are not going to be + * mutated, so we don't ref count those at all. Other than that, we try to + * keep a reference to all the user-created objects while we work with them. + * output_val assumes that a reference is already held. The *caller* is + * responsible for handling references + */ + + switch (type) { + + case T_BOOL: { + int v = PyObject_IsTrue(value); + if (v == -1) { + return false; + } + + writeByte(output, (int8_t) v); + break; + } + case T_I08: { + int32_t val; + + if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) { + return false; + } + + writeByte(output, (int8_t) val); + break; + } + case T_I16: { + int32_t val; + + if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) { + return false; + } + + writeI16(output, (int16_t) val); + break; + } + case T_I32: { + int32_t val; + + if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) { + return false; + } + + writeI32(output, val); + break; + } + case T_I64: { + int64_t nval = PyLong_AsLongLong(value); + + if (INT_CONV_ERROR_OCCURRED(nval)) { + return false; + } + + if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) { + PyErr_SetString(PyExc_OverflowError, "int out of range"); + return false; + } + + writeI64(output, nval); + break; + } + + case T_DOUBLE: { + double nval = PyFloat_AsDouble(value); + if (nval == -1.0 && PyErr_Occurred()) { + return false; + } + + writeDouble(output, nval); + break; + } + + case T_STRING: { + Py_ssize_t len = PyString_Size(value); + + if (!check_ssize_t_32(len)) { + return false; + } + + writeI32(output, (int32_t) len); + PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len); + break; + } + + case T_LIST: + case T_SET: { + Py_ssize_t len; + SetListTypeArgs parsedargs; + PyObject *item; + PyObject *iterator; + + if (!parse_set_list_args(&parsedargs, typeargs)) { + return false; + } + + len = PyObject_Length(value); + + if (!check_ssize_t_32(len)) { + return false; + } + + writeByte(output, parsedargs.element_type); + writeI32(output, (int32_t) len); + + iterator = PyObject_GetIter(value); + if (iterator == NULL) { + return false; + } + + while ((item = PyIter_Next(iterator))) { + if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) { + Py_DECREF(item); + Py_DECREF(iterator); + return false; + } + Py_DECREF(item); + } + + Py_DECREF(iterator); + + if (PyErr_Occurred()) { + return false; + } + + break; + } + + case T_MAP: { + PyObject *k, *v; + Py_ssize_t pos = 0; + Py_ssize_t len; + + MapTypeArgs parsedargs; + + len = PyDict_Size(value); + if (!check_ssize_t_32(len)) { + return false; + } + + if (!parse_map_args(&parsedargs, typeargs)) { + return false; + } + + writeByte(output, parsedargs.ktag); + writeByte(output, parsedargs.vtag); + writeI32(output, len); + + // TODO(bmaurer): should support any mapping, not just dicts + while (PyDict_Next(value, &pos, &k, &v)) { + // TODO(dreiss): Think hard about whether these INCREFs actually + // turn any unsafe scenarios into safe scenarios. + Py_INCREF(k); + Py_INCREF(v); + + if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs) + || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) { + Py_DECREF(k); + Py_DECREF(v); + return false; + } + Py_DECREF(k); + Py_DECREF(v); + } + break; + } + + // TODO(dreiss): Consider breaking this out as a function + // the way we did for decode_struct. + case T_STRUCT: { + StructTypeArgs parsedargs; + Py_ssize_t nspec; + Py_ssize_t i; + + if (!parse_struct_args(&parsedargs, typeargs)) { + return false; + } + + nspec = PyTuple_Size(parsedargs.spec); + + if (nspec == -1) { + return false; + } + + for (i = 0; i < nspec; i++) { + StructItemSpec parsedspec; + PyObject* spec_tuple; + PyObject* instval = NULL; + + spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i); + if (spec_tuple == Py_None) { + continue; + } + + if (!parse_struct_item_spec (&parsedspec, spec_tuple)) { + return false; + } + + instval = PyObject_GetAttr(value, parsedspec.attrname); + + if (!instval) { + return false; + } + + if (instval == Py_None) { + Py_DECREF(instval); + continue; + } + + writeByte(output, (int8_t) parsedspec.type); + writeI16(output, parsedspec.tag); + + if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) { + Py_DECREF(instval); + return false; + } + + Py_DECREF(instval); + } + + writeByte(output, (int8_t)T_STOP); + break; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return false; + + } + + return true; +} + + +/* --- TOP-LEVEL WRAPPER FOR OUTPUT -- */ + +static PyObject * +encode_binary(PyObject *self, PyObject *args) { + PyObject* enc_obj; + PyObject* type_args; + PyObject* buf; + PyObject* ret = NULL; + + if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) { + return NULL; + } + + buf = PycStringIO->NewOutput(INIT_OUTBUF_SIZE); + if (output_val(buf, enc_obj, T_STRUCT, type_args)) { + ret = PycStringIO->cgetvalue(buf); + } + + Py_DECREF(buf); + return ret; +} + +/* ====== END WRITING FUNCTIONS ====== */ + + +/* ====== BEGIN READING FUNCTIONS ====== */ + +/* --- LOW-LEVEL READING FUNCTIONS --- */ + +static void +free_decodebuf(DecodeBuffer* d) { + Py_XDECREF(d->stringiobuf); + Py_XDECREF(d->refill_callable); +} + +static bool +decode_buffer_from_obj(DecodeBuffer* dest, PyObject* obj) { + dest->stringiobuf = PyObject_GetAttr(obj, INTERN_STRING(cstringio_buf)); + if (!dest->stringiobuf) { + return false; + } + + if (!PycStringIO_InputCheck(dest->stringiobuf)) { + free_decodebuf(dest); + PyErr_SetString(PyExc_TypeError, "expecting stringio input"); + return false; + } + + dest->refill_callable = PyObject_GetAttr(obj, INTERN_STRING(cstringio_refill)); + + if(!dest->refill_callable) { + free_decodebuf(dest); + return false; + } + + if (!PyCallable_Check(dest->refill_callable)) { + free_decodebuf(dest); + PyErr_SetString(PyExc_TypeError, "expecting callable"); + return false; + } + + return true; +} + +static bool readBytes(DecodeBuffer* input, char** output, int len) { + int read; + + // TODO(dreiss): Don't fear the malloc. Think about taking a copy of + // the partial read instead of forcing the transport + // to prepend it to its buffer. + + read = PycStringIO->cread(input->stringiobuf, output, len); + + if (read == len) { + return true; + } else if (read == -1) { + return false; + } else { + PyObject* newiobuf; + + // using building functions as this is a rare codepath + newiobuf = PyObject_CallFunction( + input->refill_callable, "s#i", *output, read, len, NULL); + if (newiobuf == NULL) { + return false; + } + + // must do this *AFTER* the call so that we don't deref the io buffer + Py_CLEAR(input->stringiobuf); + input->stringiobuf = newiobuf; + + read = PycStringIO->cread(input->stringiobuf, output, len); + + if (read == len) { + return true; + } else if (read == -1) { + return false; + } else { + // TODO(dreiss): This could be a valid code path for big binary blobs. + PyErr_SetString(PyExc_TypeError, + "refill claimed to have refilled the buffer, but didn't!!"); + return false; + } + } +} + +static int8_t readByte(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int8_t))) { + return -1; + } + + return *(int8_t*) buf; +} + +static int16_t readI16(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int16_t))) { + return -1; + } + + return (int16_t) ntohs(*(int16_t*) buf); +} + +static int32_t readI32(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int32_t))) { + return -1; + } + return (int32_t) ntohl(*(int32_t*) buf); +} + + +static int64_t readI64(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int64_t))) { + return -1; + } + + return (int64_t) ntohll(*(int64_t*) buf); +} + +static double readDouble(DecodeBuffer* input) { + union { + int64_t f; + double t; + } transfer; + + transfer.f = readI64(input); + if (transfer.f == -1) { + return -1; + } + return transfer.t; +} + +static bool +checkTypeByte(DecodeBuffer* input, TType expected) { + TType got = readByte(input); + if (INT_CONV_ERROR_OCCURRED(got)) { + return false; + } + + if (expected != got) { + PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field"); + return false; + } + return true; +} + +static bool +skip(DecodeBuffer* input, TType type) { +#define SKIPBYTES(n) \ + do { \ + if (!readBytes(input, &dummy_buf, (n))) { \ + return false; \ + } \ + } while(0) + + char* dummy_buf; + + switch (type) { + + case T_BOOL: + case T_I08: SKIPBYTES(1); break; + case T_I16: SKIPBYTES(2); break; + case T_I32: SKIPBYTES(4); break; + case T_I64: + case T_DOUBLE: SKIPBYTES(8); break; + + case T_STRING: { + // TODO(dreiss): Find out if these check_ssize_t32s are really necessary. + int len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + SKIPBYTES(len); + break; + } + + case T_LIST: + case T_SET: { + TType etype; + int len, i; + + etype = readByte(input); + if (etype == -1) { + return false; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + for (i = 0; i < len; i++) { + if (!skip(input, etype)) { + return false; + } + } + break; + } + + case T_MAP: { + TType ktype, vtype; + int len, i; + + ktype = readByte(input); + if (ktype == -1) { + return false; + } + + vtype = readByte(input); + if (vtype == -1) { + return false; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + for (i = 0; i < len; i++) { + if (!(skip(input, ktype) && skip(input, vtype))) { + return false; + } + } + break; + } + + case T_STRUCT: { + while (true) { + TType type; + + type = readByte(input); + if (type == -1) { + return false; + } + + if (type == T_STOP) + break; + + SKIPBYTES(2); // tag + if (!skip(input, type)) { + return false; + } + } + break; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return false; + + } + + return true; + +#undef SKIPBYTES +} + + +/* --- HELPER FUNCTION FOR DECODE_VAL --- */ + +static PyObject* +decode_val(DecodeBuffer* input, TType type, PyObject* typeargs); + +static bool +decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) { + int spec_seq_len = PyTuple_Size(spec_seq); + if (spec_seq_len == -1) { + return false; + } + + while (true) { + TType type; + int16_t tag; + PyObject* item_spec; + PyObject* fieldval = NULL; + StructItemSpec parsedspec; + + type = readByte(input); + if (type == -1) { + return false; + } + if (type == T_STOP) { + break; + } + tag = readI16(input); + if (INT_CONV_ERROR_OCCURRED(tag)) { + return false; + } + if (tag >= 0 && tag < spec_seq_len) { + item_spec = PyTuple_GET_ITEM(spec_seq, tag); + } else { + item_spec = Py_None; + } + + if (item_spec == Py_None) { + if (!skip(input, type)) { + return false; + } else { + continue; + } + } + + if (!parse_struct_item_spec(&parsedspec, item_spec)) { + return false; + } + if (parsedspec.type != type) { + if (!skip(input, type)) { + PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading and can't be skipped"); + return false; + } else { + continue; + } + } + + fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs); + if (fieldval == NULL) { + return false; + } + + if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) { + Py_DECREF(fieldval); + return false; + } + Py_DECREF(fieldval); + } + return true; +} + + +/* --- MAIN RECURSIVE INPUT FUCNTION --- */ + +// Returns a new reference. +static PyObject* +decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) { + switch (type) { + + case T_BOOL: { + int8_t v = readByte(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + + switch (v) { + case 0: Py_RETURN_FALSE; + case 1: Py_RETURN_TRUE; + // Don't laugh. This is a potentially serious issue. + default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL; + } + break; + } + case T_I08: { + int8_t v = readByte(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + + return PyInt_FromLong(v); + } + case T_I16: { + int16_t v = readI16(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + return PyInt_FromLong(v); + } + case T_I32: { + int32_t v = readI32(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + return PyInt_FromLong(v); + } + + case T_I64: { + int64_t v = readI64(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + // TODO(dreiss): Find out if we can take this fastpath always when + // sizeof(long) == sizeof(long long). + if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) { + return PyInt_FromLong((long) v); + } + + return PyLong_FromLongLong(v); + } + + case T_DOUBLE: { + double v = readDouble(input); + if (v == -1.0 && PyErr_Occurred()) { + return false; + } + return PyFloat_FromDouble(v); + } + + case T_STRING: { + Py_ssize_t len = readI32(input); + char* buf; + if (!readBytes(input, &buf, len)) { + return NULL; + } + + return PyString_FromStringAndSize(buf, len); + } + + case T_LIST: + case T_SET: { + SetListTypeArgs parsedargs; + int32_t len; + PyObject* ret = NULL; + int i; + + if (!parse_set_list_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!checkTypeByte(input, parsedargs.element_type)) { + return NULL; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return NULL; + } + + ret = PyList_New(len); + if (!ret) { + return NULL; + } + + for (i = 0; i < len; i++) { + PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs); + if (!item) { + Py_DECREF(ret); + return NULL; + } + PyList_SET_ITEM(ret, i, item); + } + + // TODO(dreiss): Consider biting the bullet and making two separate cases + // for list and set, avoiding this post facto conversion. + if (type == T_SET) { + PyObject* setret; +#if (PY_VERSION_HEX < 0x02050000) + // hack needed for older versions + setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL); +#else + // official version + setret = PySet_New(ret); +#endif + Py_DECREF(ret); + return setret; + } + return ret; + } + + case T_MAP: { + int32_t len; + int i; + MapTypeArgs parsedargs; + PyObject* ret = NULL; + + if (!parse_map_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!checkTypeByte(input, parsedargs.ktag)) { + return NULL; + } + if (!checkTypeByte(input, parsedargs.vtag)) { + return NULL; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + ret = PyDict_New(); + if (!ret) { + goto error; + } + + for (i = 0; i < len; i++) { + PyObject* k = NULL; + PyObject* v = NULL; + k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs); + if (k == NULL) { + goto loop_error; + } + v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs); + if (v == NULL) { + goto loop_error; + } + if (PyDict_SetItem(ret, k, v) == -1) { + goto loop_error; + } + + Py_DECREF(k); + Py_DECREF(v); + continue; + + // Yuck! Destructors, anyone? + loop_error: + Py_XDECREF(k); + Py_XDECREF(v); + goto error; + } + + return ret; + + error: + Py_XDECREF(ret); + return NULL; + } + + case T_STRUCT: { + StructTypeArgs parsedargs; + if (!parse_struct_args(&parsedargs, typeargs)) { + return NULL; + } + + PyObject* ret = PyObject_CallObject(parsedargs.klass, NULL); + if (!ret) { + return NULL; + } + + if (!decode_struct(input, ret, parsedargs.spec)) { + Py_DECREF(ret); + return NULL; + } + + return ret; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return NULL; + } +} + + +/* --- TOP-LEVEL WRAPPER FOR INPUT -- */ + +static PyObject* +decode_binary(PyObject *self, PyObject *args) { + PyObject* output_obj = NULL; + PyObject* transport = NULL; + PyObject* typeargs = NULL; + StructTypeArgs parsedargs; + DecodeBuffer input = {}; + + if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) { + return NULL; + } + + if (!parse_struct_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!decode_buffer_from_obj(&input, transport)) { + return NULL; + } + + if (!decode_struct(&input, output_obj, parsedargs.spec)) { + free_decodebuf(&input); + return NULL; + } + + free_decodebuf(&input); + + Py_RETURN_NONE; +} + +/* ====== END READING FUNCTIONS ====== */ + + +/* -- PYTHON MODULE SETUP STUFF --- */ + +static PyMethodDef ThriftFastBinaryMethods[] = { + + {"encode_binary", encode_binary, METH_VARARGS, ""}, + {"decode_binary", decode_binary, METH_VARARGS, ""}, + + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +PyMODINIT_FUNC +initfastbinary(void) { +#define INIT_INTERN_STRING(value) \ + do { \ + INTERN_STRING(value) = PyString_InternFromString(#value); \ + if(!INTERN_STRING(value)) return; \ + } while(0) + + INIT_INTERN_STRING(cstringio_buf); + INIT_INTERN_STRING(cstringio_refill); +#undef INIT_INTERN_STRING + + PycString_IMPORT; + if (PycStringIO == NULL) return; + + (void) Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods); +} diff --git a/nebu/thrift/server/THttpServer.py b/nebu/thrift/server/THttpServer.py new file mode 100644 index 0000000..3047d9c --- /dev/null +++ b/nebu/thrift/server/THttpServer.py @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import BaseHTTPServer + +from thrift.server import TServer +from thrift.transport import TTransport + +class ResponseException(Exception): + """Allows handlers to override the HTTP response + + Normally, THttpServer always sends a 200 response. If a handler wants + to override this behavior (e.g., to simulate a misconfigured or + overloaded web server during testing), it can raise a ResponseException. + The function passed to the constructor will be called with the + RequestHandler as its only argument. + """ + def __init__(self, handler): + self.handler = handler + + +class THttpServer(TServer.TServer): + """A simple HTTP-based Thrift server + + This class is not very performant, but it is useful (for example) for + acting as a mock version of an Apache-based PHP Thrift endpoint.""" + + def __init__(self, processor, server_address, + inputProtocolFactory, outputProtocolFactory = None, + server_class = BaseHTTPServer.HTTPServer): + """Set up protocol factories and HTTP server. + + See BaseHTTPServer for server_address. + See TServer for protocol factories.""" + + if outputProtocolFactory is None: + outputProtocolFactory = inputProtocolFactory + + TServer.TServer.__init__(self, processor, None, None, None, + inputProtocolFactory, outputProtocolFactory) + + thttpserver = self + + class RequestHander(BaseHTTPServer.BaseHTTPRequestHandler): + def do_POST(self): + # Don't care about the request path. + itrans = TTransport.TFileObjectTransport(self.rfile) + otrans = TTransport.TFileObjectTransport(self.wfile) + itrans = TTransport.TBufferedTransport(itrans, int(self.headers['Content-Length'])) + otrans = TTransport.TMemoryBuffer() + iprot = thttpserver.inputProtocolFactory.getProtocol(itrans) + oprot = thttpserver.outputProtocolFactory.getProtocol(otrans) + try: + thttpserver.processor.process(iprot, oprot) + except ResponseException, exn: + exn.handler(self) + else: + self.send_response(200) + self.send_header("content-type", "application/x-thrift") + self.end_headers() + self.wfile.write(otrans.getvalue()) + + self.httpd = server_class(server_address, RequestHander) + + def serve(self): + self.httpd.serve_forever() diff --git a/nebu/thrift/server/TNonblockingServer.py b/nebu/thrift/server/TNonblockingServer.py new file mode 100644 index 0000000..ea348a0 --- /dev/null +++ b/nebu/thrift/server/TNonblockingServer.py @@ -0,0 +1,310 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +"""Implementation of non-blocking server. + +The main idea of the server is reciving and sending requests +only from main thread. + +It also makes thread pool server in tasks terms, not connections. +""" +import threading +import socket +import Queue +import select +import struct +import logging + +from thrift.transport import TTransport +from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory + +__all__ = ['TNonblockingServer'] + +class Worker(threading.Thread): + """Worker is a small helper to process incoming connection.""" + def __init__(self, queue): + threading.Thread.__init__(self) + self.queue = queue + + def run(self): + """Process queries from task queue, stop if processor is None.""" + while True: + try: + processor, iprot, oprot, otrans, callback = self.queue.get() + if processor is None: + break + processor.process(iprot, oprot) + callback(True, otrans.getvalue()) + except Exception: + logging.exception("Exception while processing request") + callback(False, '') + +WAIT_LEN = 0 +WAIT_MESSAGE = 1 +WAIT_PROCESS = 2 +SEND_ANSWER = 3 +CLOSED = 4 + +def locked(func): + "Decorator which locks self.lock." + def nested(self, *args, **kwargs): + self.lock.acquire() + try: + return func(self, *args, **kwargs) + finally: + self.lock.release() + return nested + +def socket_exception(func): + "Decorator close object on socket.error." + def read(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except socket.error: + self.close() + return read + +class Connection: + """Basic class is represented connection. + + It can be in state: + WAIT_LEN --- connection is reading request len. + WAIT_MESSAGE --- connection is reading request. + WAIT_PROCESS --- connection has just read whole request and + waits for call ready routine. + SEND_ANSWER --- connection is sending answer string (including length + of answer). + CLOSED --- socket was closed and connection should be deleted. + """ + def __init__(self, new_socket, wake_up): + self.socket = new_socket + self.socket.setblocking(False) + self.status = WAIT_LEN + self.len = 0 + self.message = '' + self.lock = threading.Lock() + self.wake_up = wake_up + + def _read_len(self): + """Reads length of request. + + It's really paranoic routine and it may be replaced by + self.socket.recv(4).""" + read = self.socket.recv(4 - len(self.message)) + if len(read) == 0: + # if we read 0 bytes and self.message is empty, it means client close + # connection + if len(self.message) != 0: + logging.error("can't read frame size from socket") + self.close() + return + self.message += read + if len(self.message) == 4: + self.len, = struct.unpack('!i', self.message) + if self.len < 0: + logging.error("negative frame size, it seems client"\ + " doesn't use FramedTransport") + self.close() + elif self.len == 0: + logging.error("empty frame, it's really strange") + self.close() + else: + self.message = '' + self.status = WAIT_MESSAGE + + @socket_exception + def read(self): + """Reads data from stream and switch state.""" + assert self.status in (WAIT_LEN, WAIT_MESSAGE) + if self.status == WAIT_LEN: + self._read_len() + # go back to the main loop here for simplicity instead of + # falling through, even though there is a good chance that + # the message is already available + elif self.status == WAIT_MESSAGE: + read = self.socket.recv(self.len - len(self.message)) + if len(read) == 0: + logging.error("can't read frame from socket (get %d of %d bytes)" % + (len(self.message), self.len)) + self.close() + return + self.message += read + if len(self.message) == self.len: + self.status = WAIT_PROCESS + + @socket_exception + def write(self): + """Writes data from socket and switch state.""" + assert self.status == SEND_ANSWER + sent = self.socket.send(self.message) + if sent == len(self.message): + self.status = WAIT_LEN + self.message = '' + self.len = 0 + else: + self.message = self.message[sent:] + + @locked + def ready(self, all_ok, message): + """Callback function for switching state and waking up main thread. + + This function is the only function witch can be called asynchronous. + + The ready can switch Connection to three states: + WAIT_LEN if request was oneway. + SEND_ANSWER if request was processed in normal way. + CLOSED if request throws unexpected exception. + + The one wakes up main thread. + """ + assert self.status == WAIT_PROCESS + if not all_ok: + self.close() + self.wake_up() + return + self.len = '' + if len(message) == 0: + # it was a oneway request, do not write answer + self.message = '' + self.status = WAIT_LEN + else: + self.message = struct.pack('!i', len(message)) + message + self.status = SEND_ANSWER + self.wake_up() + + @locked + def is_writeable(self): + "Returns True if connection should be added to write list of select." + return self.status == SEND_ANSWER + + # it's not necessary, but... + @locked + def is_readable(self): + "Returns True if connection should be added to read list of select." + return self.status in (WAIT_LEN, WAIT_MESSAGE) + + @locked + def is_closed(self): + "Returns True if connection is closed." + return self.status == CLOSED + + def fileno(self): + "Returns the file descriptor of the associated socket." + return self.socket.fileno() + + def close(self): + "Closes connection" + self.status = CLOSED + self.socket.close() + +class TNonblockingServer: + """Non-blocking server.""" + def __init__(self, processor, lsocket, inputProtocolFactory=None, + outputProtocolFactory=None, threads=10): + self.processor = processor + self.socket = lsocket + self.in_protocol = inputProtocolFactory or TBinaryProtocolFactory() + self.out_protocol = outputProtocolFactory or self.in_protocol + self.threads = int(threads) + self.clients = {} + self.tasks = Queue.Queue() + self._read, self._write = socket.socketpair() + self.prepared = False + + def setNumThreads(self, num): + """Set the number of worker threads that should be created.""" + # implement ThreadPool interface + assert not self.prepared, "You can't change number of threads for working server" + self.threads = num + + def prepare(self): + """Prepares server for serve requests.""" + self.socket.listen() + for _ in xrange(self.threads): + thread = Worker(self.tasks) + thread.setDaemon(True) + thread.start() + self.prepared = True + + def wake_up(self): + """Wake up main thread. + + The server usualy waits in select call in we should terminate one. + The simplest way is using socketpair. + + Select always wait to read from the first socket of socketpair. + + In this case, we can just write anything to the second socket from + socketpair.""" + self._write.send('1') + + def _select(self): + """Does select on open connections.""" + readable = [self.socket.handle.fileno(), self._read.fileno()] + writable = [] + for i, connection in self.clients.items(): + if connection.is_readable(): + readable.append(connection.fileno()) + if connection.is_writeable(): + writable.append(connection.fileno()) + if connection.is_closed(): + del self.clients[i] + return select.select(readable, writable, readable) + + def handle(self): + """Handle requests. + + WARNING! You must call prepare BEFORE calling handle. + """ + assert self.prepared, "You have to call prepare before handle" + rset, wset, xset = self._select() + for readable in rset: + if readable == self._read.fileno(): + # don't care i just need to clean readable flag + self._read.recv(1024) + elif readable == self.socket.handle.fileno(): + client = self.socket.accept().handle + self.clients[client.fileno()] = Connection(client, self.wake_up) + else: + connection = self.clients[readable] + connection.read() + if connection.status == WAIT_PROCESS: + itransport = TTransport.TMemoryBuffer(connection.message) + otransport = TTransport.TMemoryBuffer() + iprot = self.in_protocol.getProtocol(itransport) + oprot = self.out_protocol.getProtocol(otransport) + self.tasks.put([self.processor, iprot, oprot, + otransport, connection.ready]) + for writeable in wset: + self.clients[writeable].write() + for oob in xset: + self.clients[oob].close() + del self.clients[oob] + + def close(self): + """Closes the server.""" + for _ in xrange(self.threads): + self.tasks.put([None, None, None, None, None]) + self.socket.close() + self.prepared = False + + def serve(self): + """Serve forever.""" + self.prepare() + while True: + self.handle() diff --git a/nebu/thrift/server/TServer.py b/nebu/thrift/server/TServer.py new file mode 100644 index 0000000..8456e2d --- /dev/null +++ b/nebu/thrift/server/TServer.py @@ -0,0 +1,274 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import logging +import sys +import os +import traceback +import threading +import Queue + +from thrift.Thrift import TProcessor +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol + +class TServer: + + """Base interface for a server, which must have a serve method.""" + + """ 3 constructors for all servers: + 1) (processor, serverTransport) + 2) (processor, serverTransport, transportFactory, protocolFactory) + 3) (processor, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory)""" + def __init__(self, *args): + if (len(args) == 2): + self.__initArgs__(args[0], args[1], + TTransport.TTransportFactoryBase(), + TTransport.TTransportFactoryBase(), + TBinaryProtocol.TBinaryProtocolFactory(), + TBinaryProtocol.TBinaryProtocolFactory()) + elif (len(args) == 4): + self.__initArgs__(args[0], args[1], args[2], args[2], args[3], args[3]) + elif (len(args) == 6): + self.__initArgs__(args[0], args[1], args[2], args[3], args[4], args[5]) + + def __initArgs__(self, processor, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory): + self.processor = processor + self.serverTransport = serverTransport + self.inputTransportFactory = inputTransportFactory + self.outputTransportFactory = outputTransportFactory + self.inputProtocolFactory = inputProtocolFactory + self.outputProtocolFactory = outputProtocolFactory + + def serve(self): + pass + +class TSimpleServer(TServer): + + """Simple single-threaded server that just pumps around one transport.""" + + def __init__(self, *args): + TServer.__init__(self, *args) + + def serve(self): + self.serverTransport.listen() + while True: + client = self.serverTransport.accept() + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + +class TThreadedServer(TServer): + + """Threaded server that spawns a new thread per each connection.""" + + def __init__(self, *args, **kwargs): + TServer.__init__(self, *args) + self.daemon = kwargs.get("daemon", False) + + def serve(self): + self.serverTransport.listen() + while True: + try: + client = self.serverTransport.accept() + t = threading.Thread(target = self.handle, args=(client,)) + t.setDaemon(self.daemon) + t.start() + except KeyboardInterrupt: + raise + except Exception, x: + logging.exception(x) + + def handle(self, client): + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + +class TThreadPoolServer(TServer): + + """Server with a fixed size pool of threads which service requests.""" + + def __init__(self, *args, **kwargs): + TServer.__init__(self, *args) + self.clients = Queue.Queue() + self.threads = 10 + self.daemon = kwargs.get("daemon", False) + + def setNumThreads(self, num): + """Set the number of worker threads that should be created""" + self.threads = num + + def serveThread(self): + """Loop around getting clients from the shared queue and process them.""" + while True: + try: + client = self.clients.get() + self.serveClient(client) + except Exception, x: + logging.exception(x) + + def serveClient(self, client): + """Process input/output from a client for as long as possible""" + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + + def serve(self): + """Start a fixed number of worker threads and put client into a queue""" + for i in range(self.threads): + try: + t = threading.Thread(target = self.serveThread) + t.setDaemon(self.daemon) + t.start() + except Exception, x: + logging.exception(x) + + # Pump the socket for clients + self.serverTransport.listen() + while True: + try: + client = self.serverTransport.accept() + self.clients.put(client) + except Exception, x: + logging.exception(x) + + +class TForkingServer(TServer): + + """A Thrift server that forks a new process for each request""" + """ + This is more scalable than the threaded server as it does not cause + GIL contention. + + Note that this has different semantics from the threading server. + Specifically, updates to shared variables will no longer be shared. + It will also not work on windows. + + This code is heavily inspired by SocketServer.ForkingMixIn in the + Python stdlib. + """ + + def __init__(self, *args): + TServer.__init__(self, *args) + self.children = [] + + def serve(self): + def try_close(file): + try: + file.close() + except IOError, e: + logging.warning(e, exc_info=True) + + + self.serverTransport.listen() + while True: + client = self.serverTransport.accept() + try: + pid = os.fork() + + if pid: # parent + # add before collect, otherwise you race w/ waitpid + self.children.append(pid) + self.collect_children() + + # Parent must close socket or the connection may not get + # closed promptly + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + try_close(itrans) + try_close(otrans) + else: + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + + ecode = 0 + try: + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, e: + logging.exception(e) + ecode = 1 + finally: + try_close(itrans) + try_close(otrans) + + os._exit(ecode) + + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + + def collect_children(self): + while self.children: + try: + pid, status = os.waitpid(0, os.WNOHANG) + except os.error: + pid = None + + if pid: + self.children.remove(pid) + else: + break + + diff --git a/nebu/thrift/server/__init__.py b/nebu/thrift/server/__init__.py new file mode 100644 index 0000000..1bf6e25 --- /dev/null +++ b/nebu/thrift/server/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TServer', 'TNonblockingServer'] diff --git a/nebu/thrift/transport/THttpClient.py b/nebu/thrift/transport/THttpClient.py new file mode 100644 index 0000000..5026978 --- /dev/null +++ b/nebu/thrift/transport/THttpClient.py @@ -0,0 +1,126 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TTransport import * +from cStringIO import StringIO + +import urlparse +import httplib +import warnings +import socket + +class THttpClient(TTransportBase): + + """Http implementation of TTransport base.""" + + def __init__(self, uri_or_host, port=None, path=None): + """THttpClient supports two different types constructor parameters. + + THttpClient(host, port, path) - deprecated + THttpClient(uri) + + Only the second supports https.""" + + if port is not None: + warnings.warn("Please use the THttpClient('http://host:port/path') syntax", DeprecationWarning, stacklevel=2) + self.host = uri_or_host + self.port = port + assert path + self.path = path + self.scheme = 'http' + else: + parsed = urlparse.urlparse(uri_or_host) + self.scheme = parsed.scheme + assert self.scheme in ('http', 'https') + if self.scheme == 'http': + self.port = parsed.port or httplib.HTTP_PORT + elif self.scheme == 'https': + self.port = parsed.port or httplib.HTTPS_PORT + self.host = parsed.hostname + self.path = parsed.path + if parsed.query: + self.path += '?%s' % parsed.query + self.__wbuf = StringIO() + self.__http = None + self.__timeout = None + + def open(self): + if self.scheme == 'http': + self.__http = httplib.HTTP(self.host, self.port) + else: + self.__http = httplib.HTTPS(self.host, self.port) + + def close(self): + self.__http.close() + self.__http = None + + def isOpen(self): + return self.__http != None + + def setTimeout(self, ms): + if not hasattr(socket, 'getdefaulttimeout'): + raise NotImplementedError + + if ms is None: + self.__timeout = None + else: + self.__timeout = ms/1000.0 + + def read(self, sz): + return self.__http.file.read(sz) + + def write(self, buf): + self.__wbuf.write(buf) + + def __withTimeout(f): + def _f(*args, **kwargs): + orig_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(args[0].__timeout) + result = f(*args, **kwargs) + socket.setdefaulttimeout(orig_timeout) + return result + return _f + + def flush(self): + if self.isOpen(): + self.close() + self.open(); + + # Pull data out of buffer + data = self.__wbuf.getvalue() + self.__wbuf = StringIO() + + # HTTP request + self.__http.putrequest('POST', self.path) + + # Write headers + self.__http.putheader('Host', self.host) + self.__http.putheader('Content-Type', 'application/x-thrift') + self.__http.putheader('Content-Length', str(len(data))) + self.__http.endheaders() + + # Write payload + self.__http.send(data) + + # Get reply to flush the request + self.code, self.message, self.headers = self.__http.getreply() + + # Decorate if we know how to timeout + if hasattr(socket, 'getdefaulttimeout'): + flush = __withTimeout(flush) diff --git a/nebu/thrift/transport/TSocket.py b/nebu/thrift/transport/TSocket.py new file mode 100644 index 0000000..d77e358 --- /dev/null +++ b/nebu/thrift/transport/TSocket.py @@ -0,0 +1,163 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TTransport import * +import os +import errno +import socket +import sys + +class TSocketBase(TTransportBase): + def _resolveAddr(self): + if self._unix_socket is not None: + return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, self._unix_socket)] + else: + return socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE | socket.AI_ADDRCONFIG) + + def close(self): + if self.handle: + self.handle.close() + self.handle = None + +class TSocket(TSocketBase): + """Socket implementation of TTransport base.""" + + def __init__(self, host='localhost', port=9090, unix_socket=None): + """Initialize a TSocket + + @param host(str) The host to connect to. + @param port(int) The (TCP) port to connect to. + @param unix_socket(str) The filename of a unix socket to connect to. + (host and port will be ignored.) + """ + + self.host = host + self.port = port + self.handle = None + self._unix_socket = unix_socket + self._timeout = None + + def setHandle(self, h): + self.handle = h + + def isOpen(self): + return self.handle != None + + def setTimeout(self, ms): + if ms is None: + self._timeout = None + else: + self._timeout = ms/1000.0 + + if (self.handle != None): + self.handle.settimeout(self._timeout) + + def open(self): + try: + res0 = self._resolveAddr() + for res in res0: + self.handle = socket.socket(res[0], res[1]) + self.handle.settimeout(self._timeout) + try: + self.handle.connect(res[4]) + except socket.error, e: + if res is not res0[-1]: + continue + else: + raise e + break + except socket.error, e: + if self._unix_socket: + message = 'Could not connect to socket %s' % self._unix_socket + else: + message = 'Could not connect to %s:%d' % (self.host, self.port) + raise TTransportException(type=TTransportException.NOT_OPEN, message=message) + + def read(self, sz): + try: + buff = self.handle.recv(sz) + except socket.error, e: + if (e.args[0] == errno.ECONNRESET and + (sys.platform == 'darwin' or sys.platform.startswith('freebsd'))): + # freebsd and Mach don't follow POSIX semantic of recv + # and fail with ECONNRESET if peer performed shutdown. + # See corresponding comment and code in TSocket::read() + # in lib/cpp/src/transport/TSocket.cpp. + self.close() + # Trigger the check to raise the END_OF_FILE exception below. + buff = '' + else: + raise + if len(buff) == 0: + raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket read 0 bytes') + return buff + + def write(self, buff): + if not self.handle: + raise TTransportException(type=TTransportException.NOT_OPEN, message='Transport not open') + sent = 0 + have = len(buff) + while sent < have: + plus = self.handle.send(buff) + if plus == 0: + raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket sent 0 bytes') + sent += plus + buff = buff[plus:] + + def flush(self): + pass + +class TServerSocket(TSocketBase, TServerTransportBase): + """Socket implementation of TServerTransport base.""" + + def __init__(self, port=9090, unix_socket=None): + self.host = None + self.port = port + self._unix_socket = unix_socket + self.handle = None + + def listen(self): + res0 = self._resolveAddr() + for res in res0: + if res[0] is socket.AF_INET6 or res is res0[-1]: + break + + # We need remove the old unix socket if the file exists and + # nobody is listening on it. + if self._unix_socket: + tmp = socket.socket(res[0], res[1]) + try: + tmp.connect(res[4]) + except socket.error, err: + eno, message = err.args + if eno == errno.ECONNREFUSED: + os.unlink(res[4]) + + self.handle = socket.socket(res[0], res[1]) + self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(self.handle, 'set_timeout'): + self.handle.set_timeout(None) + self.handle.bind(res[4]) + self.handle.listen(128) + + def accept(self): + client, addr = self.handle.accept() + result = TSocket() + result.setHandle(client) + return result diff --git a/nebu/thrift/transport/TTransport.py b/nebu/thrift/transport/TTransport.py new file mode 100644 index 0000000..12e51a9 --- /dev/null +++ b/nebu/thrift/transport/TTransport.py @@ -0,0 +1,331 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from cStringIO import StringIO +from struct import pack,unpack +from thrift.Thrift import TException + +class TTransportException(TException): + + """Custom Transport Exception class""" + + UNKNOWN = 0 + NOT_OPEN = 1 + ALREADY_OPEN = 2 + TIMED_OUT = 3 + END_OF_FILE = 4 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + +class TTransportBase: + + """Base class for Thrift transport layer.""" + + def isOpen(self): + pass + + def open(self): + pass + + def close(self): + pass + + def read(self, sz): + pass + + def readAll(self, sz): + buff = '' + have = 0 + while (have < sz): + chunk = self.read(sz-have) + have += len(chunk) + buff += chunk + + if len(chunk) == 0: + raise EOFError() + + return buff + + def write(self, buf): + pass + + def flush(self): + pass + +# This class should be thought of as an interface. +class CReadableTransport: + """base class for transports that are readable from C""" + + # TODO(dreiss): Think about changing this interface to allow us to use + # a (Python, not c) StringIO instead, because it allows + # you to write after reading. + + # NOTE: This is a classic class, so properties will NOT work + # correctly for setting. + @property + def cstringio_buf(self): + """A cStringIO buffer that contains the current chunk we are reading.""" + pass + + def cstringio_refill(self, partialread, reqlen): + """Refills cstringio_buf. + + Returns the currently used buffer (which can but need not be the same as + the old cstringio_buf). partialread is what the C code has read from the + buffer, and should be inserted into the buffer before any more reads. The + return value must be a new, not borrowed reference. Something along the + lines of self._buf should be fine. + + If reqlen bytes can't be read, throw EOFError. + """ + pass + +class TServerTransportBase: + + """Base class for Thrift server transports.""" + + def listen(self): + pass + + def accept(self): + pass + + def close(self): + pass + +class TTransportFactoryBase: + + """Base class for a Transport Factory""" + + def getTransport(self, trans): + return trans + +class TBufferedTransportFactory: + + """Factory transport that builds buffered transports""" + + def getTransport(self, trans): + buffered = TBufferedTransport(trans) + return buffered + + +class TBufferedTransport(TTransportBase,CReadableTransport): + + """Class that wraps another transport and buffers its I/O. + + The implementation uses a (configurable) fixed-size read buffer + but buffers all writes until a flush is performed. + """ + + DEFAULT_BUFFER = 4096 + + def __init__(self, trans, rbuf_size = DEFAULT_BUFFER): + self.__trans = trans + self.__wbuf = StringIO() + self.__rbuf = StringIO("") + self.__rbuf_size = rbuf_size + + def isOpen(self): + return self.__trans.isOpen() + + def open(self): + return self.__trans.open() + + def close(self): + return self.__trans.close() + + def read(self, sz): + ret = self.__rbuf.read(sz) + if len(ret) != 0: + return ret + + self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size))) + return self.__rbuf.read(sz) + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + out = self.__wbuf.getvalue() + # reset wbuf before write/flush to preserve state on underlying failure + self.__wbuf = StringIO() + self.__trans.write(out) + self.__trans.flush() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self.__rbuf + + def cstringio_refill(self, partialread, reqlen): + retstring = partialread + if reqlen < self.__rbuf_size: + # try to make a read of as much as we can. + retstring += self.__trans.read(self.__rbuf_size) + + # but make sure we do read reqlen bytes. + if len(retstring) < reqlen: + retstring += self.__trans.readAll(reqlen - len(retstring)) + + self.__rbuf = StringIO(retstring) + return self.__rbuf + +class TMemoryBuffer(TTransportBase, CReadableTransport): + """Wraps a cStringIO object as a TTransport. + + NOTE: Unlike the C++ version of this class, you cannot write to it + then immediately read from it. If you want to read from a + TMemoryBuffer, you must either pass a string to the constructor. + TODO(dreiss): Make this work like the C++ version. + """ + + def __init__(self, value=None): + """value -- a value to read from for stringio + + If value is set, this will be a transport for reading, + otherwise, it is for writing""" + if value is not None: + self._buffer = StringIO(value) + else: + self._buffer = StringIO() + + def isOpen(self): + return not self._buffer.closed + + def open(self): + pass + + def close(self): + self._buffer.close() + + def read(self, sz): + return self._buffer.read(sz) + + def write(self, buf): + self._buffer.write(buf) + + def flush(self): + pass + + def getvalue(self): + return self._buffer.getvalue() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self._buffer + + def cstringio_refill(self, partialread, reqlen): + # only one shot at reading... + raise EOFError() + +class TFramedTransportFactory: + + """Factory transport that builds framed transports""" + + def getTransport(self, trans): + framed = TFramedTransport(trans) + return framed + + +class TFramedTransport(TTransportBase, CReadableTransport): + + """Class that wraps another transport and frames its I/O when writing.""" + + def __init__(self, trans,): + self.__trans = trans + self.__rbuf = StringIO() + self.__wbuf = StringIO() + + def isOpen(self): + return self.__trans.isOpen() + + def open(self): + return self.__trans.open() + + def close(self): + return self.__trans.close() + + def read(self, sz): + ret = self.__rbuf.read(sz) + if len(ret) != 0: + return ret + + self.readFrame() + return self.__rbuf.read(sz) + + def readFrame(self): + buff = self.__trans.readAll(4) + sz, = unpack('!i', buff) + self.__rbuf = StringIO(self.__trans.readAll(sz)) + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + wout = self.__wbuf.getvalue() + wsz = len(wout) + # reset wbuf before write/flush to preserve state on underlying failure + self.__wbuf = StringIO() + # N.B.: Doing this string concatenation is WAY cheaper than making + # two separate calls to the underlying socket object. Socket writes in + # Python turn out to be REALLY expensive, but it seems to do a pretty + # good job of managing string buffer operations without excessive copies + buf = pack("!i", wsz) + wout + self.__trans.write(buf) + self.__trans.flush() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self.__rbuf + + def cstringio_refill(self, prefix, reqlen): + # self.__rbuf will already be empty here because fastbinary doesn't + # ask for a refill until the previous buffer is empty. Therefore, + # we can start reading new frames immediately. + while len(prefix) < reqlen: + self.readFrame() + prefix += self.__rbuf.getvalue() + self.__rbuf = StringIO(prefix) + return self.__rbuf + + +class TFileObjectTransport(TTransportBase): + """Wraps a file-like object to make it work as a Thrift transport.""" + + def __init__(self, fileobj): + self.fileobj = fileobj + + def isOpen(self): + return True + + def close(self): + self.fileobj.close() + + def read(self, sz): + return self.fileobj.read(sz) + + def write(self, buf): + self.fileobj.write(buf) + + def flush(self): + self.fileobj.flush() diff --git a/nebu/thrift/transport/TTwisted.py b/nebu/thrift/transport/TTwisted.py new file mode 100644 index 0000000..b6dcb4e --- /dev/null +++ b/nebu/thrift/transport/TTwisted.py @@ -0,0 +1,219 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from zope.interface import implements, Interface, Attribute +from twisted.internet.protocol import Protocol, ServerFactory, ClientFactory, \ + connectionDone +from twisted.internet import defer +from twisted.protocols import basic +from twisted.python import log +from twisted.web import server, resource, http + +from thrift.transport import TTransport +from cStringIO import StringIO + + +class TMessageSenderTransport(TTransport.TTransportBase): + + def __init__(self): + self.__wbuf = StringIO() + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + msg = self.__wbuf.getvalue() + self.__wbuf = StringIO() + self.sendMessage(msg) + + def sendMessage(self, message): + raise NotImplementedError + + +class TCallbackTransport(TMessageSenderTransport): + + def __init__(self, func): + TMessageSenderTransport.__init__(self) + self.func = func + + def sendMessage(self, message): + self.func(message) + + +class ThriftClientProtocol(basic.Int32StringReceiver): + + MAX_LENGTH = 2 ** 31 - 1 + + def __init__(self, client_class, iprot_factory, oprot_factory=None): + self._client_class = client_class + self._iprot_factory = iprot_factory + if oprot_factory is None: + self._oprot_factory = iprot_factory + else: + self._oprot_factory = oprot_factory + + self.recv_map = {} + self.started = defer.Deferred() + + def dispatch(self, msg): + self.sendString(msg) + + def connectionMade(self): + tmo = TCallbackTransport(self.dispatch) + self.client = self._client_class(tmo, self._oprot_factory) + self.started.callback(self.client) + + def connectionLost(self, reason=connectionDone): + for k,v in self.client._reqs.iteritems(): + tex = TTransport.TTransportException( + type=TTransport.TTransportException.END_OF_FILE, + message='Connection closed') + v.errback(tex) + + def stringReceived(self, frame): + tr = TTransport.TMemoryBuffer(frame) + iprot = self._iprot_factory.getProtocol(tr) + (fname, mtype, rseqid) = iprot.readMessageBegin() + + try: + method = self.recv_map[fname] + except KeyError: + method = getattr(self.client, 'recv_' + fname) + self.recv_map[fname] = method + + method(iprot, mtype, rseqid) + + +class ThriftServerProtocol(basic.Int32StringReceiver): + + MAX_LENGTH = 2 ** 31 - 1 + + def dispatch(self, msg): + self.sendString(msg) + + def processError(self, error): + self.transport.loseConnection() + + def processOk(self, _, tmo): + msg = tmo.getvalue() + + if len(msg) > 0: + self.dispatch(msg) + + def stringReceived(self, frame): + tmi = TTransport.TMemoryBuffer(frame) + tmo = TTransport.TMemoryBuffer() + + iprot = self.factory.iprot_factory.getProtocol(tmi) + oprot = self.factory.oprot_factory.getProtocol(tmo) + + d = self.factory.processor.process(iprot, oprot) + d.addCallbacks(self.processOk, self.processError, + callbackArgs=(tmo,)) + + +class IThriftServerFactory(Interface): + + processor = Attribute("Thrift processor") + + iprot_factory = Attribute("Input protocol factory") + + oprot_factory = Attribute("Output protocol factory") + + +class IThriftClientFactory(Interface): + + client_class = Attribute("Thrift client class") + + iprot_factory = Attribute("Input protocol factory") + + oprot_factory = Attribute("Output protocol factory") + + +class ThriftServerFactory(ServerFactory): + + implements(IThriftServerFactory) + + protocol = ThriftServerProtocol + + def __init__(self, processor, iprot_factory, oprot_factory=None): + self.processor = processor + self.iprot_factory = iprot_factory + if oprot_factory is None: + self.oprot_factory = iprot_factory + else: + self.oprot_factory = oprot_factory + + +class ThriftClientFactory(ClientFactory): + + implements(IThriftClientFactory) + + protocol = ThriftClientProtocol + + def __init__(self, client_class, iprot_factory, oprot_factory=None): + self.client_class = client_class + self.iprot_factory = iprot_factory + if oprot_factory is None: + self.oprot_factory = iprot_factory + else: + self.oprot_factory = oprot_factory + + def buildProtocol(self, addr): + p = self.protocol(self.client_class, self.iprot_factory, + self.oprot_factory) + p.factory = self + return p + + +class ThriftResource(resource.Resource): + + allowedMethods = ('POST',) + + def __init__(self, processor, inputProtocolFactory, + outputProtocolFactory=None): + resource.Resource.__init__(self) + self.inputProtocolFactory = inputProtocolFactory + if outputProtocolFactory is None: + self.outputProtocolFactory = inputProtocolFactory + else: + self.outputProtocolFactory = outputProtocolFactory + self.processor = processor + + def getChild(self, path, request): + return self + + def _cbProcess(self, _, request, tmo): + msg = tmo.getvalue() + request.setResponseCode(http.OK) + request.setHeader("content-type", "application/x-thrift") + request.write(msg) + request.finish() + + def render_POST(self, request): + request.content.seek(0, 0) + data = request.content.read() + tmi = TTransport.TMemoryBuffer(data) + tmo = TTransport.TMemoryBuffer() + + iprot = self.inputProtocolFactory.getProtocol(tmi) + oprot = self.outputProtocolFactory.getProtocol(tmo) + + d = self.processor.process(iprot, oprot) + d.addCallback(self._cbProcess, request, tmo) + return server.NOT_DONE_YET diff --git a/nebu/thrift/transport/__init__.py b/nebu/thrift/transport/__init__.py new file mode 100644 index 0000000..02c6048 --- /dev/null +++ b/nebu/thrift/transport/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TTransport', 'TSocket', 'THttpClient'] diff --git a/nebu/tsv_parser.py b/nebu/tsv_parser.py new file mode 100644 index 0000000..a7101ce --- /dev/null +++ b/nebu/tsv_parser.py @@ -0,0 +1,44 @@ +import sys +from django.utils import simplejson as json + +def help(): + print """ +# tsv_parser.py +# parses a tsv file, and dumps it as JSON. +# +# This is useful to create files for populator.py from FreeBase dumps + +# Usage: +tsv_parser.py +""" + + +if __name__ == "__main__": + if not len(sys.argv) == 2: + help() + sys.exit(-1) + + file = open(sys.argv[1], 'r') + + header = file.next() + header = header.replace('\n', '') + + field_names = header.split('\t') + + for line in file: + line = line.replace('\n', '') + fields = {} + values = line.split('\t') + + for i in range(len(values)): + fields[field_names[i]] = values[i] + + final_map = {} + + final_map['docid'] = fields['docid'] + del fields['docid'] + + final_map['fields'] = fields + + print json.dumps(final_map) + diff --git a/nebu/upgrade_frontend.sh b/nebu/upgrade_frontend.sh new file mode 100755 index 0000000..e9dbe67 --- /dev/null +++ b/nebu/upgrade_frontend.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# +# This script is intended to be executed after updating +# the "nebu" folder of a frontend instance. +# It should be executed while the frontend nebu processes are down: +# - worker manager +# - deploy manager +# - supervisor +# It will then issue commands to every worker so +# they can update their nebu installations from this +# frontend and restart their controllers. +# +# author: santip +# + +export DJANGO_SETTINGS_MODULE=settings +export PYTHONPATH=../:. + +python upgrade_workers.py diff --git a/nebu/upgrade_workers.py b/nebu/upgrade_workers.py new file mode 100644 index 0000000..9c183d5 --- /dev/null +++ b/nebu/upgrade_workers.py @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# +# This script is used from the upgrade_frontend.sh +# to issue commands to every worker so they can update +# their nebu installations from this frontend and +# restart their controllers. +# +# author: santip +# + +from nebu.models import Worker +import rpc +import socket +from thrift.transport import TTransport + +for w in Worker.objects.all(): + print 'Upgrading worker %d at %s' % (w.id, w.wan_dns) + dns = w.lan_dns + controller = rpc.getThriftControllerClient(dns) + host = socket.gethostbyname_ex(socket.gethostname())[0] + retcode = controller.update_worker(host) + if retcode == 0: + try: + print 'Worker %s updated. Restarting...' % dns + controller.restart_controller() + print "Restart controller didn't throw an exception. Did it restart?" + except TTransport.TTransportException: + # restart will always fail + pass +print 'Done' diff --git a/nebu/upgrade_workers.sh b/nebu/upgrade_workers.sh new file mode 100755 index 0000000..e9dbe67 --- /dev/null +++ b/nebu/upgrade_workers.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# +# This script is intended to be executed after updating +# the "nebu" folder of a frontend instance. +# It should be executed while the frontend nebu processes are down: +# - worker manager +# - deploy manager +# - supervisor +# It will then issue commands to every worker so +# they can update their nebu installations from this +# frontend and restart their controllers. +# +# author: santip +# + +export DJANGO_SETTINGS_MODULE=settings +export PYTHONPATH=../:. + +python upgrade_workers.py diff --git a/nebu/worker_manager.py b/nebu/worker_manager.py new file mode 100644 index 0000000..c7e6a90 --- /dev/null +++ b/nebu/worker_manager.py @@ -0,0 +1,178 @@ +#!/usr/bin/python + +from amazon_credential import AMAZON_USER, AMAZON_PASSWORD + +from flaptor.indextank.rpc import WorkerManager as TWorkerManager +from nebu.models import Worker + +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol +from thrift.server import TServer + +import boto, time +import socket +from lib import flaptor_logging, mail +import rpc + +IMAGE_ID = 'ami-c6fa07af' + +logger = flaptor_logging.get_logger('WorkerMgr') + +def logerrors(func): + def decorated(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception, e: + logger.exception("Failed while executing %s", func.__name__) + raise e + return decorated + + +class WorkerManager: + + @logerrors + def ec2_connection(self): + return boto.connect_ec2(AMAZON_USER, AMAZON_PASSWORD) + + """ Return values for add_worker """ + WORKER_CREATION_FAILED = 0 + WORKER_CREATED = 1 + + @logerrors + def add_worker(self, instance_type): + ''' table of instance_type -> available ram. ''' + AVAILABLE_RAM = { + 'm1.large': 7500, + 'm1.xlarge' : 15000, + 'm2.xlarge' : 17000, + 'm2.2xlarge' : 34000, + 'm2.4xlarge' : 68000, + } + if not instance_type in AVAILABLE_RAM: + logger.error("instance type %s is not on AVAILABLE_RAM table. Choose another type or update the table.") + return WorkerManager.WORKER_CREATION_FAILED + + logger.info("Creating new worker using image %s, using a %s instance", IMAGE_ID, instance_type) + conn = self.ec2_connection() + res = conn.run_instances(image_id=IMAGE_ID, security_groups=['indextank-worker'], instance_type=instance_type, placement='us-east-1a') + if len(res.instances) == 0: + logger.error("New instance failed") + return WorkerManager.WORKER_CREATION_FAILED + else: + w = Worker() + w.status = Worker.States.created + w.instance_name = res.instances[0].id + w.ram = AVAILABLE_RAM[instance_type] + w.save() + ''' + UNCOMMENT ME WHEN WE UPDATE BOTO'S VERSION + conn.create_tags(res, {'Name': 'Worker:%i' % (w.id)}) + ''' + mail.report_new_worker(w) + return WorkerManager.WORKER_CREATED + + """ Return values for update_status """ + WORKER_CONTROLLABLE = 0 + WORKER_UPDATING = 1 + WORKER_INITIALIZING = 2 + WORKER_NOT_READY = 3 + + @logerrors + def update_status(self, instance_name): + worker = Worker.objects.filter(instance_name=instance_name)[0] + if worker.status == Worker.States.created: + conn = self.ec2_connection() + reservations = conn.get_all_instances([instance_name]) + instance = reservations[0].instances[0] + if instance.state == 'running': + logger.info('Worker %s is now initializing: %s', worker.instance_name, instance.public_dns_name) + worker.status = Worker.States.initializing + worker.lan_dns = instance.private_dns_name + worker.wan_dns = instance.public_dns_name + worker.save() + mail.report_new_worker(worker) + else: + logger.debug('Worker %s is still reporting as %s', worker.instance_name, instance.state) + return WorkerManager.WORKER_NOT_READY + + if worker.status == Worker.States.initializing: + logger.debug('Trying to update controller on %s', worker.instance_name) + if self.update_worker(worker.lan_dns): + worker.status = Worker.States.updating + worker.save() + logger.info('Worker %s is now updating', worker.instance_name) + return WorkerManager.WORKER_UPDATING + else: + return WorkerManager.WORKER_INITIALIZING + + if worker.status == Worker.States.updating: + logger.debug('Checking if controller is up on %s', worker.instance_name) + try: + controller = rpc.getThriftControllerClient(worker.lan_dns) + controller.get_worker_load_stats() + worker.status = Worker.States.controllable + worker.save() + logger.info('Worker %s is now controllable', worker.instance_name) + return WorkerManager.WORKER_CONTROLLABLE + except Exception, e: + if isinstance(e, TTransport.TTransportException) and e.type == TTransport.TTransportException.NOT_OPEN: + logger.info('Controller on worker %s not responding yet.', worker.lan_dns) + else: + logger.exception('Unexpected exception while checking worker %s', worker.lan_dns) + return WorkerManager.WORKER_UPDATING + + @logerrors + def update_worker(self, dns): + try: + controller = rpc.getThriftControllerClient(dns) + host = socket.gethostbyname_ex(socket.gethostname())[0] + retcode = controller.update_worker(host) + if retcode == 0: + try: + logger.debug('Worker %s updated. Restarting...', dns) + controller.restart_controller() + logger.warn("Restart controller didn't throw an exception. Did it restart?") + except TTransport.TTransportException: + # restart will always fail + pass + except Exception, e: + if isinstance(e, TTransport.TTransportException) and e.type == TTransport.TTransportException.NOT_OPEN: + logger.info('Controller on worker %s not responding yet.', dns) + else: + logger.exception('Unexpected exception while updating worker %s', dns) + return False + return True + + @logerrors + def remove_worker(self,instance_name): + print 'removing host' + return 1 + + + # TODO this method should be called periodically + @logerrors + def poll_controllers(self): + for worker in Worker.objects.all(): + controller = rpc.getThriftControllerClient(worker.lan_dns) + if controller: + stats = controller.stats() + print controller,stats + # TODO update worker stats. + else: + print "could not connect to controller on %s" % worker + + +if __name__ == "__main__": + handler = WorkerManager() + processor = TWorkerManager.Processor(handler) + transport = TSocket.TServerSocket(rpc.worker_manager_port) + tfactory = TTransport.TBufferedTransportFactory() + pfactory = TBinaryProtocol.TBinaryProtocolFactory() + + server = TServer.TThreadPoolServer(processor, transport, tfactory, pfactory) + print 'Starting the server...' + server.serve() + print 'done.' + + diff --git a/nebu/worker_manager_client.py b/nebu/worker_manager_client.py new file mode 100644 index 0000000..09a0231 --- /dev/null +++ b/nebu/worker_manager_client.py @@ -0,0 +1,16 @@ +from rpc import getThriftWorkerManagerClient +import sys + + +# Missing a way to close transport + +if __name__ == '__main__': + client = getThriftWorkerManagerClient('workermanager') + if len(sys.argv) > 1: + itype = sys.argv[1] + retcode = client.add_worker(itype) + print "Finished adding worker of type [%s] : %s " % (itype, retcode) + else: + retcode = client.add_worker() + print "Finished adding worker: %s" % retcode + diff --git a/storefront/.gitignore b/storefront/.gitignore new file mode 100644 index 0000000..9fb6d6c --- /dev/null +++ b/storefront/.gitignore @@ -0,0 +1 @@ +localdb.sqlite diff --git a/storefront/.project b/storefront/.project new file mode 100644 index 0000000..5e316c4 --- /dev/null +++ b/storefront/.project @@ -0,0 +1,18 @@ + + + storefront + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + org.python.pydev.django.djangoNature + + diff --git a/storefront/.pydevproject b/storefront/.pydevproject new file mode 100644 index 0000000..176c816 --- /dev/null +++ b/storefront/.pydevproject @@ -0,0 +1,19 @@ + + + + +Default +python 2.6 + +DJANGO_MANAGE_LOCATION +manage.py +DJANGO_SETTINGS_MODULE +storefront.settings + + +/storefront + + +../ + + diff --git a/storefront/__init__.py b/storefront/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/storefront/amazon_credential.py b/storefront/amazon_credential.py new file mode 100644 index 0000000..c2910f5 --- /dev/null +++ b/storefront/amazon_credential.py @@ -0,0 +1,2 @@ +AMAZON_USER = "" +AMAZON_PASSWORD = "" diff --git a/storefront/api_linked_models.py b/storefront/api_linked_models.py new file mode 100644 index 0000000..0908f60 --- /dev/null +++ b/storefront/api_linked_models.py @@ -0,0 +1,938 @@ +import hashlib +import random +import binascii + +from lib.indextank.client import ApiClient, IndexAlreadyExists +from lib.authorizenet import AuthorizeNet, BillingException + +from django.db import models +from django.contrib.auth.models import User +from django.utils import simplejson as json +from django.db import IntegrityError +from django.db.models.aggregates import Sum, Count + +from lib import encoder, flaptor_logging + +from django.conf import settings +from datetime import datetime + +logger = flaptor_logging.get_logger('Models') + +# idea taken from https://www.grc.com/passwords.htm +def generate_apikey(id): + key = "2A1A8AE7CAEFAC47D6F74920CE4B0CE46430CDA6CF03D254C1C29402D727E570" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:14] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return unique_part + '-' + random_part + +def generate_onetimepass(id): + key = "CAEFAC47D6F7D727E57024920CE4B0CE46430CDA6CF03D254C1C29402A1A8AE7" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:5] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return unique_part + random_part + +def generate_forgotpass(id): + key = "E57024920CE4B0CE4643CAEFAC47D6F7D7270CDA6CF03D254C1C29402A1A8AE7" + while True: + hash = hashlib.md5() + hash.update('%d' % id) + hash.update(key) + hash.update('%d' % random.randint(0,1000000)) + random_part = binascii.b2a_base64(hash.digest())[:6] + if not '/' in random_part: + break + + unique_part = encoder.to_key(id) + + return random_part + unique_part + + +# StoreFront models +class Account(models.Model): + apikey = models.CharField(max_length=22, unique=True) + creation_time = models.DateTimeField() + package = models.ForeignKey('Package', null=True) + status = models.CharField(max_length=30, null=False) + provisioner = models.ForeignKey('Provisioner', null=True) + + configuration = models.ForeignKey('IndexConfiguration', null=True) + default_analyzer = models.ForeignKey('Analyzer', null=True, related_name="accounts") + + class Statuses: + operational = 'OPERATIONAL' + creating = 'CREATING' + closed = 'CLOSED' + + def __repr__(self): + return 'Account (%s):\n\tuser_email: %s\n\tapikey: %s\n\tcreation_time: %s\n\tstatus: %s\n\tpackage: %s\n\tconfiguration: %s\n' % (self.id, PFUser.objects.filter(account=self)[0].email, str(self.apikey), str(self.creation_time), str(self.status), self.package.name, self.configuration.description) + + def __str__(self): + return '(apikey: %s; creation_time: %s; status: %s)' % (str(self.apikey), str(self.creation_time), str(self.status)) + + def count_indexes(self): + return self.indexes.aggregate(cnt=Count('id'))['cnt'] + + def count_documents(self): + return self.indexes.aggregate(cnt=Sum('current_docs_number'))['cnt'] + + def is_operational(self): + return self.status == Account.Statuses.operational + + def is_heroku(self): + # HACK UNTIL HEROKU IS A PROVISIONER + return self.package.code.startswith('HEROKU_') + #return self.provisioner and self.provisioner.name == 'heroku' + + @classmethod + def create_account(cls, dt, email=None, password=None): + account = Account() + + account.creation_time = datetime.now() + account.status = Account.Statuses.creating + account.save() + + account.apikey = generate_apikey(account.id) + account.save() + + unique_part, random_part = account.apikey.split('-', 1) + if email is None: + email = '%s@indextank.com' % unique_part + + if password is None: + password = random_part + + try: + user = User.objects.create_user(email, '', password) + except IntegrityError, e: + account.delete() + raise e + + try: + pfu = PFUser() + pfu.account = account + pfu.user = user + pfu.email = email + + pfu.save() + except IntegrityError, e: + account.delete() + user.delete() + raise e + + return account, pfu + + def create_index(self, index_name, public_search=None): + index = Index() + + # basic index data + index.populate_for_account(self) + index.name = index_name + index.creation_time = datetime.now() + index.language_code = 'en' + index.status = Index.States.new + if not public_search is None: + index.public_api = public_search + + # test for name uniqueness + # raises IntegrityError if the index name already exists + index.save() + + # define the default function + function = ScoreFunction() + function.index = index + function.name = '0' + function.definition = '-age' + function.save() + + # deduce code from id + index.code = encoder.to_key(index.id) + index.save() + + return index + + def create_demo_index(self): + try: + dataset = DataSet.objects.get(code='DEMO') + except DataSet.DoesNotExist: + logger.exception('DemoIndex dataset not present in database. Aborting demo index creation') + return + + index = self.create_index('DemoIndex') + + index.public_api = True + index.save() + + population = IndexPopulation() + population.index = index + population.status = IndexPopulation.Statuses.created + population.dataset = dataset + population.time = datetime.now() + population.populated_size = 0 + + population.save() + + def close(self): + # Dropping an account implies: + + # - removing the payment information from the account + # - removing the subscriptions from authorize.net + for info in self.payment_informations.all(): + auth = AuthorizeNet() + for subscription in info.subscriptions.all(): + auth.subscription_cancel(subscription.reference_id, subscription.subscription_id) + subscription.delete() + info.delete() + + + # - changing the status to CLOSED + self.status = Account.Statuses.closed + + # - removing and stopping the indexes for the account + for index in self.indexes.all(): + self.drop_index(index) + + # - notify + # send_notification(//close account) + + # - FIXME: handle authorize net errors! + + + self.save() + + def drop_index(self, index): + client = ApiClient(self.get_private_apiurl()) + client.delete_index(index.name) + + def apply_package(self, package): + self.package = package + + self.configuration = package.configuration + + def update_apikey(self): + self.apikey = generate_apikey(self.id) + + def get_private_apikey(self): + return self.apikey.split('-', 1)[1] + + def get_public_apikey(self): + return self.apikey.split('-', 1)[0] + + def get_private_apiurl(self): + return 'http://:%s@%s.api.indextank.com' % (self.get_private_apikey(), self.get_public_apikey()) + + def get_public_apiurl(self): + return 'http://%s.api.indextank.com' % self.get_public_apikey() + + class Meta: + db_table = 'storefront_account' + +class AccountPayingInformation(models.Model): + account = models.ForeignKey('Account', related_name='payment_informations') + + first_name = models.CharField(max_length=50, null=True) + last_name = models.CharField(max_length=50, null=True) + address = models.CharField(max_length=60, null=True) + city = models.CharField(max_length=60, null=True) + state = models.CharField(max_length=2, null=True) + zip_code = models.CharField(max_length=60, null=True) + country = models.CharField(max_length=2, null=True) + + company = models.CharField(max_length=50, null=True) + + credit_card_last_digits = models.CharField(max_length=4, null=True) + contact_email = models.EmailField(max_length=255, null=True) + + #custom subscription + monthly_amount = models.DecimalField(max_digits=8, decimal_places=2, null=True) + subscription_status = models.CharField(max_length=30, null=True) + subscription_type = models.CharField(max_length=30, null=True) + + + class Meta: + db_table = 'storefront_accountpayinginformation' + + +class PaymentSubscription(models.Model): + account = models.ForeignKey('AccountPayingInformation', related_name='subscriptions') + + # authorizenet id + subscription_id = models.CharField(max_length=20, null=False, blank=False) + # indextank id + reference_id = models.CharField(max_length=13, null=False, blank=False) + + amount = models.DecimalField(max_digits=8, decimal_places=2, null=False) + + # Frequency + start_date = models.DateTimeField() + frequency_length = models.IntegerField(null=False) + frequency_unit = models.CharField(max_length=10, null=False, blank=False) + + class Meta: + db_table = 'storefront_paymentsubscription' + + +class EffectivePayment(models.Model): + account = models.ForeignKey('AccountPayingInformation', related_name='payments') + + transaction_date = models.DateTimeField() + + # authorizenet data + transaction_id = models.CharField(max_length=12, null=False, blank=False) + customer_id = models.CharField(max_length=8, null=False, blank=False) + transaction_message = models.CharField(max_length=300, null=True) + subscription_id = models.CharField(max_length=20, null=False, blank=False) + subscription_payment_number = models.IntegerField(null=False) + first_name = models.CharField(max_length=50, null=False, blank=False) + last_name = models.CharField(max_length=50, null=False, blank=False) + address = models.CharField(max_length=60, null=True) + city = models.CharField(max_length=60, null=True) + state = models.CharField(max_length=2, null=True) + zip_code = models.CharField(max_length=60, null=True) + country = models.CharField(max_length=2, null=True) + company = models.CharField(max_length=50, null=True) + + # Inherited data (from account information + credit_card_last_digits = models.CharField(max_length=4, null=False, blank=False) + contact_email = models.EmailField(max_length=255) + + amount = models.DecimalField(max_digits=8, decimal_places=2, null=False) + + class Meta: + db_table = 'storefront_effectivepayment' + +class DataSet(models.Model): + name = models.CharField(null=True, max_length=50, unique=True) + code = models.CharField(null=True, max_length=15, unique=True) + filename = models.CharField(null=True, max_length=100, unique=True) + size = models.IntegerField(default=0) + + class Meta: + db_table = 'storefront_dataset' + +class IndexPopulation(models.Model): + index = models.ForeignKey('Index', related_name='datasets') + dataset = models.ForeignKey('DataSet', related_name='indexes') + time = models.DateTimeField() + populated_size = models.IntegerField(default=0) + + status = models.CharField(max_length=50,null=True) + + class Statuses: + created = 'CREATED' + populating = 'POPULATING' + finished = 'FINISHED' + + class Meta: + db_table = 'storefront_indexpopulation' + + +class Index(models.Model): + account = models.ForeignKey('Account', related_name='indexes') + code = models.CharField(null=True, max_length=22, unique=True) + name = models.CharField(max_length=50) + language_code = models.CharField(max_length=2) + creation_time = models.DateTimeField() + + analyzer_config = models.TextField(null=True) + configuration = models.ForeignKey('IndexConfiguration', null=True) + public_api = models.BooleanField(default=False, null=False) + + status = models.CharField(max_length=50) + + deleted = models.BooleanField(default=False, null=False) + + class States: + new = 'NEW' + live = 'LIVE' + hibernate_requested = 'HIBERNATE_REQUESTED' + hibernated = 'HIBERNATED' + waking_up = 'WAKING_UP' + + def get_json_for_analyzer(self): + if self.analyzer_config is None: + return None + configuration = json.loads(self.analyzer_config) + final_configuration = {} + + if configuration.has_key('per_field'): + per_field_final = {} + per_field = configuration.get('per_field') + for field in per_field.keys(): + per_field_final[field] = Index.get_analyzer(per_field[field]) + final_configuration['perField'] = per_field_final + final_configuration['default'] = Index.get_analyzer(per_field.get('default')) + else: + final_configuration = Index.get_analyzer(configuration) + + return final_configuration + + @classmethod + def get_analyzer(cls, configuration): + analyzer_map = {} + code = configuration.get('code') + if code is None: + raise ValueError('Analyzer configuration has no "code" key') + + try: + analyzer = AnalyzerComponent.objects.get(code=code) + except AnalyzerComponent.DoesNotExist: + raise ValueError('Analyzer configuration "code" key doesn\'t match any analyzers') + + analyzer_map['factory'] = analyzer.factory + analyzer_map['configuration'] = json.loads(analyzer.config) + + if configuration.has_key('filters'): + filters_list = [] + for filter in configuration.get('filters'): + filters_list.append(Index.get_analyzer(filter)) + analyzer_map['configuration']['filters'] = filters_list + + return analyzer_map + +# allows_adds = models.BooleanField(null=False,default=True) +# allows_queries = models.BooleanField(null=False,default=True) + + # index creation data +# allows_snippets = models.BooleanField() +# +# allows_autocomplete = models.BooleanField(default=True) +# autocomplete_type = models.models.CharField(max_length=10, null=True) # NEW +# +# allows_faceting = models.BooleanField() +# facets_bits = models.IntegerField(null=True) # NEW +# +# max_variables = models.IntegerField(null=False) # NEW +# +# max_memory_mb = models.IntegerField(null=False) # NEW +# rti_documents_number = models.IntegerField(null=False) # NEW + + # statistics + current_size = models.FloatField(default=0) + current_docs_number = models.IntegerField(default=0) + queries_per_day = models.FloatField(default=0) + + #demo + base_port = models.IntegerField(null=True) + + def __repr__(self): + return 'Index (%s):\n\tname: %s\n\tcode: %s\n\tcreation_time: %s\n\tconfiguration: %s\n\taccount\'s package: %s\ncurrent deploys: %r' % (self.id, self.name, self.code, self.creation_time, self.configuration.description, self.account.package.name, self.deploys.all()) + + def is_populating(self): + for population in self.datasets.all(): + if not population.status == IndexPopulation.Statuses.finished: + return True + return False + + def is_demo(self): + return self.name == 'DemoIndex' and self.datasets.count() > 0 + + + def is_ready(self): + ''' + Returns True if the end-user can use the index. + (this means for read and write, and it's meant to + be shown in the storefront page). Internally, this + means that at least one deployment for this index + is readable, and at least one is writable. + ''' + return self.is_writable() and self.is_readable() + + def is_hibernated(self): + return self.status in (Index.States.hibernated, Index.States.waking_up) + + def is_writable(self): + ''' + Returns true if there's at least one index that can be written. + ''' + for deploy in self.deploys.all(): + if deploy.is_writable(): + return True + + def is_readable(self): + ''' + Returns true if there's at least one index that can be read. + ''' + for deploy in self.deploys.all(): + if deploy.is_readable(): + return True + + def populate_for_account(self, account): + self.account = account + self.configuration = account.configuration + if account.default_analyzer is not None: + self.analyzer_config = account.default_analyzer.configuration + + def searchable_deploy(self): + '''Returns a single deploy that can be used to search. If no deploy is searcheable + it returns None. Note that if more than one deploy is searcheable, there are no warranties + of wich one will be returned.''' + # TODO : should change once deploy roles are implemented + ds = self.deploys.all() + ds = [d for d in ds if d.is_readable()] + return ds[0] if ds else None + + def indexable_deploys(self): + '''Returns the list of all deploys that should be updated (adds/updates/deletes/etc)''' + # TODO : should change once deploy roles are implemented + ds = self.deploys.all() + ds = [d for d in ds if d.is_writable()] + return ds + + def get_functions_dict(self): + return dict((str(f.name), f.definition) for f in self.scorefunctions.all()) + + def get_debug_info(self): + info = 'Index: %s [%s]\n' % (self.name, self.code) +\ + 'Account: %s\n' % self.account.user.email +\ + 'Deploys:\n' + for d in self.deploys.all(): + info += ' [deploy:%d] %s on [worker:%s] %s:%s' % (d.id, d.status, d.worker.id, d.worker.wan_dns, d.base_port) + return info + + def update_status(self, new_status): + print 'updating status %s for %r' % (new_status, self) + logger.debug('Updating status to %s for idnex %r', new_status, self) + Index.objects.filter(id=self.id).update(status=new_status) + + def mark_deleted(self): + Index.objects.filter(id=self.id).update(deleted=True) + + class AutocompleTypes: + created = 'DOCUMENTS' + initializing = 'QUERIES' + + class Meta: + unique_together = (('code','account'),('name','account')) + db_table = 'storefront_index' + +class Insight(models.Model): + index = models.ForeignKey(Index, related_name='insights') + code = models.CharField(max_length=30, null=False) + data = models.TextField(null=False) + last_update = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = ('index', 'code') + db_table = 'storefront_insight' + +class IndexConfiguration(models.Model): + description = models.TextField(null=False) + creation_date = models.DateField() + json_configuration = models.TextField(null=False) + + def __repr__(self): + j_map = json.loads(self.json_configuration) + mapStr = '{\n' + for m in j_map: + mapStr += '\t\t%s -> %s\n' % (m, j_map[m]) + mapStr += '\t}\n' + return 'IndexConfiguration (%s):\n\tdescription: %s\n\tcreation_date: %s\n\tjson_configuration: %s\n' % (self.id, self.description, str(self.creation_date), mapStr) + + def __str__(self): + return '(description: %s; creation_date: %s; json_configuration: %s)' % (self.description, str(self.creation_date), self.json_configuration) + + def get_data(self): + map = json.loads(self.json_configuration) + data = {} + for k,v in map.items(): + data[str(k)] = v + data['ram'] = data.get('xmx',0) + data.get('bdb_cache',0) + return data + def set_data(self, data): + self.json_configuration = json.dumps(data) + + class Meta: + db_table = 'storefront_indexconfiguration' + +class Analyzer(models.Model): + account = models.ForeignKey('Account', related_name='analyzers') + code = models.CharField(max_length=64) + configuration = models.TextField() + + class Meta: + db_table = 'storefront_analyzer' + +class AnalyzerComponent(models.Model): + code = models.CharField(max_length=15, unique=True) + name = models.CharField(max_length=200) + description = models.CharField(max_length=1000) + config = models.TextField(null=False,blank=False) + factory = models.CharField(max_length=200) + type = models.CharField(max_length=20) + enabled = models.BooleanField() + + class Types: + tokenizer = 'TOKENIZER' + filter = 'FILTER' + + class Meta: + db_table = 'storefront_analyzercomponent' + +def create_analyzer(code, name, config, factory, type, enabled): + analyzer = None + try: + analyzer = AnalyzerComponent.objects.get(code=code) + + analyzer.name = name + analyzer.config = config + analyzer.factory = factory + analyzer.type = type + analyzer.enabled = enabled + + analyzer.save() + except AnalyzerComponent.DoesNotExist: + analyzer = AnalyzerComponent(code=code, name=name, config=config, type=type, enabled=enabled) + analyzer.save() + +class Package(models.Model): + ''' + Packages define what a user have the right to when creating an Account and how does the indexes in that Account + behave. + There are two sections for what the Package configures. A fixed section with the control and limits information + that is used by nebu, storefront or api (base_price, index_max_size, searches_per_day, max_indexes). A dynamic + section that is handled by the IndexConfiguration object. The information of that section is passed to the IndexEngine + as it is and handled by it. + ''' + name = models.CharField(max_length=50) + code = models.CharField(max_length=30) + base_price = models.FloatField() + index_max_size = models.IntegerField() + searches_per_day = models.IntegerField() + max_indexes = models.IntegerField() + + configuration = models.ForeignKey('IndexConfiguration', null=True) + + def __repr__(self): + return 'Package (%s):\n\tname: %s\n\tcode: %s\n\tbase_price: %.2f\n\tindex_max_size: %i\n\tsearches_per_day: %i\n\tmax_indexes: %i\n' % (self.id, self.name, self.code, self.base_price, self.index_max_size, self.searches_per_day, self.max_indexes) + + def __str__(self): + return '(name: %s; code: %s; base_price: %.2f; index_max_size: %i; searches_per_day: %i; max_indexes: %i)' % (self.name, self.code, self.base_price, self.index_max_size, self.searches_per_day, self.max_indexes) + + def max_size_mb(self): + return self.index_max_size * settings.INDEX_SIZE_RATIO + class Meta: + db_table = 'storefront_package' + +class ScoreFunction(models.Model): + index = models.ForeignKey(Index, related_name='scorefunctions') + name = models.IntegerField(null=False) # TODO the java API expects an int. But a String may be nicer for name. + definition = models.CharField(max_length=255, blank=False, null=True) + + class Meta: + db_table = 'storefront_scorefunction' + unique_together = (('index','name')) + + +def create_configuration(description, data, creation_date=None): + configuration = IndexConfiguration() + configuration.description = description + configuration.creation_date = creation_date or datetime.now() + configuration.json_configuration = json.dumps(data) + + configuration.save() + return configuration + +def create_package(code, name, base_price, index_max_size, searches_per_day, max_indexes, configuration_map): +# The configuration_map will only be considered if the package if new or if it didn't already have a configuration + + package = None + try: + package = Package.objects.get(code=code) + + package.name = name + package.base_price = base_price + package.index_max_size = index_max_size + package.searches_per_day = searches_per_day + package.max_indexes = max_indexes + + if not package.configuration: + package.configuration = create_configuration('package:' + code, configuration_map) + + package.save() + except Package.DoesNotExist: + configuration = create_configuration('package:' + code, configuration_map) + package = Package(code=code, base_price=base_price, index_max_size=index_max_size, searches_per_day=searches_per_day, max_indexes=max_indexes, configuration=configuration) + package.save() + +def create_provisioner(name, token, email, plans): + provisioner = None + try: + provisioner = Provisioner.objects.get(name=name) + except Provisioner.DoesNotExist: + provisioner = Provisioner() + provisioner.name = name + provisioner.token = token + provisioner.email = email + provisioner.save() + + provisioner.plans.all().delete() + for plan, code in plans.items(): + pp = ProvisionerPlan() + pp.plan = plan + pp.provisioner = provisioner + pp.package = Package.objects.get(code=code) + pp.save() + + +class AccountMovement(models.Model): + account = models.ForeignKey('Account', related_name='movements') + class Meta: + db_table = 'storefront_accountmovement' + +class ActionLog(models.Model): + account = models.ForeignKey('Account', related_name='actions') + class Meta: + db_table = 'storefront_actionlog' + +class PFUser(models.Model): + user = models.ForeignKey(User, unique=True) + account = models.OneToOneField('Account', related_name='user') + email = models.EmailField(unique=True, max_length=255) + change_password = models.BooleanField(default=False, null=False) + class Meta: + db_table = 'storefront_pfuser' + + + +MAX_USABLE_RAM_PERCENTAGE = 0.9 +# Nebulyzer stuff +class Worker(models.Model): + ''' + Describes an amazon ec2 instance. + ''' + instance_name = models.CharField(max_length=50,null=False,blank=False) + lan_dns = models.CharField(max_length=100,null=False,blank=False) + wan_dns = models.CharField(max_length=100,null=False,blank=False) + status = models.CharField(max_length=30) + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) + #physical memory in MegaBytes + ram = models.IntegerField() + + class States: + created = 'CREATED' + initializing = 'INITIALIZING' + updating = 'UPDATING' + controllable = 'CONTROLLABLE' + decommissioning = 'DECOMMISSIONING' + dying = 'DYING' + dead = 'DEAD' + + class Meta: + db_table = 'storefront_worker' + + def get_usable_ram(self): + '''Return the amount of ram that can be used in this machine for + indexengines. It's calculated as a fixed percentage of the physical + ram. Value returned in MegaBytes''' + return MAX_USABLE_RAM_PERCENTAGE * self.ram + + def get_used_ram(self): + xmx = self.deploys.aggregate(xmx=Sum('effective_xmx'))['xmx'] + bdb = self.deploys.aggregate(bdb=Sum('effective_bdb'))['bdb'] + if xmx == None: + xmx = 0 + if bdb == None: + bdb = 0 + return xmx + bdb + + def is_assignable(self): + return self.status != Worker.States.decommissioning + + def is_ready(self): + return self.status in [Worker.States.controllable, Worker.States.decommissioning] + + def __repr__(self): + return 'Worker (%s):\n\tinstance_name: %s\n\tlan_dns: %s\n\twan_dns: %s\n\tstatus: %s\n\ttimestamp: %s\n\tram: %s\n' %(self.pk, self.instance_name, self.lan_dns, self.wan_dns, self.status, self.timestamp, self.ram) + +class Service(models.Model): + name = models.CharField(max_length=50,null=False,blank=False) + type = models.CharField(max_length=50,null=True,blank=True ) + host = models.CharField(max_length=100,null=False,blank=False) + port = models.IntegerField() + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) + + class Meta: + db_table = 'storefront_service' + + def __repr__(self): + return 'Service (%s):\n\tname: %s\n\ttype: %s\n\thost: %s\n\tport: %s\n\ttimestamp: %s\n' % (self.pk, self.name, self.type, self.host, self.port, self.timestamp) + + +# CPU Stats +class WorkerMountInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="disk_infos") + timestamp = models.DateTimeField() + + mount = models.CharField(max_length=100,null=False,blank=False) + available = models.IntegerField() + used = models.IntegerField() + + class Meta: + db_table = 'storefront_workermountinfo' + + +class WorkerLoadInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="load_infos") + timestamp = models.DateTimeField() + + load_average = models.FloatField() + + class Meta: + db_table = 'storefront_workerloadinfo' + +class WorkerIndexInfo(models.Model): + worker = models.ForeignKey(Worker, related_name="indexes_infos") + timestamp = models.DateTimeField() + + deploy = models.ForeignKey('Deploy', related_name="index_infos") + used_disk = models.IntegerField() + used_mem = models.IntegerField() + + class Meta: + db_table = 'storefront_workerindexinfo' + + +class Deploy(models.Model): + ''' + Describes a deploy of an index on a worker, and it's status. + The idea is that an index can be moving from one worker to another, + so queries and indexing requests have to be mapped to one or more + index engines. + ''' + index = models.ForeignKey(Index, related_name="deploys") + worker = models.ForeignKey(Worker, related_name="deploys") + base_port = models.IntegerField() + status = models.CharField(max_length=30) + timestamp = models.DateTimeField(auto_now=True,auto_now_add=True) # Last time we updated this deploy. + parent = models.ForeignKey('self', related_name='children', null=True) # For moving deploys. + effective_xmx = models.IntegerField() + effective_bdb = models.IntegerField() + dying = models.BooleanField(default=False, null=False) + + # TODO add role fields + #searching_role = models.BooleanField() + #indexing_role = models.BooleanField() + + def __repr__(self): + return 'Deploy (%s):\n\tparent deploy: %s\n\tindex code: %s\n\tstatus: %s\n\tworker ip: %s\n\tport: %d\n\teffective_xmx: %d\n\teffective_bdb: %d\n' % (self.id, self.parent_id, self.index.code, self.status, self.worker.lan_dns, self.base_port, self.effective_xmx, self.effective_bdb) + + def __unicode__(self): + return "Deploy: %s on %s:%d" % (self.status, self.worker.lan_dns, self.base_port) + + def is_readable(self): + '''Returns true if a search can be performed on this deployment, and + the returned data is up to date''' + return self.status == Deploy.States.controllable or \ + self.status == Deploy.States.move_requested or \ + self.status == Deploy.States.moving or \ + (self.status == Deploy.States.recovering and not self.parent) + + def is_writable(self): + '''Returns True if new data has to be written to this deployment.''' + return self.status == Deploy.States.controllable or \ + self.status == Deploy.States.recovering or \ + self.status == Deploy.States.move_requested or \ + self.status == Deploy.States.moving + + def total_ram(self): + return self.effective_xmx + self.effective_bdb + + def update_status(self, new_status): + print 'updating status %s for %r' % (new_status, self) + logger.debug('Updating status to %s for deploy %r', new_status, self) + Deploy.objects.filter(id=self.id).update(status=new_status, timestamp=datetime.now()) + + def update_parent(self, new_parent): + logger.debug('Updating parent to %s for deploy %r', new_parent, self) + Deploy.objects.filter(id=self.id).update(parent=new_parent) + + class States: + created = 'CREATED' + initializing = 'INITIALIZING' + recovering = 'RECOVERING' + resurrecting = 'RESURRECTING' + controllable = 'CONTROLLABLE' + move_requested = 'MOVE_REQUESTED' + moving = 'MOVING' + decommissioning = 'DECOMMISSIONING' + + class Meta: + db_table = 'storefront_deploy' + +class BetaTestRequest(models.Model): + email = models.EmailField(unique=True, max_length=255) + site_url = models.CharField(max_length=200,null=False,blank=False) + summary = models.TextField(null=False,blank=False) + + request_date = models.DateTimeField(default=datetime.now) + status = models.CharField(max_length=50,null=True) + + class Meta: + db_table = 'storefront_betatestrequest' + +class BetaInvitation(models.Model): + password = models.CharField(max_length=20, null=True) + account = models.ForeignKey('Account', null=True) + assigned_customer = models.CharField(max_length=50, null=True) + beta_requester = models.ForeignKey('BetaTestRequest', null=True, related_name="invitation") + + invitation_date = models.DateTimeField(default=datetime.now) + forced_package = models.ForeignKey('Package', null=False) + + class Meta: + db_table = 'storefront_signupotp' + +class ContactInfo(models.Model): + name = models.CharField(max_length=64) + email = models.EmailField(unique=True, max_length=255) + request_date = models.DateTimeField(default=datetime.now) + source = models.CharField(max_length=64, null=True) + + class Meta: + db_table = 'storefront_contactinfo' + + + +class Provisioner(models.Model): + name = models.CharField(max_length=64) + token = models.CharField(max_length=64, null=False, blank=False) + email = models.EmailField(max_length=255) # contact info for the provisioner + + class Meta: + db_table = "storefront_provisioner" + +class ProvisionerPlan(models.Model): + plan = models.CharField(max_length=50) + provisioner = models.ForeignKey('Provisioner', related_name='plans') + package = models.ForeignKey('Package') + + class Meta: + db_table = "storefront_provisionerplan" + +class BlogPostInfo(models.Model): + title = models.CharField(max_length=200) + url = models.CharField(max_length=1024) + date = models.DateTimeField() + author = models.CharField(max_length=64) + + class Meta: + db_table = 'storefront_blogpost' + diff --git a/storefront/auth.py b/storefront/auth.py new file mode 100644 index 0000000..5cc3a91 --- /dev/null +++ b/storefront/auth.py @@ -0,0 +1,24 @@ +import re + +from django.contrib.auth.backends import ModelBackend +from django.contrib.auth.models import User + +from storefront.models import Account +from storefront.lib import encoder + +class ApiUrlBackend(ModelBackend): + def authenticate(self, username=None, password=None, request=None): + password = password.rstrip('/') + if username == 'apiurl@indextank.com': + code = re.search(r'.*@([^\.]+)\.api\..*', password) + if code: + code = code.group(1) + account_id = encoder.from_key(code) + account = Account.objects.get(id=account_id) + if account.get_private_apiurl() == password: + return account.user.user + else: + return None + else: + return None + return None diff --git a/storefront/authorize.settings.debug b/storefront/authorize.settings.debug new file mode 100644 index 0000000..4a57168 --- /dev/null +++ b/storefront/authorize.settings.debug @@ -0,0 +1,7 @@ +# Settings for Authorize.net + +# TEST API +host_url = 'https://apitest.authorize.net/xml/v1/request.api' +api_login_id = '2ErYn5tb5X' +transaction_key = '75mhe6B5TLNwA47b' + diff --git a/storefront/authorize.settings.prod b/storefront/authorize.settings.prod new file mode 100644 index 0000000..f49ee74 --- /dev/null +++ b/storefront/authorize.settings.prod @@ -0,0 +1,7 @@ +# Settings for Authorize.net + +# PROD API +host_url = 'https://api.authorize.net/xml/v1/request.api' +api_login_id = '9rnJ45EuM' +transaction_key = '7Jm5Lgw88P5VwK7Y' + diff --git a/storefront/blog_posts.csv b/storefront/blog_posts.csv new file mode 100644 index 0000000..dbeee67 --- /dev/null +++ b/storefront/blog_posts.csv @@ -0,0 +1,10 @@ +http://blog.indextank.com/638/new-release-of-indextank-now-with-public-search-api/,|New release of IndexTank, now with Public Search API|,diego,May 09 +http://blog.indextank.com/626/indextank-factual-contest-results/,IndexTank / Factual contest results!,diego,April 26 +http://blog.indextank.com/619/working-to-recover-from-ec2s-outage/,Working to recover from EC2′s outage,diego,April 22 +http://blog.indextank.com/609/indextank-now-free-up-to-100k-documents/,IndexTank now free up to 100K documents!,diego,April 20 +http://blog.indextank.com/602/zend-framework-client-for-indextank-courtesy-of-helpdesk/,|Zend framework client for IndexTank, courtesy of Helpdesk|,diego,April 15 +http://blog.indextank.com/598/indextankfactual-contest-deadline-extended-one-more-weekend/,|IndexTank/Factual contest deadline extended, one more weekend!|,diego,April 14 +http://blog.indextank.com/590/todays-downtime-how-we-are-fixing-it/,|Today’s downtime, how we are fixing it|,nacho,April 12 +http://blog.indextank.com/581/tanker-integrate-indextank-with-your-favorite-ruby-orm/,Tanker: Integrate IndexTank with your favorite Ruby ORM,diego,April 07 +http://blog.indextank.com/546/contest-time-build-an-app-with-indextank-factual/,Contest time! Build an app with IndexTank + Factual,diego,March 31 +http://blog.indextank.com/523/fuzzy-search-find-what-i-meant-not-what-i-said/,|Fuzzy Search – find what I meant, not what I said!|,nacho,March 25 diff --git a/storefront/boto/__init__.py b/storefront/boto/__init__.py new file mode 100644 index 0000000..fc2e592 --- /dev/null +++ b/storefront/boto/__init__.py @@ -0,0 +1,263 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.config import Config, BotoConfigLocations +import os, sys +import logging +import logging.config + +Version = '1.9b' +UserAgent = 'Boto/%s (%s)' % (Version, sys.platform) +config = Config() + +def init_logging(): + for file in BotoConfigLocations: + try: + logging.config.fileConfig(os.path.expanduser(file)) + except: + pass + +class NullHandler(logging.Handler): + def emit(self, record): + pass + +log = logging.getLogger('boto') +log.addHandler(NullHandler()) +init_logging() + +# convenience function to set logging to a particular file +def set_file_logger(name, filepath, level=logging.INFO, format_string=None): + global log + if not format_string: + format_string = "%(asctime)s %(name)s [%(levelname)s]:%(message)s" + logger = logging.getLogger(name) + logger.setLevel(level) + fh = logging.FileHandler(filepath) + fh.setLevel(level) + formatter = logging.Formatter(format_string) + fh.setFormatter(formatter) + logger.addHandler(fh) + log = logger + +def set_stream_logger(name, level=logging.DEBUG, format_string=None): + global log + if not format_string: + format_string = "%(asctime)s %(name)s [%(levelname)s]:%(message)s" + logger = logging.getLogger(name) + logger.setLevel(level) + fh = logging.StreamHandler() + fh.setLevel(level) + formatter = logging.Formatter(format_string) + fh.setFormatter(formatter) + logger.addHandler(fh) + log = logger + +def connect_sqs(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.sqs.connection.SQSConnection` + :return: A connection to Amazon's SQS + """ + from boto.sqs.connection import SQSConnection + return SQSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_s3(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.s3.connection.S3Connection` + :return: A connection to Amazon's S3 + """ + from boto.s3.connection import S3Connection + return S3Connection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_ec2(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.connection.EC2Connection` + :return: A connection to Amazon's EC2 + """ + from boto.ec2.connection import EC2Connection + return EC2Connection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_elb(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.elb.ELBConnection` + :return: A connection to Amazon's Load Balancing Service + """ + from boto.ec2.elb import ELBConnection + return ELBConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_autoscale(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.autoscale.AutoScaleConnection` + :return: A connection to Amazon's Auto Scaling Service + """ + from boto.ec2.autoscale import AutoScaleConnection + return AutoScaleConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_cloudwatch(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.cloudwatch.CloudWatchConnection` + :return: A connection to Amazon's EC2 Monitoring service + """ + from boto.ec2.cloudwatch import CloudWatchConnection + return CloudWatchConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_sdb(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.sdb.connection.SDBConnection` + :return: A connection to Amazon's SDB + """ + from boto.sdb.connection import SDBConnection + return SDBConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_fps(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.fps.connection.FPSConnection` + :return: A connection to FPS + """ + from boto.fps.connection import FPSConnection + return FPSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_cloudfront(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.fps.connection.FPSConnection` + :return: A connection to FPS + """ + from boto.cloudfront import CloudFrontConnection + return CloudFrontConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_vpc(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.vpc.VPCConnection` + :return: A connection to VPC + """ + from boto.vpc import VPCConnection + return VPCConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_rds(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.rds.RDSConnection` + :return: A connection to RDS + """ + from boto.rds import RDSConnection + return RDSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def check_extensions(module_name, module_path): + """ + This function checks for extensions to boto modules. It should be called in the + __init__.py file of all boto modules. See: + http://code.google.com/p/boto/wiki/ExtendModules + + for details. + """ + option_name = '%s_extend' % module_name + version = config.get('Boto', option_name, None) + if version: + dirname = module_path[0] + path = os.path.join(dirname, version) + if os.path.isdir(path): + log.info('extending module %s with: %s' % (module_name, path)) + module_path.insert(0, path) + +_aws_cache = {} + +def _get_aws_conn(service): + global _aws_cache + conn = _aws_cache.get(service) + if not conn: + meth = getattr(sys.modules[__name__], 'connect_'+service) + conn = meth() + _aws_cache[service] = conn + return conn + +def lookup(service, name): + global _aws_cache + conn = _get_aws_conn(service) + obj = _aws_cache.get('.'.join((service,name)), None) + if not obj: + obj = conn.lookup(name) + _aws_cache['.'.join((service,name))] = obj + return obj + diff --git a/storefront/boto/cloudfront/__init__.py b/storefront/boto/cloudfront/__init__.py new file mode 100644 index 0000000..d03d6eb --- /dev/null +++ b/storefront/boto/cloudfront/__init__.py @@ -0,0 +1,222 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import xml.sax +import base64 +import time +import boto.utils +from boto.connection import AWSAuthConnection +from boto import handler +from boto.cloudfront.distribution import * +from boto.cloudfront.identity import OriginAccessIdentity +from boto.cloudfront.identity import OriginAccessIdentityConfig +from boto.resultset import ResultSet +from boto.cloudfront.exception import CloudFrontServerError + +class CloudFrontConnection(AWSAuthConnection): + + DefaultHost = 'cloudfront.amazonaws.com' + Version = '2009-12-01' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0): + AWSAuthConnection.__init__(self, host, + aws_access_key_id, aws_secret_access_key, + True, port, proxy, proxy_port, debug=debug) + + def get_etag(self, response): + response_headers = response.msg + for key in response_headers.keys(): + if key.lower() == 'etag': + return response_headers[key] + return None + + def add_aws_auth_header(self, headers, method, path): + if not headers.has_key('Date'): + headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", + time.gmtime()) + + hmac = self.hmac.copy() + hmac.update(headers['Date']) + b64_hmac = base64.encodestring(hmac.digest()).strip() + headers['Authorization'] = "AWS %s:%s" % (self.aws_access_key_id, b64_hmac) + + # Generics + + def _get_all_objects(self, resource, tags): + if not tags: + tags=[('DistributionSummary', DistributionSummary)] + response = self.make_request('GET', '/%s/%s' % (self.Version, resource)) + body = response.read() + if response.status >= 300: + raise CloudFrontServerError(response.status, response.reason, body) + rs = ResultSet(tags) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def _get_info(self, id, resource, dist_class): + uri = '/%s/%s/%s' % (self.Version, resource, id) + response = self.make_request('GET', uri) + body = response.read() + if response.status >= 300: + raise CloudFrontServerError(response.status, response.reason, body) + d = dist_class(connection=self) + response_headers = response.msg + for key in response_headers.keys(): + if key.lower() == 'etag': + d.etag = response_headers[key] + h = handler.XmlHandler(d, self) + xml.sax.parseString(body, h) + return d + + def _get_config(self, id, resource, config_class): + uri = '/%s/%s/%s/config' % (self.Version, resource, id) + response = self.make_request('GET', uri) + body = response.read() + if response.status >= 300: + raise CloudFrontServerError(response.status, response.reason, body) + d = config_class(connection=self) + d.etag = self.get_etag(response) + h = handler.XmlHandler(d, self) + xml.sax.parseString(body, h) + return d + + def _set_config(self, distribution_id, etag, config): + if isinstance(config, StreamingDistributionConfig): + resource = 'streaming-distribution' + else: + resource = 'distribution' + uri = '/%s/%s/%s/config' % (self.Version, resource, distribution_id) + headers = {'If-Match' : etag, 'Content-Type' : 'text/xml'} + response = self.make_request('PUT', uri, headers, config.to_xml()) + body = response.read() + return self.get_etag(response) + if response.status != 200: + raise CloudFrontServerError(response.status, response.reason, body) + + def _create_object(self, config, resource, dist_class): + response = self.make_request('POST', '/%s/%s' % (self.Version, resource), + {'Content-Type' : 'text/xml'}, data=config.to_xml()) + body = response.read() + if response.status == 201: + d = dist_class(connection=self) + h = handler.XmlHandler(d, self) + xml.sax.parseString(body, h) + return d + else: + raise CloudFrontServerError(response.status, response.reason, body) + + def _delete_object(self, id, etag, resource): + uri = '/%s/%s/%s' % (self.Version, resource, id) + response = self.make_request('DELETE', uri, {'If-Match' : etag}) + body = response.read() + if response.status != 204: + raise CloudFrontServerError(response.status, response.reason, body) + + # Distributions + + def get_all_distributions(self): + tags=[('DistributionSummary', DistributionSummary)] + return self._get_all_objects('distribution', tags) + + def get_distribution_info(self, distribution_id): + return self._get_info(distribution_id, 'distribution', Distribution) + + def get_distribution_config(self, distribution_id): + return self._get_config(distribution_id, 'distribution', + DistributionConfig) + + def set_distribution_config(self, distribution_id, etag, config): + return self._set_config(distribution_id, etag, config) + + def create_distribution(self, origin, enabled, caller_reference='', + cnames=None, comment=''): + config = DistributionConfig(origin=origin, enabled=enabled, + caller_reference=caller_reference, + cnames=cnames, comment=comment) + return self._create_object(config, 'distribution', Distribution) + + def delete_distribution(self, distribution_id, etag): + return self._delete_object(distribution_id, etag, 'distribution') + + # Streaming Distributions + + def get_all_streaming_distributions(self): + tags=[('StreamingDistributionSummary', StreamingDistributionSummary)] + return self._get_all_objects('streaming-distribution', tags) + + def get_streaming_distribution_info(self, distribution_id): + return self._get_info(distribution_id, 'streaming-distribution', + StreamingDistribution) + + def get_streaming_distribution_config(self, distribution_id): + return self._get_config(distribution_id, 'streaming-distribution', + StreamingDistributionConfig) + + def set_streaming_distribution_config(self, distribution_id, etag, config): + return self._set_config(distribution_id, etag, config) + + def create_streaming_distribution(self, origin, enabled, + caller_reference='', + cnames=None, comment=''): + config = StreamingDistributionConfig(origin=origin, enabled=enabled, + caller_reference=caller_reference, + cnames=cnames, comment=comment) + return self._create_object(config, 'streaming-distribution', + StreamingDistribution) + + def delete_streaming_distribution(self, distribution_id, etag): + return self._delete_object(distribution_id, etag, 'streaming-distribution') + + # Origin Access Identity + + def get_all_origin_access_identity(self): + tags=[('CloudFrontOriginAccessIdentitySummary', + OriginAccessIdentitySummary)] + return self._get_all_objects('origin-access-identity/cloudfront', tags) + + def get_origin_access_identity_info(self, access_id): + return self._get_info(access_id, 'origin-access-identity/cloudfront', + OriginAccessIdentity) + + def get_origin_access_identity_config(self, access_id): + return self._get_config(access_id, + 'origin-access-identity/cloudfront', + OriginAccessIdentityConfig) + + def set_origin_access_identity_config(self, access_id, + etag, config): + return self._set_config(access_id, etag, config) + + def create_origin_access_identity(self, caller_reference='', comment=''): + config = OriginAccessIdentityConfig(caller_reference=caller_reference, + comment=comment) + return self._create_object(config, 'origin-access-identity/cloudfront', + OriginAccessIdentity) + + def delete_origin_access_identity(self, access_id, etag): + return self._delete_object(access_id, etag, + 'origin-access-identity/cloudfront') + + diff --git a/storefront/boto/cloudfront/distribution.py b/storefront/boto/cloudfront/distribution.py new file mode 100644 index 0000000..cd36add --- /dev/null +++ b/storefront/boto/cloudfront/distribution.py @@ -0,0 +1,470 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import uuid +from boto.cloudfront.identity import OriginAccessIdentity +from boto.cloudfront.object import Object, StreamingObject +from boto.cloudfront.signers import Signer, ActiveTrustedSigners, TrustedSigners +from boto.cloudfront.logging import LoggingInfo +from boto.s3.acl import ACL + +class DistributionConfig: + + def __init__(self, connection=None, origin='', enabled=False, + caller_reference='', cnames=None, comment='', + origin_access_identity=None, trusted_signers=None): + self.connection = connection + self.origin = origin + self.enabled = enabled + if caller_reference: + self.caller_reference = caller_reference + else: + self.caller_reference = str(uuid.uuid4()) + self.cnames = [] + if cnames: + self.cnames = cnames + self.comment = comment + self.origin_access_identity = origin_access_identity + self.trusted_signers = trusted_signers + self.logging = None + + def get_oai_value(self): + if isinstance(self.origin_access_identity, OriginAccessIdentity): + return self.origin_access_identity.uri() + else: + return self.origin_access_identity + + def to_xml(self): + s = '\n' + s += '\n' + s += ' %s\n' % self.origin + s += ' %s\n' % self.caller_reference + for cname in self.cnames: + s += ' %s\n' % cname + if self.comment: + s += ' %s\n' % self.comment + s += ' ' + if self.enabled: + s += 'true' + else: + s += 'false' + s += '\n' + if self.origin_access_identity: + val = self.get_oai_value() + s += '%s\n' % val + if self.trusted_signers: + s += '\n' + for signer in self.trusted_signers: + if signer == 'Self': + s += ' \n' + else: + s += ' %s\n' % signer + s += '\n' + if self.logging: + s += '\n' + s += ' %s\n' % self.logging.bucket + s += ' %s\n' % self.logging.prefix + s += '\n' + s += '\n' + return s + + def startElement(self, name, attrs, connection): + if name == 'TrustedSigners': + self.trusted_signers = TrustedSigners() + return self.trusted_signers + elif name == 'Logging': + self.logging = LoggingInfo() + return self.logging + else: + return None + + def endElement(self, name, value, connection): + if name == 'CNAME': + self.cnames.append(value) + elif name == 'Origin': + self.origin = value + elif name == 'Comment': + self.comment = value + elif name == 'Enabled': + if value.lower() == 'true': + self.enabled = True + else: + self.enabled = False + elif name == 'CallerReference': + self.caller_reference = value + elif name == 'OriginAccessIdentity': + self.origin_access_identity = value + else: + setattr(self, name, value) + +class StreamingDistributionConfig(DistributionConfig): + + def __init__(self, connection=None, origin='', enabled=False, + caller_reference='', cnames=None, comment=''): + DistributionConfig.__init__(self, connection, origin, + enabled, caller_reference, + cnames, comment) + + def to_xml(self): + s = '\n' + s += '\n' + s += ' %s\n' % self.origin + s += ' %s\n' % self.caller_reference + for cname in self.cnames: + s += ' %s\n' % cname + if self.comment: + s += ' %s\n' % self.comment + s += ' ' + if self.enabled: + s += 'true' + else: + s += 'false' + s += '\n' + s += '\n' + return s + + def startElement(self, name, attrs, connection): + pass + +class DistributionSummary: + + def __init__(self, connection=None, domain_name='', id='', + last_modified_time=None, status='', origin='', + cname='', comment='', enabled=False): + self.connection = connection + self.domain_name = domain_name + self.id = id + self.last_modified_time = last_modified_time + self.status = status + self.origin = origin + self.enabled = enabled + self.cnames = [] + if cname: + self.cnames.append(cname) + self.comment = comment + self.trusted_signers = None + self.etag = None + self.streaming = False + + def startElement(self, name, attrs, connection): + if name == 'TrustedSigners': + self.trusted_signers = TrustedSigners() + return self.trusted_signers + return None + + def endElement(self, name, value, connection): + if name == 'Id': + self.id = value + elif name == 'Status': + self.status = value + elif name == 'LastModifiedTime': + self.last_modified_time = value + elif name == 'DomainName': + self.domain_name = value + elif name == 'Origin': + self.origin = value + elif name == 'CNAME': + self.cnames.append(value) + elif name == 'Comment': + self.comment = value + elif name == 'Enabled': + if value.lower() == 'true': + self.enabled = True + else: + self.enabled = False + elif name == 'StreamingDistributionSummary': + self.streaming = True + else: + setattr(self, name, value) + + def get_distribution(self): + return self.connection.get_distribution_info(self.id) + +class StreamingDistributionSummary(DistributionSummary): + + def get_distribution(self): + return self.connection.get_streaming_distribution_info(self.id) + +class Distribution: + + def __init__(self, connection=None, config=None, domain_name='', + id='', last_modified_time=None, status=''): + self.connection = connection + self.config = config + self.domain_name = domain_name + self.id = id + self.last_modified_time = last_modified_time + self.status = status + self.active_signers = None + self.etag = None + self._bucket = None + + def startElement(self, name, attrs, connection): + if name == 'DistributionConfig': + self.config = DistributionConfig() + return self.config + elif name == 'ActiveTrustedSigners': + self.active_signers = ActiveTrustedSigners() + return self.active_signers + else: + return None + + def endElement(self, name, value, connection): + if name == 'Id': + self.id = value + elif name == 'LastModifiedTime': + self.last_modified_time = value + elif name == 'Status': + self.status = value + elif name == 'DomainName': + self.domain_name = value + else: + setattr(self, name, value) + + def update(self, enabled=None, cnames=None, comment=None, + origin_access_identity=None, + trusted_signers=None): + """ + Update the configuration of the Distribution. + + :type enabled: bool + :param enabled: Whether the Distribution is active or not. + + :type cnames: list of str + :param cnames: The DNS CNAME's associated with this + Distribution. Maximum of 10 values. + + :type comment: str or unicode + :param comment: The comment associated with the Distribution. + + :type origin_access_identity: :class:`boto.cloudfront.identity.OriginAccessIdentity` + :param origin_access_identity: The CloudFront origin access identity + associated with the distribution. This + must be provided if you want the + distribution to serve private content. + + :type trusted_signers: :class:`boto.cloudfront.signers.TrustedSigner` + :param trusted_signers: The AWS users who are authorized to sign + URL's for private content in this Distribution. + + """ + new_config = DistributionConfig(self.connection, self.config.origin, + self.config.enabled, self.config.caller_reference, + self.config.cnames, self.config.comment, + self.config.origin_access_identity, + self.config.trusted_signers) + if enabled != None: + new_config.enabled = enabled + if cnames != None: + new_config.cnames = cnames + if comment != None: + new_config.comment = comment + if origin_access_identity != None: + new_config.origin_access_identity = origin_access_identity + if trusted_signers: + new_config.trusted_signers = trusted_signers + self.etag = self.connection.set_distribution_config(self.id, self.etag, new_config) + self.config = new_config + self._object_class = Object + + def enable(self): + """ + Deactivate the Distribution. A convenience wrapper around + the update method. + """ + self.update(enabled=True) + + def disable(self): + """ + Activate the Distribution. A convenience wrapper around + the update method. + """ + self.update(enabled=False) + + def delete(self): + """ + Delete this CloudFront Distribution. The content + associated with the Distribution is not deleted from + the underlying Origin bucket in S3. + """ + self.connection.delete_distribution(self.id, self.etag) + + def _get_bucket(self): + if not self._bucket: + bucket_name = self.config.origin.split('.')[0] + from boto.s3.connection import S3Connection + s3 = S3Connection(self.connection.aws_access_key_id, + self.connection.aws_secret_access_key, + proxy=self.connection.proxy, + proxy_port=self.connection.proxy_port, + proxy_user=self.connection.proxy_user, + proxy_pass=self.connection.proxy_pass) + self._bucket = s3.get_bucket(bucket_name) + self._bucket.distribution = self + self._bucket.set_key_class(self._object_class) + return self._bucket + + def get_objects(self): + """ + Return a list of all content objects in this distribution. + + :rtype: list of :class:`boto.cloudfront.object.Object` + :return: The content objects + """ + bucket = self._get_bucket() + objs = [] + for key in bucket: + objs.append(key) + return objs + + def set_permissions(self, object, replace=False): + """ + Sets the S3 ACL grants for the given object to the appropriate + value based on the type of Distribution. If the Distribution + is serving private content the ACL will be set to include the + Origin Access Identity associated with the Distribution. If + the Distribution is serving public content the content will + be set up with "public-read". + + :type object: :class:`boto.cloudfront.object.Object` + :param enabled: The Object whose ACL is being set + + :type replace: bool + :param replace: If False, the Origin Access Identity will be + appended to the existing ACL for the object. + If True, the ACL for the object will be + completely replaced with one that grants + READ permission to the Origin Access Identity. + + """ + if self.config.origin_access_identity: + id = self.config.origin_access_identity.split('/')[-1] + oai = self.connection.get_origin_access_identity_info(id) + policy = object.get_acl() + if replace: + policy.acl = ACL() + policy.acl.add_user_grant('READ', oai.s3_user_id) + object.set_acl(policy) + else: + object.set_canned_acl('public-read') + + def set_permissions_all(self, replace=False): + """ + Sets the S3 ACL grants for all objects in the Distribution + to the appropriate value based on the type of Distribution. + + :type replace: bool + :param replace: If False, the Origin Access Identity will be + appended to the existing ACL for the object. + If True, the ACL for the object will be + completely replaced with one that grants + READ permission to the Origin Access Identity. + + """ + bucket = self._get_bucket() + for key in bucket: + self.set_permissions(key) + + def add_object(self, name, content, headers=None, replace=True): + """ + Adds a new content object to the Distribution. The content + for the object will be copied to a new Key in the S3 Bucket + and the permissions will be set appropriately for the type + of Distribution. + + :type name: str or unicode + :param name: The name or key of the new object. + + :type content: file-like object + :param content: A file-like object that contains the content + for the new object. + + :type headers: dict + :param headers: A dictionary containing additional headers + you would like associated with the new + object in S3. + + :rtype: :class:`boto.cloudfront.object.Object` + :return: The newly created object. + """ + if self.config.origin_access_identity: + policy = 'private' + else: + policy = 'public-read' + bucket = self._get_bucket() + object = bucket.new_key(name) + object.set_contents_from_file(content, headers=headers, policy=policy) + if self.config.origin_access_identity: + self.set_permissions(object, replace) + return object + +class StreamingDistribution(Distribution): + + def __init__(self, connection=None, config=None, domain_name='', + id='', last_modified_time=None, status=''): + Distribution.__init__(self, connection, config, domain_name, + id, last_modified_time, status) + self._object_class = StreamingObject + + def startElement(self, name, attrs, connection): + if name == 'StreamingDistributionConfig': + self.config = StreamingDistributionConfig() + return self.config + else: + return None + + def update(self, enabled=None, cnames=None, comment=None): + """ + Update the configuration of the Distribution. + + :type enabled: bool + :param enabled: Whether the Distribution is active or not. + + :type cnames: list of str + :param cnames: The DNS CNAME's associated with this + Distribution. Maximum of 10 values. + + :type comment: str or unicode + :param comment: The comment associated with the Distribution. + + """ + new_config = StreamingDistributionConfig(self.connection, + self.config.origin, + self.config.enabled, + self.config.caller_reference, + self.config.cnames, + self.config.comment) + if enabled != None: + new_config.enabled = enabled + if cnames != None: + new_config.cnames = cnames + if comment != None: + new_config.comment = comment + + self.etag = self.connection.set_streaming_distribution_config(self.id, + self.etag, + new_config) + self.config = new_config + + def delete(self): + self.connection.delete_streaming_distribution(self.id, self.etag) + + diff --git a/storefront/boto/cloudfront/exception.py b/storefront/boto/cloudfront/exception.py new file mode 100644 index 0000000..7680642 --- /dev/null +++ b/storefront/boto/cloudfront/exception.py @@ -0,0 +1,26 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import BotoServerError + +class CloudFrontServerError(BotoServerError): + + pass diff --git a/storefront/boto/cloudfront/identity.py b/storefront/boto/cloudfront/identity.py new file mode 100644 index 0000000..711b8b7 --- /dev/null +++ b/storefront/boto/cloudfront/identity.py @@ -0,0 +1,98 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import uuid + +class OriginAccessIdentity: + + def __init__(self, connection=None, config=None, id='', + s3_user_id='', comment=''): + self.connection = connection + self.config = config + self.id = id + self.s3_user_id = s3_user_id + self.comment = comment + self.etag = None + + def startElement(self, name, attrs, connection): + if name == 'CloudFrontOriginAccessIdentityConfig': + self.config = OriginAccessIdentityConfig() + return self.config + else: + return None + + def endElement(self, name, value, connection): + if name == 'Id': + self.id = value + elif name == 'S3CanonicalUserId': + self.s3_user_id = value + elif name == 'Comment': + self.comment = value + else: + setattr(self, name, value) + + def update(self, comment=None): + new_config = OriginAccessIdentifyConfig(self.connection, + self.config.caller_reference, + self.config.comment) + if comment != None: + new_config.comment = comment + self.etag = self.connection.set_origin_identity_config(self.id, self.etag, new_config) + self.config = new_config + + def delete(self): + return self.connection.delete_distribution(self.id, self.etag) + + def uri(self): + return 'origin-access-identity/cloudfront/%s' % id + +class OriginAccessIdentityConfig: + + def __init__(self, connection=None, caller_reference='', comment=''): + self.connection = connection + if caller_reference: + self.caller_reference = caller_reference + else: + self.caller_reference = str(uuid.uuid4()) + self.comment = comment + + def to_xml(self): + s = '\n' + s += '\n' + s += ' %s\n' % self.caller_reference + if self.comment: + s += ' %s\n' % self.comment + s += '\n' + return s + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Comment': + self.comment = value + elif name == 'CallerReference': + self.caller_reference = value + else: + setattr(self, name, value) + + + diff --git a/storefront/boto/cloudfront/logging.py b/storefront/boto/cloudfront/logging.py new file mode 100644 index 0000000..6c2f4fd --- /dev/null +++ b/storefront/boto/cloudfront/logging.py @@ -0,0 +1,38 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class LoggingInfo(object): + + def __init__(self, bucket='', prefix=''): + self.bucket = bucket + self.prefix = prefix + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Bucket': + self.bucket = value + elif name == 'Prefix': + self.prefix = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/cloudfront/object.py b/storefront/boto/cloudfront/object.py new file mode 100644 index 0000000..3574d13 --- /dev/null +++ b/storefront/boto/cloudfront/object.py @@ -0,0 +1,48 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.s3.key import Key + +class Object(Key): + + def __init__(self, bucket, name=None): + Key.__init__(self, bucket, name=name) + self.distribution = bucket.distribution + + def __repr__(self): + return '' % (self.distribution.config.origin, self.name) + + def url(self, scheme='http'): + url = '%s://' % scheme + url += self.distribution.domain_name + if scheme.lower().startswith('rtmp'): + url += '/cfx/st/' + else: + url += '/' + url += self.name + return url + +class StreamingObject(Object): + + def url(self, scheme='rtmp'): + return Object.url(self, scheme) + + diff --git a/storefront/boto/cloudfront/signers.py b/storefront/boto/cloudfront/signers.py new file mode 100644 index 0000000..0b0cd50 --- /dev/null +++ b/storefront/boto/cloudfront/signers.py @@ -0,0 +1,60 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Signer: + + def __init__(self): + self.id = None + self.key_pair_ids = [] + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Self': + self.id = 'Self' + elif name == 'AwsAccountNumber': + self.id = value + elif name == 'KeyPairId': + self.key_pair_ids.append(value) + +class ActiveTrustedSigners(list): + + def startElement(self, name, attrs, connection): + if name == 'Signer': + s = Signer() + self.append(s) + return s + + def endElement(self, name, value, connection): + pass + +class TrustedSigners(list): + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Self': + self.append(name) + elif name == 'AwsAccountNumber': + self.append(value) + diff --git a/storefront/boto/connection.py b/storefront/boto/connection.py new file mode 100644 index 0000000..9a443f7 --- /dev/null +++ b/storefront/boto/connection.py @@ -0,0 +1,648 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2009 The Echo Nest Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# +# Parts of this code were copied or derived from sample code supplied by AWS. +# The following notice applies to that code. +# +# This software code is made available "AS IS" without warranties of any +# kind. You may copy, display, modify and redistribute the software +# code either by itself or as incorporated into your code; provided that +# you do not remove any proprietary notices. Your use of this software +# code is at your own risk and you waive any claim against Amazon +# Digital Services, Inc. or its affiliates with respect to your use of +# this software code. (c) 2006 Amazon Digital Services, Inc. or its +# affiliates. + +""" +Handles basic connections to AWS +""" + +import base64 +import hmac +import httplib +import socket, errno +import re +import sys +import time +import urllib, urlparse +import os +import xml.sax +import Queue +import boto +from boto.exception import AWSConnectionError, BotoClientError, BotoServerError +from boto.resultset import ResultSet +import boto.utils +from boto import config, UserAgent, handler + +# +# the following is necessary because of the incompatibilities +# between Python 2.4, 2.5, and 2.6 as well as the fact that some +# people running 2.4 have installed hashlib as a separate module +# this fix was provided by boto user mccormix. +# see: http://code.google.com/p/boto/issues/detail?id=172 +# for more details. +# +try: + from hashlib import sha1 as sha + from hashlib import sha256 as sha256 + + if sys.version[:3] == "2.4": + # we are using an hmac that expects a .new() method. + class Faker: + def __init__(self, which): + self.which = which + self.digest_size = self.which().digest_size + + def new(self, *args, **kwargs): + return self.which(*args, **kwargs) + + sha = Faker(sha) + sha256 = Faker(sha256) + +except ImportError: + import sha + sha256 = None + +PORTS_BY_SECURITY = { True: 443, False: 80 } + +class ConnectionPool: + def __init__(self, hosts, connections_per_host): + self._hosts = boto.utils.LRUCache(hosts) + self.connections_per_host = connections_per_host + + def __getitem__(self, key): + if key not in self._hosts: + self._hosts[key] = Queue.Queue(self.connections_per_host) + return self._hosts[key] + + def __repr__(self): + return 'ConnectionPool:%s' % ','.join(self._hosts._dict.keys()) + +class AWSAuthConnection: + def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, path='/'): + """ + :type host: string + :param host: The host to make the connection to + + :type aws_access_key_id: string + :param aws_access_key_id: AWS Access Key ID (provided by Amazon) + + :type aws_secret_access_key: string + :param aws_secret_access_key: Secret Access Key (provided by Amazon) + + :type is_secure: boolean + :param is_secure: Whether the connection is over SSL + + :type https_connection_factory: list or tuple + :param https_connection_factory: A pair of an HTTP connection + factory and the exceptions to catch. + The factory should have a similar + interface to L{httplib.HTTPSConnection}. + + :type proxy: + :param proxy: + + :type proxy_port: int + :param proxy_port: The port to use when connecting over a proxy + + :type proxy_user: string + :param proxy_user: The username to connect with on the proxy + + :type proxy_pass: string + :param proxy_pass: The password to use when connection over a proxy. + + :type port: integer + :param port: The port to use to connect + """ + + self.num_retries = 5 + self.is_secure = is_secure + self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) + # define exceptions from httplib that we want to catch and retry + self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror) + # define values in socket exceptions we don't want to catch + self.socket_exception_values = (errno.EINTR,) + if https_connection_factory is not None: + self.https_connection_factory = https_connection_factory[0] + self.http_exceptions += https_connection_factory[1] + else: + self.https_connection_factory = None + if (is_secure): + self.protocol = 'https' + else: + self.protocol = 'http' + self.host = host + self.path = path + if debug: + self.debug = debug + else: + self.debug = config.getint('Boto', 'debug', debug) + if port: + self.port = port + else: + self.port = PORTS_BY_SECURITY[is_secure] + + if aws_access_key_id: + self.aws_access_key_id = aws_access_key_id + elif os.environ.has_key('AWS_ACCESS_KEY_ID'): + self.aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID'] + elif config.has_option('Credentials', 'aws_access_key_id'): + self.aws_access_key_id = config.get('Credentials', 'aws_access_key_id') + + if aws_secret_access_key: + self.aws_secret_access_key = aws_secret_access_key + elif os.environ.has_key('AWS_SECRET_ACCESS_KEY'): + self.aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY'] + elif config.has_option('Credentials', 'aws_secret_access_key'): + self.aws_secret_access_key = config.get('Credentials', 'aws_secret_access_key') + + # initialize an HMAC for signatures, make copies with each request + self.hmac = hmac.new(self.aws_secret_access_key, digestmod=sha) + if sha256: + self.hmac_256 = hmac.new(self.aws_secret_access_key, digestmod=sha256) + else: + self.hmac_256 = None + + # cache up to 20 connections per host, up to 20 hosts + self._pool = ConnectionPool(20, 20) + self._connection = (self.server_name(), self.is_secure) + self._last_rs = None + + def __repr__(self): + return '%s:%s' % (self.__class__.__name__, self.host) + + def _cached_name(self, host, is_secure): + if host is None: + host = self.server_name() + cached_name = is_secure and 'https://' or 'http://' + cached_name += host + return cached_name + + def connection(self): + return self.get_http_connection(*self._connection) + + connection = property(connection) + + def get_path(self, path='/'): + pos = path.find('?') + if pos >= 0: + params = path[pos:] + path = path[:pos] + else: + params = None + if path[-1] == '/': + need_trailing = True + else: + need_trailing = False + path_elements = self.path.split('/') + path_elements.extend(path.split('/')) + path_elements = [p for p in path_elements if p] + path = '/' + '/'.join(path_elements) + if path[-1] != '/' and need_trailing: + path += '/' + if params: + path = path + params + return path + + def server_name(self, port=None): + if not port: + port = self.port + if port == 80: + signature_host = self.host + else: + # This unfortunate little hack can be attributed to + # a difference in the 2.6 version of httplib. In old + # versions, it would append ":443" to the hostname sent + # in the Host header and so we needed to make sure we + # did the same when calculating the V2 signature. In 2.6 + # it no longer does that. Hence, this kludge. + if sys.version[:3] == "2.6" and port == 443: + signature_host = self.host + else: + signature_host = '%s:%d' % (self.host, port) + return signature_host + + def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): + self.proxy = proxy + self.proxy_port = proxy_port + self.proxy_user = proxy_user + self.proxy_pass = proxy_pass + if os.environ.has_key('http_proxy') and not self.proxy: + pattern = re.compile( + '(?:http://)?' \ + '(?:(?P\w+):(?P.*)@)?' \ + '(?P[\w\-\.]+)' \ + '(?::(?P\d+))?' + ) + match = pattern.match(os.environ['http_proxy']) + if match: + self.proxy = match.group('host') + self.proxy_port = match.group('port') + self.proxy_user = match.group('user') + self.proxy_pass = match.group('pass') + else: + if not self.proxy: + self.proxy = config.get_value('Boto', 'proxy', None) + if not self.proxy_port: + self.proxy_port = config.get_value('Boto', 'proxy_port', None) + if not self.proxy_user: + self.proxy_user = config.get_value('Boto', 'proxy_user', None) + if not self.proxy_pass: + self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) + + if not self.proxy_port and self.proxy: + print "http_proxy environment variable does not specify " \ + "a port, using default" + self.proxy_port = self.port + self.use_proxy = (self.proxy != None) + + def get_http_connection(self, host, is_secure): + queue = self._pool[self._cached_name(host, is_secure)] + try: + return queue.get_nowait() + except Queue.Empty: + return self.new_http_connection(host, is_secure) + + def new_http_connection(self, host, is_secure): + if self.use_proxy: + host = '%s:%d' % (self.proxy, int(self.proxy_port)) + if host is None: + host = self.server_name() + boto.log.debug('establishing HTTP connection') + if is_secure: + if self.use_proxy: + connection = self.proxy_ssl() + elif self.https_connection_factory: + connection = self.https_connection_factory(host) + else: + connection = httplib.HTTPSConnection(host) + else: + connection = httplib.HTTPConnection(host) + if self.debug > 1: + connection.set_debuglevel(self.debug) + # self.connection must be maintained for backwards-compatibility + # however, it must be dynamically pulled from the connection pool + # set a private variable which will enable that + if host.split(':')[0] == self.host and is_secure == self.is_secure: + self._connection = (host, is_secure) + return connection + + def put_http_connection(self, host, is_secure, connection): + try: + self._pool[self._cached_name(host, is_secure)].put_nowait(connection) + except Queue.Full: + # gracefully fail in case of pool overflow + connection.close() + + def proxy_ssl(self): + host = '%s:%d' % (self.host, self.port) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.connect((self.proxy, int(self.proxy_port))) + except: + raise + sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) + sock.sendall("User-Agent: %s\r\n" % UserAgent) + if self.proxy_user and self.proxy_pass: + for k, v in self.get_proxy_auth_header().items(): + sock.sendall("%s: %s\r\n" % (k, v)) + sock.sendall("\r\n") + resp = httplib.HTTPResponse(sock, strict=True) + resp.begin() + + if resp.status != 200: + # Fake a socket error, use a code that make it obvious it hasn't + # been generated by the socket library + raise socket.error(-71, + "Error talking to HTTP proxy %s:%s: %s (%s)" % + (self.proxy, self.proxy_port, resp.status, resp.reason)) + + # We can safely close the response, it duped the original socket + resp.close() + + h = httplib.HTTPConnection(host) + + # Wrap the socket in an SSL socket + if hasattr(httplib, 'ssl'): + sslSock = httplib.ssl.SSLSocket(sock) + else: # Old Python, no ssl module + sslSock = socket.ssl(sock, None, None) + sslSock = httplib.FakeSocket(sock, sslSock) + # This is a bit unclean + h.sock = sslSock + return h + + def prefix_proxy_to_path(self, path, host=None): + path = self.protocol + '://' + (host or self.server_name()) + path + return path + + def get_proxy_auth_header(self): + auth = base64.encodestring(self.proxy_user+':'+self.proxy_pass) + return {'Proxy-Authorization': 'Basic %s' % auth} + + def _mexe(self, method, path, data, headers, host=None, sender=None): + """ + mexe - Multi-execute inside a loop, retrying multiple times to handle + transient Internet errors by simply trying again. + Also handles redirects. + + This code was inspired by the S3Utils classes posted to the boto-users + Google group by Larry Bates. Thanks! + """ + boto.log.debug('Method: %s' % method) + boto.log.debug('Path: %s' % path) + boto.log.debug('Data: %s' % data) + boto.log.debug('Headers: %s' % headers) + boto.log.debug('Host: %s' % host) + response = None + body = None + e = None + num_retries = config.getint('Boto', 'num_retries', self.num_retries) + i = 0 + connection = self.get_http_connection(host, self.is_secure) + while i <= num_retries: + try: + if callable(sender): + response = sender(connection, method, path, data, headers) + else: + connection.request(method, path, data, headers) + response = connection.getresponse() + location = response.getheader('location') + # -- gross hack -- + # httplib gets confused with chunked responses to HEAD requests + # so I have to fake it out + if method == 'HEAD' and getattr(response, 'chunked', False): + response.chunked = 0 + if response.status == 500 or response.status == 503: + boto.log.debug('received %d response, retrying in %d seconds' % (response.status, 2**i)) + body = response.read() + elif response.status == 408: + body = response.read() + print '-------------------------' + print ' 4 0 8 ' + print 'path=%s' % path + print body + print '-------------------------' + elif response.status < 300 or response.status >= 400 or \ + not location: + self.put_http_connection(host, self.is_secure, connection) + return response + else: + scheme, host, path, params, query, fragment = \ + urlparse.urlparse(location) + if query: + path += '?' + query + boto.log.debug('Redirecting: %s' % scheme + '://' + host + path) + connection = self.get_http_connection(host, + scheme == 'https') + continue + except KeyboardInterrupt: + sys.exit('Keyboard Interrupt') + except self.http_exceptions, e: + boto.log.debug('encountered %s exception, reconnecting' % \ + e.__class__.__name__) + connection = self.new_http_connection(host, self.is_secure) + time.sleep(2**i) + i += 1 + # If we made it here, it's because we have exhausted our retries and stil haven't + # succeeded. So, if we have a response object, use it to raise an exception. + # Otherwise, raise the exception that must have already happened. + if response: + raise BotoServerError(response.status, response.reason, body) + elif e: + raise e + else: + raise BotoClientError('Please report this exception as a Boto Issue!') + + def make_request(self, method, path, headers=None, data='', host=None, + auth_path=None, sender=None): + path = self.get_path(path) + if headers == None: + headers = {} + else: + headers = headers.copy() + headers['User-Agent'] = UserAgent + if not headers.has_key('Content-Length'): + headers['Content-Length'] = str(len(data)) + if self.use_proxy: + path = self.prefix_proxy_to_path(path, host) + if self.proxy_user and self.proxy_pass and not self.is_secure: + # If is_secure, we don't have to set the proxy authentication + # header here, we did that in the CONNECT to the proxy. + headers.update(self.get_proxy_auth_header()) + request_string = auth_path or path + self.add_aws_auth_header(headers, method, request_string) + return self._mexe(method, path, data, headers, host, sender) + + def add_aws_auth_header(self, headers, method, path): + path = self.get_path(path) + if not headers.has_key('Date'): + headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", + time.gmtime()) + + c_string = boto.utils.canonical_string(method, path, headers) + boto.log.debug('Canonical: %s' % c_string) + hmac = self.hmac.copy() + hmac.update(c_string) + b64_hmac = base64.encodestring(hmac.digest()).strip() + headers['Authorization'] = "AWS %s:%s" % (self.aws_access_key_id, b64_hmac) + + def close(self): + """(Optional) Close any open HTTP connections. This is non-destructive, + and making a new request will open a connection again.""" + + boto.log.debug('closing all HTTP connections') + self.connection = None # compat field + hosts = list(self._cache.keys()) + for host in hosts: + conn = self._cache[host] + conn.close() + del self._cache[host] + +class AWSQueryConnection(AWSAuthConnection): + + APIVersion = '' + SignatureVersion = '1' + ResponseError = BotoServerError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=None, debug=0, + https_connection_factory=None, path='/'): + AWSAuthConnection.__init__(self, host, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + debug, https_connection_factory, path) + + def get_utf8_value(self, value): + if not isinstance(value, str) and not isinstance(value, unicode): + value = str(value) + if isinstance(value, unicode): + return value.encode('utf-8') + else: + return value + + def calc_signature_0(self, params): + boto.log.debug('using calc_signature_0') + hmac = self.hmac.copy() + s = params['Action'] + params['Timestamp'] + hmac.update(s) + keys = params.keys() + keys.sort(cmp = lambda x, y: cmp(x.lower(), y.lower())) + pairs = [] + for key in keys: + val = self.get_utf8_value(params[key]) + pairs.append(key + '=' + urllib.quote(val)) + qs = '&'.join(pairs) + return (qs, base64.b64encode(hmac.digest())) + + def calc_signature_1(self, params): + boto.log.debug('using calc_signature_1') + hmac = self.hmac.copy() + keys = params.keys() + keys.sort(cmp = lambda x, y: cmp(x.lower(), y.lower())) + pairs = [] + for key in keys: + hmac.update(key) + val = self.get_utf8_value(params[key]) + hmac.update(val) + pairs.append(key + '=' + urllib.quote(val)) + qs = '&'.join(pairs) + return (qs, base64.b64encode(hmac.digest())) + + def calc_signature_2(self, params, verb, path): + boto.log.debug('using calc_signature_2') + string_to_sign = '%s\n%s\n%s\n' % (verb, self.server_name().lower(), path) + if self.hmac_256: + hmac = self.hmac_256.copy() + params['SignatureMethod'] = 'HmacSHA256' + else: + hmac = self.hmac.copy() + params['SignatureMethod'] = 'HmacSHA1' + keys = params.keys() + keys.sort() + pairs = [] + for key in keys: + val = self.get_utf8_value(params[key]) + pairs.append(urllib.quote(key, safe='') + '=' + urllib.quote(val, safe='-_~')) + qs = '&'.join(pairs) + boto.log.debug('query string: %s' % qs) + string_to_sign += qs + boto.log.debug('string_to_sign: %s' % string_to_sign) + hmac.update(string_to_sign) + b64 = base64.b64encode(hmac.digest()) + boto.log.debug('len(b64)=%d' % len(b64)) + boto.log.debug('base64 encoded digest: %s' % b64) + return (qs, b64) + + def get_signature(self, params, verb, path): + if self.SignatureVersion == '0': + t = self.calc_signature_0(params) + elif self.SignatureVersion == '1': + t = self.calc_signature_1(params) + elif self.SignatureVersion == '2': + t = self.calc_signature_2(params, verb, path) + else: + raise BotoClientError('Unknown Signature Version: %s' % self.SignatureVersion) + return t + + def make_request(self, action, params=None, path='/', verb='GET'): + headers = {} + if params == None: + params = {} + params['Action'] = action + params['Version'] = self.APIVersion + params['AWSAccessKeyId'] = self.aws_access_key_id + params['SignatureVersion'] = self.SignatureVersion + params['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + qs, signature = self.get_signature(params, verb, self.get_path(path)) + if verb == 'POST': + headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8' + request_body = qs + '&Signature=' + urllib.quote(signature) + qs = path + else: + request_body = '' + qs = path + '?' + qs + '&Signature=' + urllib.quote(signature) + return AWSAuthConnection.make_request(self, verb, qs, + data=request_body, + headers=headers) + + def build_list_params(self, params, items, label): + if isinstance(items, str): + items = [items] + for i in range(1, len(items)+1): + params['%s.%d' % (label, i)] = items[i-1] + + # generics + + def get_list(self, action, params, markers, path='/', parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if response.status == 200: + rs = ResultSet(markers) + h = handler.XmlHandler(rs, parent) + xml.sax.parseString(body, h) + return rs + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_object(self, action, params, cls, path='/', parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if response.status == 200: + obj = cls(parent) + h = handler.XmlHandler(obj, parent) + xml.sax.parseString(body, h) + return obj + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_status(self, action, params, path='/', parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, parent) + xml.sax.parseString(body, h) + return rs.status + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + diff --git a/storefront/boto/contrib/__init__.py b/storefront/boto/contrib/__init__.py new file mode 100644 index 0000000..303dbb6 --- /dev/null +++ b/storefront/boto/contrib/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/storefront/boto/contrib/m2helpers.py b/storefront/boto/contrib/m2helpers.py new file mode 100644 index 0000000..82d2730 --- /dev/null +++ b/storefront/boto/contrib/m2helpers.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006,2007 Jon Colverson +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module was contributed by Jon Colverson. It provides a couple of helper +functions that allow you to use M2Crypto's implementation of HTTPSConnection +rather than the default version in httplib.py. The main benefit is that +M2Crypto's version verifies the certificate of the server. + +To use this feature, do something like this: + +from boto.ec2.connection import EC2Connection + +ec2 = EC2Connection(ACCESS_KEY_ID, SECRET_ACCESS_KEY, + https_connection_factory=https_connection_factory(cafile=CA_FILE)) + +See http://code.google.com/p/boto/issues/detail?id=57 for more details. +""" +from M2Crypto import SSL +from M2Crypto.httpslib import HTTPSConnection + +def secure_context(cafile=None, capath=None): + ctx = SSL.Context() + ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, depth=9) + if ctx.load_verify_locations(cafile=cafile, capath=capath) != 1: + raise Exception("Couldn't load certificates") + return ctx + +def https_connection_factory(cafile=None, capath=None): + def factory(*args, **kwargs): + return HTTPSConnection( + ssl_context=secure_context(cafile=cafile, capath=capath), + *args, **kwargs) + return (factory, (SSL.SSLError,)) diff --git a/storefront/boto/contrib/ymlmessage.py b/storefront/boto/contrib/ymlmessage.py new file mode 100644 index 0000000..22e5c62 --- /dev/null +++ b/storefront/boto/contrib/ymlmessage.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006,2007 Chris Moyer +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module was contributed by Chris Moyer. It provides a subclass of the +SQS Message class that supports YAML as the body of the message. + +This module requires the yaml module. +""" +from boto.sqs.message import Message +import yaml + +class YAMLMessage(Message): + """ + The YAMLMessage class provides a YAML compatible message. Encoding and + decoding are handled automaticaly. + + Access this message data like such: + + m.data = [ 1, 2, 3] + m.data[0] # Returns 1 + + This depends on the PyYAML package + """ + + def __init__(self, queue=None, body='', xml_attrs=None): + self.data = None + Message.__init__(self, queue, body) + + def set_body(self, body): + self.data = yaml.load(body) + + def get_body(self): + return yaml.dump(self.data) diff --git a/storefront/boto/ec2/__init__.py b/storefront/boto/ec2/__init__.py new file mode 100644 index 0000000..8bb3f53 --- /dev/null +++ b/storefront/boto/ec2/__init__.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +service from AWS. +""" +from boto.ec2.connection import EC2Connection + +def regions(**kw_params): + """ + Get all available regions for the EC2 service. + You may pass any of the arguments accepted by the EC2Connection + object's constructor as keyword arguments and they will be + passed along to the EC2Connection object. + + :rtype: list + :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` + """ + c = EC2Connection(**kw_params) + return c.get_all_regions() + +def connect_to_region(region_name, **kw_params): + for region in regions(**kw_params): + if region.name == region_name: + return region.connect(**kw_params) + return None + +def get_region(region_name, **kw_params): + for region in regions(**kw_params): + if region.name == region_name: + return region + return None + diff --git a/storefront/boto/ec2/address.py b/storefront/boto/ec2/address.py new file mode 100644 index 0000000..b2af107 --- /dev/null +++ b/storefront/boto/ec2/address.py @@ -0,0 +1,51 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Elastic IP Address +""" + +from boto.ec2.ec2object import EC2Object + +class Address(EC2Object): + + def __init__(self, connection=None, public_ip=None, instance_id=None): + EC2Object.__init__(self, connection) + self.connection = connection + self.public_ip = public_ip + self.instance_id = instance_id + + def __repr__(self): + return 'Address:%s' % self.public_ip + + def endElement(self, name, value, connection): + if name == 'publicIp': + self.public_ip = value + elif name == 'instanceId': + self.instance_id = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_address(self.public_ip) + + + diff --git a/storefront/boto/ec2/autoscale/__init__.py b/storefront/boto/ec2/autoscale/__init__.py new file mode 100644 index 0000000..d7c5946 --- /dev/null +++ b/storefront/boto/ec2/autoscale/__init__.py @@ -0,0 +1,206 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +Auto Scaling service. +""" + +import boto +from boto import config +from boto.connection import AWSQueryConnection +from boto.resultset import ResultSet +from boto.ec2.regioninfo import RegionInfo +from boto.ec2.autoscale.request import Request +from boto.ec2.autoscale.trigger import Trigger +from boto.ec2.autoscale.launchconfig import LaunchConfiguration +from boto.ec2.autoscale.group import AutoScalingGroup +from boto.ec2.autoscale.activity import Activity + + +class AutoScaleConnection(AWSQueryConnection): + APIVersion = boto.config.get('Boto', 'autoscale_version', '2009-05-15') + Endpoint = boto.config.get('Boto', 'autoscale_endpoint', + 'autoscaling.amazonaws.com') + SignatureVersion = '2' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=Endpoint, debug=1, + https_connection_factory=None, region=None, path='/'): + """ + Init method to create a new connection to the AutoScaling service. + + B{Note:} The host argument is overridden by the host specified in the + boto configuration file. + """ + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, host, debug, + https_connection_factory, path=path) + + def build_list_params(self, params, items, label): + """ items is a list of dictionaries or strings: + [{'Protocol' : 'HTTP', + 'LoadBalancerPort' : '80', + 'InstancePort' : '80'},..] etc. + or + ['us-east-1b',...] + """ + # different from EC2 list params + for i in xrange(1, len(items)+1): + if isinstance(items[i-1], dict): + for k, v in items[i-1].iteritems(): + params['%s.member.%d.%s' % (label, i, k)] = v + elif isinstance(items[i-1], basestring): + params['%s.member.%d' % (label, i)] = items[i-1] + + def _update_group(self, op, as_group): + params = { + 'AutoScalingGroupName' : as_group.name, + 'Cooldown' : as_group.cooldown, + 'LaunchConfigurationName' : as_group.launch_config_name, + 'MinSize' : as_group.min_size, + 'MaxSize' : as_group.max_size, + } + if op.startswith('Create'): + if as_group.availability_zones: + zones = self.availability_zones + else: + zones = [as_group.availability_zone] + self.build_list_params(params, as_group.load_balancers, + 'LoadBalancerNames') + self.build_list_params(params, zones, + 'AvailabilityZones') + return self.get_object(op, params, Request) + + def create_auto_scaling_group(self, as_group): + """ + Create auto scaling group. + """ + return self._update_group('CreateAutoScalingGroup', as_group) + + def create_launch_configuration(self, launch_config): + """ + Creates a new Launch Configuration. + + :type launch_config: boto.ec2.autoscale.launchconfig.LaunchConfiguration + :param launch_config: LaunchConfiguraiton object. + + """ + params = { + 'ImageId' : launch_config.image_id, + 'KeyName' : launch_config.key_name, + 'LaunchConfigurationName' : launch_config.name, + 'InstanceType' : launch_config.instance_type, + } + if launch_config.user_data: + params['UserData'] = launch_config.user_data + if launch_config.kernel_id: + params['KernelId'] = launch_config.kernel_id + if launch_config.ramdisk_id: + params['RamdiskId'] = launch_config.ramdisk_id + if launch_config.block_device_mappings: + self.build_list_params(params, launch_config.block_device_mappings, + 'BlockDeviceMappings') + self.build_list_params(params, launch_config.security_groups, + 'SecurityGroups') + return self.get_object('CreateLaunchConfiguration', params, + Request) + + def create_trigger(self, trigger): + """ + + """ + params = {'TriggerName' : trigger.name, + 'AutoScalingGroupName' : trigger.autoscale_group.name, + 'MeasureName' : trigger.measure_name, + 'Statistic' : trigger.statistic, + 'Period' : trigger.period, + 'Unit' : trigger.unit, + 'LowerThreshold' : trigger.lower_threshold, + 'LowerBreachScaleIncrement' : trigger.lower_breach_scale_increment, + 'UpperThreshold' : trigger.upper_threshold, + 'UpperBreachScaleIncrement' : trigger.upper_breach_scale_increment, + 'BreachDuration' : trigger.breach_duration} + # dimensions should be a list of tuples + dimensions = [] + for dim in trigger.dimensions: + name, value = dim + dimensions.append(dict(Name=name, Value=value)) + self.build_list_params(params, dimensions, 'Dimensions') + + req = self.get_object('CreateOrUpdateScalingTrigger', params, + Request) + return req + + def get_all_groups(self, names=None): + """ + """ + params = {} + if names: + self.build_list_params(params, names, 'AutoScalingGroupNames') + return self.get_list('DescribeAutoScalingGroups', params, + [('member', AutoScalingGroup)]) + + def get_all_launch_configurations(self, names=None): + """ + """ + params = {} + if names: + self.build_list_params(params, names, 'LaunchConfigurationNames') + return self.get_list('DescribeLaunchConfigurations', params, + [('member', LaunchConfiguration)]) + + def get_all_activities(self, autoscale_group, + activity_ids=None, + max_records=100): + """ + Get all activities for the given autoscaling group. + + :type autoscale_group: str or AutoScalingGroup object + :param autoscale_group: The auto scaling group to get activities on. + + @max_records: int + :param max_records: Maximum amount of activities to return. + """ + name = autoscale_group + if isinstance(autoscale_group, AutoScalingGroup): + name = autoscale_group.name + params = {'AutoScalingGroupName' : name} + if activity_ids: + self.build_list_params(params, activity_ids, 'ActivityIds') + return self.get_list('DescribeScalingActivities', params, + [('member', Activity)]) + + def get_all_triggers(self, autoscale_group): + params = {'AutoScalingGroupName' : autoscale_group} + return self.get_list('DescribeTriggers', params, + [('member', Trigger)]) + + def terminate_instance(self, instance_id, decrement_capacity=True): + params = { + 'InstanceId' : instance_id, + 'ShouldDecrementDesiredCapacity' : decrement_capacity + } + return self.get_object('TerminateInstanceInAutoScalingGroup', params, + Activity) + diff --git a/storefront/boto/ec2/autoscale/activity.py b/storefront/boto/ec2/autoscale/activity.py new file mode 100644 index 0000000..f895d65 --- /dev/null +++ b/storefront/boto/ec2/autoscale/activity.py @@ -0,0 +1,55 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +class Activity(object): + def __init__(self, connection=None): + self.connection = connection + self.start_time = None + self.activity_id = None + self.progress = None + self.status_code = None + self.cause = None + self.description = None + + def __repr__(self): + return 'Activity:%s status:%s progress:%s' % (self.description, + self.status_code, + self.progress) + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'ActivityId': + self.activity_id = value + elif name == 'StartTime': + self.start_time = value + elif name == 'Progress': + self.progress = value + elif name == 'Cause': + self.cause = value + elif name == 'Description': + self.description = value + elif name == 'StatusCode': + self.status_code = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/ec2/autoscale/group.py b/storefront/boto/ec2/autoscale/group.py new file mode 100644 index 0000000..d9df39f --- /dev/null +++ b/storefront/boto/ec2/autoscale/group.py @@ -0,0 +1,190 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import weakref + +from boto.ec2.zone import Zone +from boto.ec2.elb.listelement import ListElement +from boto.resultset import ResultSet +from boto.ec2.autoscale.trigger import Trigger +from boto.ec2.autoscale.request import Request + +class Instance(object): + def __init__(self, connection=None): + self.connection = connection + self.instance_id = '' + + def __repr__(self): + return 'Instance:%s' % self.instance_id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'InstanceId': + self.instance_id = value + else: + setattr(self, name, value) + + +class AutoScalingGroup(object): + def __init__(self, connection=None, group_name=None, + availability_zone=None, launch_config=None, + availability_zones=None, + load_balancers=None, cooldown=0, + min_size=None, max_size=None): + """ + Creates a new AutoScalingGroup with the specified name. + + You must not have already used up your entire quota of + AutoScalingGroups in order for this call to be successful. Once the + creation request is completed, the AutoScalingGroup is ready to be + used in other calls. + + :type name: str + :param name: Name of autoscaling group. + + :type availability_zone: str + :param availability_zone: An availability zone. DEPRECATED - use the + availability_zones parameter, which expects + a list of availability zone + strings + + :type availability_zone: list + :param availability_zone: List of availability zones. + + :type launch_config: str + :param launch_config: Name of launch configuration name. + + :type load_balancers: list + :param load_balancers: List of load balancers. + + :type minsize: int + :param minsize: Minimum size of group + + :type maxsize: int + :param maxsize: Maximum size of group + + :type cooldown: int + :param cooldown: Amount of time after a Scaling Activity completes + before any further scaling activities can start. + + :rtype: tuple + :return: Updated healthcheck for the instances. + """ + self.name = group_name + self.connection = connection + self.min_size = min_size + self.max_size = max_size + self.created_time = None + self.cooldown = cooldown + self.launch_config = launch_config + if self.launch_config: + self.launch_config_name = self.launch_config.name + else: + self.launch_config_name = None + self.desired_capacity = None + lbs = load_balancers or [] + self.load_balancers = ListElement(lbs) + zones = availability_zones or [] + self.availability_zone = availability_zone + self.availability_zones = ListElement(zones) + self.instances = None + + def __repr__(self): + return 'AutoScalingGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'Instances': + self.instances = ResultSet([('member', Instance)]) + return self.instances + elif name == 'LoadBalancerNames': + return self.load_balancers + elif name == 'AvailabilityZones': + return self.availability_zones + else: + return + + def endElement(self, name, value, connection): + if name == 'MinSize': + self.min_size = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'Cooldown': + self.cooldown = value + elif name == 'LaunchConfigurationName': + self.launch_config_name = value + elif name == 'DesiredCapacity': + self.desired_capacity = value + elif name == 'MaxSize': + self.max_size = value + elif name == 'AutoScalingGroupName': + self.name = value + else: + setattr(self, name, value) + + def set_capacity(self, capacity): + """ Set the desired capacity for the group. """ + params = { + 'AutoScalingGroupName' : self.name, + 'DesiredCapacity' : capacity, + } + req = self.connection.get_object('SetDesiredCapacity', params, + Request) + self.connection.last_request = req + return req + + def update(self): + """ Sync local changes with AutoScaling group. """ + return self.connection._update_group('UpdateAutoScalingGroup', self) + + def shutdown_instances(self): + """ Convenience method which shuts down all instances associated with + this group. + """ + self.min_size = 0 + self.max_size = 0 + self.update() + + def get_all_triggers(self): + """ Get all triggers for this auto scaling group. """ + params = {'AutoScalingGroupName' : self.name} + triggers = self.connection.get_list('DescribeTriggers', params, + [('member', Trigger)]) + + # allow triggers to be able to access the autoscale group + for tr in triggers: + tr.autoscale_group = weakref.proxy(self) + + return triggers + + def delete(self): + """ Delete this auto-scaling group. """ + params = {'AutoScalingGroupName' : self.name} + return self.connection.get_object('DeleteAutoScalingGroup', params, + Request) + + def get_activities(self, activity_ids=None, max_records=100): + """ + Get all activies for this group. + """ + return self.connection.get_all_activities(self, activity_ids, max_records) + diff --git a/storefront/boto/ec2/autoscale/instance.py b/storefront/boto/ec2/autoscale/instance.py new file mode 100644 index 0000000..33f2ae6 --- /dev/null +++ b/storefront/boto/ec2/autoscale/instance.py @@ -0,0 +1,51 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +class Instance(object): + def __init__(self, connection=None): + self.connection = connection + self.instance_id = '' + self.lifecycle_state = None + self.availability_zone = '' + + def __repr__(self): + return 'Instance:%s' % self.instance_id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'InstanceId': + self.instance_id = value + elif name == 'LifecycleState': + self.lifecycle_state = value + elif name == 'AvailabilityZone': + self.availability_zone = value + else: + setattr(self, name, value) + + def terminate(self): + """ Terminate this instance. """ + params = {'LaunchConfigurationName' : self.instance_id} + return self.get_object('DeleteLaunchConfiguration', params, + Request) + diff --git a/storefront/boto/ec2/autoscale/launchconfig.py b/storefront/boto/ec2/autoscale/launchconfig.py new file mode 100644 index 0000000..7587cb6 --- /dev/null +++ b/storefront/boto/ec2/autoscale/launchconfig.py @@ -0,0 +1,98 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +from boto.ec2.autoscale.request import Request +from boto.ec2.elb.listelement import ListElement + + +class LaunchConfiguration(object): + def __init__(self, connection=None, name=None, image_id=None, + key_name=None, security_groups=None, user_data=None, + instance_type='m1.small', kernel_id=None, + ramdisk_id=None, block_device_mappings=None): + """ + A launch configuration. + + :type name: str + :param name: Name of the launch configuration to create. + + :type image_id: str + :param image_id: Unique ID of the Amazon Machine Image (AMI) which was + assigned during registration. + + :type key_name: str + :param key_name: The name of the EC2 key pair. + + :type security_groups: list + :param security_groups: Names of the security groups with which to + associate the EC2 instances. + + """ + self.connection = connection + self.name = name + self.instance_type = instance_type + self.block_device_mappings = block_device_mappings + self.key_name = key_name + sec_groups = security_groups or [] + self.security_groups = ListElement(sec_groups) + self.image_id = image_id + self.ramdisk_id = ramdisk_id + self.created_time = None + self.kernel_id = kernel_id + self.user_data = user_data + self.created_time = None + + def __repr__(self): + return 'LaunchConfiguration:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'SecurityGroups': + return self.security_groups + else: + return + + def endElement(self, name, value, connection): + if name == 'InstanceType': + self.instance_type = value + elif name == 'LaunchConfigurationName': + self.name = value + elif name == 'KeyName': + self.key_name = value + elif name == 'ImageId': + self.image_id = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'KernelId': + self.kernel_id = value + elif name == 'RamdiskId': + self.ramdisk_id = value + elif name == 'UserData': + self.user_data = value + else: + setattr(self, name, value) + + def delete(self): + """ Delete this launch configuration. """ + params = {'LaunchConfigurationName' : self.name} + return self.connection.get_object('DeleteLaunchConfiguration', params, + Request) + diff --git a/storefront/boto/ec2/autoscale/request.py b/storefront/boto/ec2/autoscale/request.py new file mode 100644 index 0000000..c066dff --- /dev/null +++ b/storefront/boto/ec2/autoscale/request.py @@ -0,0 +1,38 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Request(object): + def __init__(self, connection=None): + self.connection = connection + self.request_id = '' + + def __repr__(self): + return 'Request:%s' % self.request_id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'RequestId': + self.request_id = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/ec2/autoscale/trigger.py b/storefront/boto/ec2/autoscale/trigger.py new file mode 100644 index 0000000..197803d --- /dev/null +++ b/storefront/boto/ec2/autoscale/trigger.py @@ -0,0 +1,137 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import weakref + +from boto.ec2.autoscale.request import Request + + +class Trigger(object): + """ + An auto scaling trigger. + """ + + def __init__(self, connection=None, name=None, autoscale_group=None, + dimensions=None, measure_name=None, + statistic=None, unit=None, period=60, + lower_threshold=None, + lower_breach_scale_increment=None, + upper_threshold=None, + upper_breach_scale_increment=None, + breach_duration=None): + """ + Initialize an auto-scaling trigger object. + + :type name: str + :param name: The name for this trigger + + :type autoscale_group: str + :param autoscale_group: The name of the AutoScalingGroup that will be + associated with the trigger. The AutoScalingGroup + that will be affected by the trigger when it is + activated. + + :type dimensions: list + :param dimensions: List of tuples, i.e. + ('ImageId', 'i-13lasde') etc. + + :type measure_name: str + :param measure_name: The measure name associated with the metric used by + the trigger to determine when to activate, for + example, CPU, network I/O, or disk I/O. + + :type statistic: str + :param statistic: The particular statistic used by the trigger when + fetching metric statistics to examine. + + :type period: int + :param period: The period associated with the metric statistics in + seconds. Valid Values: 60 or a multiple of 60. + + :type unit: + :param unit + + :type lower_threshold: + :param lower_threshold + """ + self.name = name + self.connection = connection + self.dimensions = dimensions + self.breach_duration = breach_duration + self.upper_breach_scale_increment = upper_breach_scale_increment + self.created_time = None + self.upper_threshold = upper_threshold + self.status = None + self.lower_threshold = lower_threshold + self.period = period + self.lower_breach_scale_increment = lower_breach_scale_increment + self.statistic = statistic + self.unit = unit + self.namespace = None + if autoscale_group: + self.autoscale_group = weakref.proxy(autoscale_group) + else: + self.autoscale_group = None + self.measure_name = measure_name + + def __repr__(self): + return 'Trigger:%s' % (self.name) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'BreachDuration': + self.breach_duration = value + elif name == 'TriggerName': + self.name = value + elif name == 'Period': + self.period = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'Statistic': + self.statistic = value + elif name == 'Unit': + self.unit = value + elif name == 'Namespace': + self.namespace = value + elif name == 'AutoScalingGroupName': + self.autoscale_group_name = value + elif name == 'MeasureName': + self.measure_name = value + else: + setattr(self, name, value) + + def update(self): + """ Write out differences to trigger. """ + self.connection.create_trigger(self) + + def delete(self): + """ Delete this trigger. """ + params = { + 'TriggerName' : self.name, + 'AutoScalingGroupName' : self.autoscale_group_name, + } + req =self.connection.get_object('DeleteTrigger', params, + Request) + self.connection.last_request = req + return req + diff --git a/storefront/boto/ec2/blockdevicemapping.py b/storefront/boto/ec2/blockdevicemapping.py new file mode 100644 index 0000000..ef7163a --- /dev/null +++ b/storefront/boto/ec2/blockdevicemapping.py @@ -0,0 +1,89 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +class EBSBlockDeviceType(object): + + def __init__(self, connection=None): + self.connection = connection + self.volume_id = None + self.snapshot_id = None + self.status = None + self.attach_time = None + self.delete_on_termination = False + self.size = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name =='volumeId': + self.volume_id = value + elif name =='snapshotId': + self.snapshot_id = value + elif name == 'volumeSize': + self.size = int(value) + elif name == 'status': + self.status = value + elif name == 'attachTime': + self.attach_time = value + elif name == 'deleteOnTermination': + if value == 'true': + self.delete_on_termination = True + else: + self.delete_on_termination = False + else: + setattr(self, name, value) + +class BlockDeviceMapping(dict): + + def __init__(self, connection=None): + dict.__init__(self) + self.connection = connection + self.current_name = None + self.current_value = None + + def startElement(self, name, attrs, connection): + if name == 'ebs': + self.current_value = EBSBlockDeviceType(self) + return self.current_value + + def endElement(self, name, value, connection): + if name == 'device' or name == 'deviceName': + self.current_name = value + elif name == 'item': + self[self.current_name] = self.current_value + + def build_list_params(self, params, prefix=''): + i = 1 + for dev_name in self: + pre = '%sBlockDeviceMapping.%d' % (pre, i) + params['%s.DeviceName' % pre] = dev_name + ebs = self[dev_name] + if ebs.snapshot_id: + params['%s.Ebs.SnapshotId' % pre] = ebs.snapshot_id + if ebs.size: + params['%s.Ebs.VolumeSize' % pre] = ebs.size + if ebs.delete_on_termination: + params['%s.Ebs.DeleteOnTermination' % pre] = 'true' + else: + params['%s.Ebs.DeleteOnTermination' % pre] = 'false' + i += 1 diff --git a/storefront/boto/ec2/buyreservation.py b/storefront/boto/ec2/buyreservation.py new file mode 100644 index 0000000..ba65590 --- /dev/null +++ b/storefront/boto/ec2/buyreservation.py @@ -0,0 +1,81 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto.ec2 +from boto.sdb.db.property import * +from boto.manage import propget + +InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', 'c1.medium', 'c1.xlarge'] + +class BuyReservation(object): + + def get_region(self, params): + if not params.get('region', None): + prop = StringProperty(name='region', verbose_name='EC2 Region', + choices=boto.ec2.regions) + params['region'] = propget.get(prop, choices=boto.ec2.regions) + + def get_instance_type(self, params): + if not params.get('instance_type', None): + prop = StringProperty(name='instance_type', verbose_name='Instance Type', + choices=InstanceTypes) + params['instance_type'] = propget.get(prop) + + def get_quantity(self, params): + if not params.get('quantity', None): + prop = IntegerProperty(name='quantity', verbose_name='Number of Instances') + params['quantity'] = propget.get(prop) + + def get_zone(self, params): + if not params.get('zone', None): + prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', + choices=self.ec2.get_all_zones) + params['zone'] = propget.get(prop) + + def get(self, params): + self.get_region(params) + self.ec2 = params['region'].connect() + self.get_instance_type(params) + self.get_zone(params) + self.get_quantity(params) + +if __name__ == "__main__": + obj = BuyReservation() + params = {} + obj.get(params) + offerings = obj.ec2.get_all_reserved_instances_offerings(instance_type=params['instance_type'], + availability_zone=params['zone'].name) + print '\nThe following Reserved Instances Offerings are available:\n' + for offering in offerings: + offering.describe() + prop = StringProperty(name='offering', verbose_name='Offering', + choices=offerings) + offering = propget.get(prop) + print '\nYou have chosen this offering:' + offering.describe() + unit_price = float(offering.fixed_price) + total_price = unit_price * params['quantity'] + print '!!! You are about to purchase %d of these offerings for a total of $%.2f !!!' % (params['quantity'], total_price) + answer = raw_input('Are you sure you want to do this? If so, enter YES: ') + if answer.strip().lower() == 'yes': + offering.purchase(params['quantity']) + else: + print 'Purchase cancelled' diff --git a/storefront/boto/ec2/cloudwatch/__init__.py b/storefront/boto/ec2/cloudwatch/__init__.py new file mode 100644 index 0000000..1c606a1 --- /dev/null +++ b/storefront/boto/ec2/cloudwatch/__init__.py @@ -0,0 +1,206 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +CloudWatch service from AWS. + +The 5 Minute How-To Guide +------------------------- +First, make sure you have something to monitor. You can either create a +LoadBalancer or enable monitoring on an existing EC2 instance. To enable +monitoring, you can either call the monitor_instance method on the +EC2Connection object or call the monitor method on the Instance object. + +It takes a while for the monitoring data to start accumulating but once +it does, you can do this: + +>>> import boto +>>> c = boto.connect_cloudwatch() +>>> metrics = c.list_metrics() +>>> metrics +[Metric:NetworkIn, + Metric:NetworkOut, + Metric:NetworkOut(InstanceType,m1.small), + Metric:NetworkIn(InstanceId,i-e573e68c), + Metric:CPUUtilization(InstanceId,i-e573e68c), + Metric:DiskWriteBytes(InstanceType,m1.small), + Metric:DiskWriteBytes(ImageId,ami-a1ffb63), + Metric:NetworkOut(ImageId,ami-a1ffb63), + Metric:DiskWriteOps(InstanceType,m1.small), + Metric:DiskReadBytes(InstanceType,m1.small), + Metric:DiskReadOps(ImageId,ami-a1ffb63), + Metric:CPUUtilization(InstanceType,m1.small), + Metric:NetworkIn(ImageId,ami-a1ffb63), + Metric:DiskReadOps(InstanceType,m1.small), + Metric:DiskReadBytes, + Metric:CPUUtilization, + Metric:DiskWriteBytes(InstanceId,i-e573e68c), + Metric:DiskWriteOps(InstanceId,i-e573e68c), + Metric:DiskWriteOps, + Metric:DiskReadOps, + Metric:CPUUtilization(ImageId,ami-a1ffb63), + Metric:DiskReadOps(InstanceId,i-e573e68c), + Metric:NetworkOut(InstanceId,i-e573e68c), + Metric:DiskReadBytes(ImageId,ami-a1ffb63), + Metric:DiskReadBytes(InstanceId,i-e573e68c), + Metric:DiskWriteBytes, + Metric:NetworkIn(InstanceType,m1.small), + Metric:DiskWriteOps(ImageId,ami-a1ffb63)] + +The list_metrics call will return a list of all of the available metrics +that you can query against. Each entry in the list is a Metric object. +As you can see from the list above, some of the metrics are generic metrics +and some have Dimensions associated with them (e.g. InstanceType=m1.small). +The Dimension can be used to refine your query. So, for example, I could +query the metric Metric:CPUUtilization which would create the desired statistic +by aggregating cpu utilization data across all sources of information available +or I could refine that by querying the metric +Metric:CPUUtilization(InstanceId,i-e573e68c) which would use only the data +associated with the instance identified by the instance ID i-e573e68c. + +Because for this example, I'm only monitoring a single instance, the set +of metrics available to me are fairly limited. If I was monitoring many +instances, using many different instance types and AMI's and also several +load balancers, the list of available metrics would grow considerably. + +Once you have the list of available metrics, you can actually +query the CloudWatch system for that metric. Let's choose the CPU utilization +metric for our instance. + +>>> m = metrics[5] +>>> m +Metric:CPUUtilization(InstanceId,i-e573e68c) + +The Metric object has a query method that lets us actually perform +the query against the collected data in CloudWatch. To call that, +we need a start time and end time to control the time span of data +that we are interested in. For this example, let's say we want the +data for the previous hour: + +>>> import datetime +>>> end = datetime.datetime.now() +>>> start = end - datetime.timedelta(hours=1) + +We also need to supply the Statistic that we want reported and +the Units to use for the results. The Statistic can be one of these +values: + +['Minimum', 'Maximum', 'Sum', 'Average', 'Samples'] + +And Units must be one of the following: + +['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', +'Bytes/Second', 'Bits/Second', 'Count/Second'] + +The query method also takes an optional parameter, period. This +parameter controls the granularity (in seconds) of the data returned. +The smallest period is 60 seconds and the value must be a multiple +of 60 seconds. So, let's ask for the average as a percent: + +>>> datapoints = m.query(start, end, 'Average', 'Percent') +>>> len(datapoints) +60 + +Our period was 60 seconds and our duration was one hour so +we should get 60 data points back and we can see that we did. +Each element in the datapoints list is a DataPoint object +which is a simple subclass of a Python dict object. Each +Datapoint object contains all of the information available +about that particular data point. + +>>> d = datapoints[0] +>>> d +{u'Average': 0.0, + u'Samples': 1.0, + u'Timestamp': u'2009-05-21T19:55:00Z', + u'Unit': u'Percent'} + +My server obviously isn't very busy right now! +""" +from boto.connection import AWSQueryConnection +from boto.ec2.cloudwatch.metric import Metric +from boto.ec2.cloudwatch.datapoint import Datapoint +import boto +import datetime + +class CloudWatchConnection(AWSQueryConnection): + + APIVersion = boto.config.get('Boto', 'cloudwatch_version', '2009-05-15') + Endpoint = boto.config.get('Boto', 'cloudwatch_endpoint', 'monitoring.amazonaws.com') + SignatureVersion = '2' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=Endpoint, debug=0, + https_connection_factory=None, path='/'): + """ + Init method to create a new connection to EC2 Monitoring Service. + + B{Note:} The host argument is overridden by the host specified in the boto configuration file. + """ + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory, path) + + def build_list_params(self, params, items, label): + if isinstance(items, str): + items = [items] + for i in range(1, len(items)+1): + params[label % i] = items[i-1] + + def get_metric_statistics(self, period, start_time, end_time, measure_name, + namespace, statistics=None, dimensions=None, unit=None): + """ + Get time-series data for one or more statistics of a given metric. + + :type measure_name: string + :param measure_name: CPUUtilization|NetworkIO-in|NetworkIO-out|DiskIO-ALL-read| + DiskIO-ALL-write|DiskIO-ALL-read-bytes|DiskIO-ALL-write-bytes + + :rtype: list + :return: A list of :class:`boto.ec2.image.Image` + """ + params = {'Period' : period, + 'MeasureName' : measure_name, + 'Namespace' : namespace, + 'StartTime' : start_time.isoformat(), + 'EndTime' : end_time.isoformat()} + if dimensions: + i = 1 + for name in dimensions: + params['Dimensions.member.%d.Name' % i] = name + params['Dimensions.member.%d.Value' % i] = dimensions[name] + i += 1 + if statistics: + self.build_list_params(params, statistics, 'Statistics.member.%d') + return self.get_list('GetMetricStatistics', params, [('member', Datapoint)]) + + def list_metrics(self): + """ + Returns a list of the valid metrics for which there is recorded data available. + """ + response = self.make_request('ListMetrics') + body = response.read() + return self.get_list('ListMetrics', None, [('member', Metric)]) + + + diff --git a/storefront/boto/ec2/cloudwatch/datapoint.py b/storefront/boto/ec2/cloudwatch/datapoint.py new file mode 100644 index 0000000..1860f4a --- /dev/null +++ b/storefront/boto/ec2/cloudwatch/datapoint.py @@ -0,0 +1,37 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +class Datapoint(dict): + + def __init__(self, connection=None): + dict.__init__(self) + self.connection = connection + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name in ['Average', 'Maximum', 'Minimum', 'Samples', 'Sum']: + self[name] = float(value) + elif name != 'member': + self[name] = value + diff --git a/storefront/boto/ec2/cloudwatch/metric.py b/storefront/boto/ec2/cloudwatch/metric.py new file mode 100644 index 0000000..e4661f4 --- /dev/null +++ b/storefront/boto/ec2/cloudwatch/metric.py @@ -0,0 +1,71 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +class Dimensions(dict): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'Name': + self._name = value + elif name == 'Value': + self[self._name] = value + elif name != 'Dimensions' and name != 'member': + self[name] = value + +class Metric(object): + + Statistics = ['Minimum', 'Maximum', 'Sum', 'Average', 'Samples'] + Units = ['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', + 'Bytes/Second', 'Bits/Second', 'Count/Second'] + + def __init__(self, connection=None): + self.connection = connection + self.name = None + self.namespace = None + self.dimensions = None + + def __repr__(self): + s = 'Metric:%s' % self.name + if self.dimensions: + for name,value in self.dimensions.items(): + s += '(%s,%s)' % (name, value) + return s + + def startElement(self, name, attrs, connection): + if name == 'Dimensions': + self.dimensions = Dimensions() + return self.dimensions + + def endElement(self, name, value, connection): + if name == 'MeasureName': + self.name = value + elif name == 'Namespace': + self.namespace = value + else: + setattr(self, name, value) + + def query(self, start_time, end_time, statistic, unit, period=60): + return self.connection.get_metric_statistics(period, start_time, end_time, + self.name, self.namespace, [statistic], + self.dimensions, unit) diff --git a/storefront/boto/ec2/connection.py b/storefront/boto/ec2/connection.py new file mode 100644 index 0000000..9574986 --- /dev/null +++ b/storefront/boto/ec2/connection.py @@ -0,0 +1,1520 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a connection to the EC2 service. +""" + +import urllib +import xml.sax +import base64 +import boto +from boto import config +from boto.connection import AWSQueryConnection +from boto.resultset import ResultSet +from boto.ec2.image import Image, ImageAttribute +from boto.ec2.instance import Reservation, Instance, ConsoleOutput, InstanceAttribute +from boto.ec2.keypair import KeyPair +from boto.ec2.address import Address +from boto.ec2.volume import Volume +from boto.ec2.snapshot import Snapshot +from boto.ec2.snapshot import SnapshotAttribute +from boto.ec2.zone import Zone +from boto.ec2.securitygroup import SecurityGroup +from boto.ec2.regioninfo import RegionInfo +from boto.ec2.instanceinfo import InstanceInfo +from boto.ec2.reservedinstance import ReservedInstancesOffering, ReservedInstance +from boto.ec2.spotinstancerequest import SpotInstanceRequest +from boto.ec2.spotpricehistory import SpotPriceHistory +from boto.ec2.spotdatafeedsubscription import SpotDatafeedSubscription +from boto.ec2.launchspecification import LaunchSpecification +from boto.exception import EC2ResponseError + +#boto.set_stream_logger('ec2') + +class EC2Connection(AWSQueryConnection): + + APIVersion = boto.config.get('Boto', 'ec2_version', '2009-11-30') + DefaultRegionName = boto.config.get('Boto', 'ec2_region_name', 'us-east-1') + DefaultRegionEndpoint = boto.config.get('Boto', 'ec2_region_endpoint', + 'ec2.amazonaws.com') + SignatureVersion = '2' + ResponseError = EC2ResponseError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, host=None, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/'): + """ + Init method to create a new connection to EC2. + + B{Note:} The host argument is overridden by the host specified in the boto configuration file. + """ + if not region: + region = RegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, + self.region.endpoint, debug, + https_connection_factory, path) + + def get_params(self): + """ + Returns a dictionary containing the value of of all of the keyword + arguments passed when constructing this connection. + """ + param_names = ['aws_access_key_id', 'aws_secret_access_key', 'is_secure', + 'port', 'proxy', 'proxy_port', 'proxy_user', 'proxy_pass', + 'debug', 'https_connection_factory'] + params = {} + for name in param_names: + params[name] = getattr(self, name) + return params + + # Image methods + + def get_all_images(self, image_ids=None, owners=None, executable_by=None): + """ + Retrieve all the EC2 images available on your account. + + :type image_ids: list + :param image_ids: A list of strings with the image IDs wanted + + :type owners: list + :param owners: A list of owner IDs + + :type executable_by: + :param executable_by: + + :rtype: list + :return: A list of :class:`boto.ec2.image.Image` + """ + params = {} + if image_ids: + self.build_list_params(params, image_ids, 'ImageId') + if owners: + self.build_list_params(params, owners, 'Owner') + if executable_by: + self.build_list_params(params, executable_by, 'ExecutableBy') + return self.get_list('DescribeImages', params, [('item', Image)]) + + def get_all_kernels(self, kernel_ids=None, owners=None): + """ + Retrieve all the EC2 kernels available on your account. Simply filters the list returned + by get_all_images because EC2 does not provide a way to filter server-side. + + :type kernel_ids: list + :param kernel_ids: A list of strings with the image IDs wanted + + :type owners: list + :param owners: A list of owner IDs + + :rtype: list + :return: A list of :class:`boto.ec2.image.Image` + """ + rs = self.get_all_images(kernel_ids, owners) + kernels = [] + for image in rs: + if image.type == 'kernel': + kernels.append(image) + return kernels + + def get_all_ramdisks(self, ramdisk_ids=None, owners=None): + """ + Retrieve all the EC2 ramdisks available on your account. + Simply filters the list returned by get_all_images because + EC2 does not provide a way to filter server-side. + + :type ramdisk_ids: list + :param ramdisk_ids: A list of strings with the image IDs wanted + + :type owners: list + :param owners: A list of owner IDs + + :rtype: list + :return: A list of :class:`boto.ec2.image.Image` + """ + rs = self.get_all_images(ramdisk_ids, owners) + ramdisks = [] + for image in rs: + if image.type == 'ramdisk': + ramdisks.append(image) + return ramdisks + + def get_image(self, image_id): + """ + Shortcut method to retrieve a specific image (AMI). + + :type image_id: string + :param image_id: the ID of the Image to retrieve + + :rtype: :class:`boto.ec2.image.Image` + :return: The EC2 Image specified or None if the image is not found + """ + try: + return self.get_all_images(image_ids=[image_id])[0] + except IndexError: # None of those images available + return None + + def register_image(self, name, description=None, image_location=None, + architecture=None, kernel_id=None, ramdisk_id=None, + root_device_name=None, block_device_map=None): + """ + Register an image. + + :type name: string + :param name: The name of the AMI. + + :type description: string + :param description: The description of the AMI. + + :type image_location: string + :param image_location: Full path to your AMI manifest in Amazon S3 storage. + Only used for S3-based AMI's. + + :type architecture: string + :param architecture: The architecture of the AMI. Valid choices are: + i386 | x86_64 + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the instances + + :type root_device_name: string + :param root_device_name: The root device name (e.g. /dev/sdh) + + :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` + :param block_device_map: A BlockDeviceMapping data structure + describing the EBS volumes associated + with the Image. + + :rtype: string + :return: The new image id + """ + params = {'Name': name} + if description: + params['Description'] = description + if architecture: + params['Architecture'] = architecture + if kernel_id: + params['KernelId'] = kernel_id + if ramdisk_id: + params['RamdiskId'] = ramdisk_id + if image_location: + params['Location'] = image_location + if root_device_name: + params['RootDeviceName'] = root_device_name + if block_device_map: + block_device_map.build_list_params(params) + rs = self.get_object('RegisterImage', params, ResultSet) + image_id = getattr(rs, 'imageId', None) + return image_id + + def deregister_image(self, image_id): + """ + Unregister an AMI. + + :type image_id: string + :param image_id: the ID of the Image to unregister + + :rtype: bool + :return: True if successful + """ + return self.get_status('DeregisterImage', {'ImageId':image_id}) + + def create_image(self, instance_id, name, description=None, no_reboot=False): + """ + Will create an AMI from the instance in the running or stopped + state. + + :type instance_id: string + :param instance_id: the ID of the instance to image. + + :type name: string + :param name: The name of the new image + + :type description: string + :param description: An optional human-readable string describing + the contents and purpose of the AMI. + + :type no_reboot: bool + :param no_reboot: An optional flag indicating that the bundling process + should not attempt to shutdown the instance before + bundling. If this flag is True, the responsibility + of maintaining file system integrity is left to the + owner of the instance. + + :rtype: string + :return: The new image id + """ + params = {'InstanceId' : instance_id, + 'Name' : name} + if description: + params['Description'] = description + if no_reboot: + params['NoReboot'] = 'true' + rs = self.get_object('CreateImage', params, Image) + image_id = getattr(rs, 'imageId', None) + if not image_id: + image_id = getattr(rs, 'ImageId', None) + return image_id + + # ImageAttribute methods + + def get_image_attribute(self, image_id, attribute='launchPermission'): + """ + Gets an attribute from an image. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-DescribeImageAttribute.html + + :type image_id: string + :param image_id: The Amazon image id for which you want info about + + :type attribute: string + :param attribute: The attribute you need information about. + Valid choices are: + * launchPermission + * productCodes + * blockDeviceMapping + + :rtype: :class:`boto.ec2.image.ImageAttribute` + :return: An ImageAttribute object representing the value of the attribute requested + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute} + return self.get_object('DescribeImageAttribute', params, ImageAttribute) + + def modify_image_attribute(self, image_id, attribute='launchPermission', + operation='add', user_ids=None, groups=None, + product_codes=None): + """ + Changes an attribute of an image. + See http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-ModifyImageAttribute.html + + :type image_id: string + :param image_id: The image id you wish to change + + :type attribute: string + :param attribute: The attribute you wish to change + + :type operation: string + :param operation: Either add or remove (this is required for changing launchPermissions) + + :type user_ids: list + :param user_ids: The Amazon IDs of users to add/remove attributes + + :type groups: list + :param groups: The groups to add/remove attributes + + :type product_codes: list + :param product_codes: Amazon DevPay product code. Currently only one + product code can be associated with an AMI. Once + set, the product code cannot be changed or reset. + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute, + 'OperationType' : operation} + if user_ids: + self.build_list_params(params, user_ids, 'UserId') + if groups: + self.build_list_params(params, groups, 'UserGroup') + if product_codes: + self.build_list_params(params, product_codes, 'ProductCode') + return self.get_status('ModifyImageAttribute', params) + + def reset_image_attribute(self, image_id, attribute='launchPermission'): + """ + Resets an attribute of an AMI to its default value. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-ResetImageAttribute.html + + :type image_id: string + :param image_id: ID of the AMI for which an attribute will be described + + :type attribute: string + :param attribute: The attribute to reset + + :rtype: bool + :return: Whether the operation succeeded or not + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute} + return self.get_status('ResetImageAttribute', params) + + # Instance methods + + def get_all_instances(self, instance_ids=None): + """ + Retrieve all the instances associated with your account. + + :type instance_ids: list + :param instance_ids: A list of strings of instance IDs + + :rtype: list + :return: A list of :class:`boto.ec2.instance.Reservation` + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('DescribeInstances', params, [('item', Reservation)]) + + def run_instances(self, image_id, min_count=1, max_count=1, + key_name=None, security_groups=None, + user_data=None, addressing_type=None, + instance_type='m1.small', placement=None, + kernel_id=None, ramdisk_id=None, + monitoring_enabled=False, subnet_id=None, + block_device_map=None): + """ + Runs an image on EC2. + + :type image_id: string + :param image_id: The ID of the image to run + + :type min_count: int + :param min_count: The minimum number of instances to launch + + :type max_count: int + :param max_count: The maximum number of instances to launch + + :type key_name: string + :param key_name: The name of the key pair with which to launch instances + + :type security_groups: list of strings + :param security_groups: The names of the security groups with which to associate instances + + :type user_data: string + :param user_data: The user data passed to the launched instances + + :type instance_type: string + :param instance_type: The type of instance to run (m1.small, m1.large, m1.xlarge) + + :type placement: string + :param placement: The availability zone in which to launch the instances + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the instances + + :type ramdisk_id: string + :param ramdisk_id: The ID of the RAM disk with which to launch the instances + + :type monitoring_enabled: bool + :param monitoring_enabled: Enable CloudWatch monitoring on the instance. + + :type subnet_id: string + :param subnet_id: The subnet ID within which to launch the instances for VPC. + + :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` + :param block_device_map: A BlockDeviceMapping data structure + describing the EBS volumes associated + with the Image. + + :rtype: Reservation + :return: The :class:`boto.ec2.instance.Reservation` associated with the request for machines + """ + params = {'ImageId':image_id, + 'MinCount':min_count, + 'MaxCount': max_count} + if key_name: + params['KeyName'] = key_name + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, SecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, 'SecurityGroup') + if user_data: + params['UserData'] = base64.b64encode(user_data) + if addressing_type: + params['AddressingType'] = addressing_type + if instance_type: + params['InstanceType'] = instance_type + if placement: + params['Placement.AvailabilityZone'] = placement + if kernel_id: + params['KernelId'] = kernel_id + if ramdisk_id: + params['RamdiskId'] = ramdisk_id + if monitoring_enabled: + params['Monitoring.Enabled'] = 'true' + if subnet_id: + params['SubnetId'] = subnet_id + if block_device_map: + block_device_map.build_list_params(params) + return self.get_object('RunInstances', params, Reservation, verb='POST') + + def terminate_instances(self, instance_ids=None): + """ + Terminate the instances specified + + :type instance_ids: list + :param instance_ids: A list of strings of the Instance IDs to terminate + + :rtype: list + :return: A list of the instances terminated + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('TerminateInstances', params, [('item', Instance)]) + + def stop_instances(self, instance_ids=None): + """ + Stop the instances specified + + :type instance_ids: list + :param instance_ids: A list of strings of the Instance IDs to stop + + :rtype: list + :return: A list of the instances stopped + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('StopInstances', params, [('item', Instance)]) + + def start_instances(self, instance_ids=None): + """ + Start the instances specified + + :type instance_ids: list + :param instance_ids: A list of strings of the Instance IDs to start + + :rtype: list + :return: A list of the instances started + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('StartInstances', params, [('item', Instance)]) + + def get_console_output(self, instance_id): + """ + Retrieves the console output for the specified instance. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-GetConsoleOutput.html + + :type instance_id: string + :param instance_id: The instance ID of a running instance on the cloud. + + :rtype: :class:`boto.ec2.instance.ConsoleOutput` + :return: The console output as a ConsoleOutput object + """ + params = {} + self.build_list_params(params, [instance_id], 'InstanceId') + return self.get_object('GetConsoleOutput', params, ConsoleOutput) + + def reboot_instances(self, instance_ids=None): + """ + Reboot the specified instances. + + :type instance_ids: list + :param instance_ids: The instances to terminate and reboot + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_status('RebootInstances', params) + + def confirm_product_instance(self, product_code, instance_id): + params = {'ProductCode' : product_code, + 'InstanceId' : instance_id} + rs = self.get_object('ConfirmProductInstance', params, ResultSet) + return (rs.status, rs.ownerId) + + # InstanceAttribute methods + + def get_instance_attribute(self, instance_id, attribute): + """ + Gets an attribute from an instance. + + :type instance_id: string + :param instance_id: The Amazon id of the instance + + :type attribute: string + :param attribute: The attribute you need information about + Valid choices are: + instanceType|kernel|ramdisk|userData| + disableApiTermination| + instanceInitiatedShutdownBehavior| + rootDeviceName|blockDeviceMapping + + :rtype: :class:`boto.ec2.image.ImageAttribute` + :return: An ImageAttribute object representing the value of the attribute requested + """ + params = {'InstanceId' : instance_id} + if attribute: + params['Attribute'] = attribute + return self.get_object('DescribeInstanceAttribute', params, InstanceAttribute) + + def modify_image_attribute(self, image_id, attribute='launchPermission', + operation='add', user_ids=None, groups=None, + product_codes=None): + """ + Changes an attribute of an image. + See http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-ModifyImageAttribute.html + + :type image_id: string + :param image_id: The image id you wish to change + + :type attribute: string + :param attribute: The attribute you wish to change + + :type operation: string + :param operation: Either add or remove (this is required for changing launchPermissions) + + :type user_ids: list + :param user_ids: The Amazon IDs of users to add/remove attributes + + :type groups: list + :param groups: The groups to add/remove attributes + + :type product_codes: list + :param product_codes: Amazon DevPay product code. Currently only one + product code can be associated with an AMI. Once + set, the product code cannot be changed or reset. + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute, + 'OperationType' : operation} + if user_ids: + self.build_list_params(params, user_ids, 'UserId') + if groups: + self.build_list_params(params, groups, 'UserGroup') + if product_codes: + self.build_list_params(params, product_codes, 'ProductCode') + return self.get_status('ModifyImageAttribute', params) + + def reset_image_attribute(self, image_id, attribute='launchPermission'): + """ + Resets an attribute of an AMI to its default value. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-ResetImageAttribute.html + + :type image_id: string + :param image_id: ID of the AMI for which an attribute will be described + + :type attribute: string + :param attribute: The attribute to reset + + :rtype: bool + :return: Whether the operation succeeded or not + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute} + return self.get_status('ResetImageAttribute', params) + + # Spot Instances + + def get_all_spot_instance_requests(self, request_ids=None): + """ + Retrieve all the spot instances requests associated with your account. + + @type request_ids: list + @param request_ids: A list of strings of spot instance request IDs + + @rtype: list + @return: A list of + :class:`boto.ec2.spotinstancerequest.SpotInstanceRequest` + """ + params = {} + if request_ids: + self.build_list_params(params, request_ids, 'SpotInstanceRequestId') + return self.get_list('DescribeSpotInstanceRequests', params, + [('item', SpotInstanceRequest)]) + + def get_spot_price_history(self, start_time=None, end_time=None, + instance_type=None, product_description=None): + """ + Retrieve the recent history of spot instances pricing. + + @type start_time: str + @param start_time: An indication of how far back to provide price + changes for. An ISO8601 DateTime string. + + @type end_time: str + @param end_time: An indication of how far forward to provide price + changes for. An ISO8601 DateTime string. + + @type instance_type: str + @param instance_type: Filter responses to a particular instance type. + + @type product_description: str + @param product_descripton: Filter responses to a particular platform. + Valid values are currently: Linux + + @rtype: list + @return: A list tuples containing price and timestamp. + """ + params = {} + if start_time: + params['StartTime'] = start_time + if end_time: + params['EndTime'] = end_time + if instance_type: + params['InstanceType'] = instance_type + if product_description: + params['ProductDescription'] = product_description + return self.get_list('DescribeSpotPriceHistory', params, [('item', SpotPriceHistory)]) + + def request_spot_instances(self, price, image_id, count=1, type=None, + valid_from=None, valid_until=None, + launch_group=None, availability_zone_group=None, + key_name=None, security_groups=None, + user_data=None, addressing_type=None, + instance_type='m1.small', placement=None, + kernel_id=None, ramdisk_id=None, + monitoring_enabled=False, subnet_id=None, + block_device_map=None): + """ + Request instances on the spot market at a particular price. + + :type price: str + :param price: The maximum price of your bid + + :type image_id: string + :param image_id: The ID of the image to run + + :type count: int + :param count: The of instances to requested + + :type type: str + :param type: Type of request. Can be 'one-time' or 'persistent'. + Default is one-time. + + :type valid_from: str + :param valid_from: Start date of the request. An ISO8601 time string. + + :type valid_until: str + :param valid_until: End date of the request. An ISO8601 time string. + + :type launch_group: str + :param launch_group: If supplied, all requests will be fulfilled + as a group. + + :type availability_zone_group: str + :param availability_zone_group: If supplied, all requests will be fulfilled + within a single availability zone. + + :type key_name: string + :param key_name: The name of the key pair with which to launch instances + + :type security_groups: list of strings + :param security_groups: The names of the security groups with which to associate instances + + :type user_data: string + :param user_data: The user data passed to the launched instances + + :type instance_type: string + :param instance_type: The type of instance to run (m1.small, m1.large, m1.xlarge) + + :type placement: string + :param placement: The availability zone in which to launch the instances + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the instances + + :type ramdisk_id: string + :param ramdisk_id: The ID of the RAM disk with which to launch the instances + + :type monitoring_enabled: bool + :param monitoring_enabled: Enable CloudWatch monitoring on the instance. + + :type subnet_id: string + :param subnet_id: The subnet ID within which to launch the instances for VPC. + + :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` + :param block_device_map: A BlockDeviceMapping data structure + describing the EBS volumes associated + with the Image. + + :rtype: Reservation + :return: The :class:`boto.ec2.instance.Reservation` associated with the request for machines + """ + params = {'LaunchSpecification.ImageId':image_id, + 'SpotPrice' : price} + if count: + params['InstanceCount'] = count + if valid_from: + params['ValidFrom'] = valid_from + if valid_until: + params['ValidUntil'] = valid_until + if launch_group: + params['LaunchGroup'] = launch_group + if availability_zone_group: + params['AvailabilityZoneGroup'] = availability_zone_group + if key_name: + params['LaunchSpecification.KeyName'] = key_name + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, SecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, + 'LaunchSpecification.SecurityGroup') + if user_data: + params['LaunchSpecification.UserData'] = base64.b64encode(user_data) + if addressing_type: + params['LaunchSpecification.AddressingType'] = addressing_type + if instance_type: + params['LaunchSpecification.InstanceType'] = instance_type + if placement: + params['LaunchSpecification.Placement.AvailabilityZone'] = placement + if kernel_id: + params['LaunchSpecification.KernelId'] = kernel_id + if ramdisk_id: + params['LaunchSpecification.RamdiskId'] = ramdisk_id + if monitoring_enabled: + params['LaunchSpecification.Monitoring.Enabled'] = 'true' + if subnet_id: + params['LaunchSpecification.SubnetId'] = subnet_id + if block_device_map: + block_device_map.build_list_params(params, 'LaunchSpecification.') + return self.get_list('RequestSpotInstances', params, + [('item', SpotInstanceRequest)], + verb='POST') + + + def cancel_spot_instance_requests(self, request_ids): + """ + Cancel the specified Spot Instance Requests. + + :type request_ids: list + :param request_ids: A list of strings of the Request IDs to terminate + + :rtype: list + :return: A list of the instances terminated + """ + params = {} + if request_ids: + self.build_list_params(params, request_ids, 'SpotInstanceRequestId') + return self.get_list('CancelSpotInstanceRequests', params, [('item', Instance)]) + + def get_spot_datafeed_subscription(self): + """ + Return the current spot instance data feed subscription + associated with this account, if any. + + :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription + :return: The datafeed subscription object or None + """ + return self.get_object('DescribeSpotDatafeedSubscription', + None, SpotDatafeedSubscription) + + def create_spot_datafeed_subscription(self, bucket, prefix): + """ + Create a spot instance datafeed subscription for this account. + + :type bucket: str or unicode + :param bucket: The name of the bucket where spot instance data + will be written. + + :type prefix: str or unicode + :param prefix: An optional prefix that will be pre-pended to all + data files written to the bucket. + + :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription + :return: The datafeed subscription object or None + """ + params = {'Bucket' : bucket} + if prefix: + params['Prefix'] = prefix + return self.get_object('CreateSpotDatafeedSubscription', + params, SpotDatafeedSubscription) + + def delete_spot_datafeed_subscription(self): + """ + Delete the current spot instance data feed subscription + associated with this account + + :rtype: bool + :return: True if successful + """ + return self.get_status('DeleteSpotDatafeedSubscription', None) + + # Zone methods + + def get_all_zones(self, zones=None): + """ + Get all Availability Zones associated with the current region. + + :type zones: list + :param zones: Optional list of zones. If this list is present, + only the Zones associated with these zone names + will be returned. + + :rtype: list of L{boto.ec2.zone.Zone} + :return: The requested Zone objects + """ + params = {} + if zones: + self.build_list_params(params, zones, 'ZoneName') + return self.get_list('DescribeAvailabilityZones', params, [('item', Zone)]) + + # Address methods + + def get_all_addresses(self, addresses=None): + """ + Get all EIP's associated with the current credentials. + + :type addresses: list + :param addresses: Optional list of addresses. If this list is present, + only the Addresses associated with these addresses + will be returned. + + :rtype: list of L{boto.ec2.address.Address} + :return: The requested Address objects + """ + params = {} + if addresses: + self.build_list_params(params, addresses, 'PublicIp') + return self.get_list('DescribeAddresses', params, [('item', Address)]) + + def allocate_address(self): + """ + Allocate a new Elastic IP address and associate it with your account. + + :rtype: L{boto.ec2.address.Address} + :return: The newly allocated Address + """ + return self.get_object('AllocateAddress', None, Address) + + def associate_address(self, instance_id, public_ip): + """ + Associate an Elastic IP address with a currently running instance. + + :type instance_id: string + :param instance_id: The ID of the instance + + :type public_ip: string + :param public_ip: The public IP address + + :rtype: bool + :return: True if successful + """ + params = {'InstanceId' : instance_id, 'PublicIp' : public_ip} + return self.get_status('AssociateAddress', params) + + def disassociate_address(self, public_ip): + """ + Disassociate an Elastic IP address from a currently running instance. + + :type public_ip: string + :param public_ip: The public IP address + + :rtype: bool + :return: True if successful + """ + params = {'PublicIp' : public_ip} + return self.get_status('DisassociateAddress', params) + + def release_address(self, public_ip): + """ + Free up an Elastic IP address + + :type public_ip: string + :param public_ip: The public IP address + + :rtype: bool + :return: True if successful + """ + params = {'PublicIp' : public_ip} + return self.get_status('ReleaseAddress', params) + + # Volume methods + + def get_all_volumes(self, volume_ids=None): + """ + Get all Volumes associated with the current credentials. + + :type volume_ids: list + :param volume_ids: Optional list of volume ids. If this list is present, + only the volumes associated with these volume ids + will be returned. + + :rtype: list of L{boto.ec2.volume.Volume} + :return: The requested Volume objects + """ + params = {} + if volume_ids: + self.build_list_params(params, volume_ids, 'VolumeId') + return self.get_list('DescribeVolumes', params, [('item', Volume)]) + + def create_volume(self, size, zone, snapshot=None): + """ + Create a new EBS Volume. + + :type size: int + :param size: The size of the new volume, in GiB + + :type zone: string or L{boto.ec2.zone.Zone} + :param zone: The availability zone in which the Volume will be created. + + :type snapshot: string or L{boto.ec2.snapshot.Snapshot} + :param snapshot: The snapshot from which the new Volume will be created. + """ + if isinstance(zone, Zone): + zone = zone.name + params = {'AvailabilityZone' : zone} + if size: + params['Size'] = size + if snapshot: + if isinstance(snapshot, Snapshot): + snapshot = snapshot.id + params['SnapshotId'] = snapshot + return self.get_object('CreateVolume', params, Volume) + + def delete_volume(self, volume_id): + """ + Delete an EBS volume. + + :type volume_id: str + :param volume_id: The ID of the volume to be delete. + + :rtype: bool + :return: True if successful + """ + params = {'VolumeId': volume_id} + return self.get_status('DeleteVolume', params) + + def attach_volume(self, volume_id, instance_id, device): + """ + Attach an EBS volume to an EC2 instance. + + :type volume_id: str + :param volume_id: The ID of the EBS volume to be attached. + + :type instance_id: str + :param instance_id: The ID of the EC2 instance to which it will + be attached. + + :type device: str + :param device: The device on the instance through which the + volume will be exposted (e.g. /dev/sdh) + + :rtype: bool + :return: True if successful + """ + params = {'InstanceId' : instance_id, + 'VolumeId' : volume_id, + 'Device' : device} + return self.get_status('AttachVolume', params) + + def detach_volume(self, volume_id, instance_id=None, device=None, force=False): + """ + Detach an EBS volume from an EC2 instance. + + :type volume_id: str + :param volume_id: The ID of the EBS volume to be attached. + + :type instance_id: str + :param instance_id: The ID of the EC2 instance from which it will + be detached. + + :type device: str + :param device: The device on the instance through which the + volume is exposted (e.g. /dev/sdh) + + :type force: bool + :param force: Forces detachment if the previous detachment attempt did + not occur cleanly. This option can lead to data loss or + a corrupted file system. Use this option only as a last + resort to detach a volume from a failed instance. The + instance will not have an opportunity to flush file system + caches nor file system meta data. If you use this option, + you must perform file system check and repair procedures. + + :rtype: bool + :return: True if successful + """ + params = {'VolumeId' : volume_id} + if instance_id: + params['InstanceId'] = instance_id + if device: + params['Device'] = device + if force: + params['Force'] = 'true' + return self.get_status('DetachVolume', params) + + # Snapshot methods + + def get_all_snapshots(self, snapshot_ids=None, owner=None, restorable_by=None): + """ + Get all EBS Snapshots associated with the current credentials. + + :type snapshot_ids: list + :param snapshot_ids: Optional list of snapshot ids. If this list is present, + only the Snapshots associated with these snapshot ids + will be returned. + + :type owner: str + :param owner: If present, only the snapshots owned by the specified user + will be returned. Valid values are: + self | amazon | AWS Account ID + + :type restorable_by: str + :param restorable_by: If present, only the snapshots that are restorable + by the specified account id will be returned. + + :rtype: list of L{boto.ec2.snapshot.Snapshot} + :return: The requested Snapshot objects + """ + params = {} + if snapshot_ids: + self.build_list_params(params, snapshot_ids, 'SnapshotId') + if owner: + params['Owner'] = owner + if restorable_by: + params['RestorableBy'] = restorable_by + return self.get_list('DescribeSnapshots', params, [('item', Snapshot)]) + + def create_snapshot(self, volume_id, description=None): + """ + Create a snapshot of an existing EBS Volume. + + :type volume_id: str + :param volume_id: The ID of the volume to be snapshot'ed + + :type description: str + :param description: A description of the snapshot. Limited to 255 characters. + + :rtype: bool + :return: True if successful + """ + params = {'VolumeId' : volume_id} + if description: + params['Description'] = description[0:255] + return self.get_object('CreateSnapshot', params, Snapshot) + + def delete_snapshot(self, snapshot_id): + params = {'SnapshotId': snapshot_id} + return self.get_status('DeleteSnapshot', params) + + def get_snapshot_attribute(self, snapshot_id, attribute='createVolumePermission'): + """ + Get information about an attribute of a snapshot. Only one attribute can be + specified per call. + + :type snapshot_id: str + :param snapshot_id: The ID of the snapshot. + + :type attribute: str + :param attribute: The requested attribute. Valid values are: + createVolumePermission + + :rtype: list of L{boto.ec2.snapshotattribute.SnapshotAttribute} + :return: The requested Snapshot attribute + """ + params = {'Attribute' : attribute} + if snapshot_id: + params['SnapshotId'] = snapshot_id + return self.get_object('DescribeSnapshotAttribute', params, SnapshotAttribute) + + def modify_snapshot_attribute(self, snapshot_id, attribute='createVolumePermission', + operation='add', user_ids=None, groups=None): + """ + Changes an attribute of an image. + + :type snapshot_id: string + :param snapshot_id: The snapshot id you wish to change + + :type attribute: string + :param attribute: The attribute you wish to change. Valid values are: + createVolumePermission + + :type operation: string + :param operation: Either add or remove (this is required for changing + snapshot ermissions) + + :type user_ids: list + :param user_ids: The Amazon IDs of users to add/remove attributes + + :type groups: list + :param groups: The groups to add/remove attributes. The only valid + value at this time is 'all'. + + """ + params = {'SnapshotId' : snapshot_id, + 'Attribute' : attribute, + 'OperationType' : operation} + if user_ids: + self.build_list_params(params, user_ids, 'UserId') + if groups: + self.build_list_params(params, groups, 'UserGroup') + return self.get_status('ModifySnapshotAttribute', params) + + def reset_snapshot_attribute(self, snapshot_id, attribute='createVolumePermission'): + """ + Resets an attribute of a snapshot to its default value. + + :type snapshot_id: string + :param snapshot_id: ID of the snapshot + + :type attribute: string + :param attribute: The attribute to reset + + :rtype: bool + :return: Whether the operation succeeded or not + """ + params = {'SnapshotId' : snapshot_id, + 'Attribute' : attribute} + return self.get_status('ResetSnapshotAttribute', params) + + # Keypair methods + + def get_all_key_pairs(self, keynames=None): + """ + Get all key pairs associated with your account. + + :type keynames: list + :param keynames: A list of the names of keypairs to retrieve. + If not provided, all key pairs will be returned. + + :rtype: list + :return: A list of :class:`boto.ec2.keypair.KeyPair` + """ + params = {} + if keynames: + self.build_list_params(params, keynames, 'KeyName') + return self.get_list('DescribeKeyPairs', params, [('item', KeyPair)]) + + def get_key_pair(self, keyname): + """ + Convenience method to retrieve a specific keypair (KeyPair). + + :type image_id: string + :param image_id: the ID of the Image to retrieve + + :rtype: :class:`boto.ec2.keypair.KeyPair` + :return: The KeyPair specified or None if it is not found + """ + try: + return self.get_all_key_pairs(keynames=[keyname])[0] + except IndexError: # None of those key pairs available + return None + + def create_key_pair(self, key_name): + """ + Create a new key pair for your account. + This will create the key pair within the region you + are currently connected to. + + :type key_name: string + :param key_name: The name of the new keypair + + :rtype: :class:`boto.ec2.keypair.KeyPair` + :return: The newly created :class:`boto.ec2.keypair.KeyPair`. + The material attribute of the new KeyPair object + will contain the the unencrypted PEM encoded RSA private key. + """ + params = {'KeyName':key_name} + return self.get_object('CreateKeyPair', params, KeyPair) + + def delete_key_pair(self, key_name): + """ + Delete a key pair from your account. + + :type key_name: string + :param key_name: The name of the keypair to delete + """ + params = {'KeyName':key_name} + return self.get_status('DeleteKeyPair', params) + + # SecurityGroup methods + + def get_all_security_groups(self, groupnames=None): + """ + Get all security groups associated with your account in a region. + + :type groupnames: list + :param groupnames: A list of the names of security groups to retrieve. + If not provided, all security groups will be returned. + + :rtype: list + :return: A list of :class:`boto.ec2.securitygroup.SecurityGroup` + """ + params = {} + if groupnames: + self.build_list_params(params, groupnames, 'GroupName') + return self.get_list('DescribeSecurityGroups', params, [('item', SecurityGroup)]) + + def create_security_group(self, name, description): + """ + Create a new security group for your account. + This will create the security group within the region you + are currently connected to. + + :type name: string + :param name: The name of the new security group + + :type description: string + :param description: The description of the new security group + + :rtype: :class:`boto.ec2.securitygroup.SecurityGroup` + :return: The newly created :class:`boto.ec2.keypair.KeyPair`. + """ + params = {'GroupName':name, 'GroupDescription':description} + group = self.get_object('CreateSecurityGroup', params, SecurityGroup) + group.name = name + group.description = description + return group + + def delete_security_group(self, name): + """ + Delete a security group from your account. + + :type key_name: string + :param key_name: The name of the keypair to delete + """ + params = {'GroupName':name} + return self.get_status('DeleteSecurityGroup', params) + + def authorize_security_group(self, group_name, src_security_group_name=None, + src_security_group_owner_id=None, + ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None): + """ + Add a new rule to an existing security group. + You need to pass in either src_security_group_name and + src_security_group_owner_id OR ip_protocol, from_port, to_port, + and cidr_ip. In other words, either you are authorizing another + group or you are authorizing some ip-based rule. + + :type group_name: string + :param group_name: The name of the security group you are adding + the rule to. + + :type src_security_group_name: string + :param src_security_group_name: The name of the security group you are + granting access to. + + :type src_security_group_owner_id: string + :param src_security_group_owner_id: The ID of the owner of the security group you are + granting access to. + + :type ip_protocol: string + :param ip_protocol: Either tcp | udp | icmp + + :type from_port: int + :param from_port: The beginning port number you are enabling + + :type to_port: int + :param to_port: The ending port number you are enabling + + :type to_port: string + :param to_port: The CIDR block you are providing access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + params = {'GroupName':group_name} + if src_security_group_name: + params['SourceSecurityGroupName'] = src_security_group_name + if src_security_group_owner_id: + params['SourceSecurityGroupOwnerId'] = src_security_group_owner_id + if ip_protocol: + params['IpProtocol'] = ip_protocol + if from_port: + params['FromPort'] = from_port + if to_port: + params['ToPort'] = to_port + if cidr_ip: + params['CidrIp'] = urllib.quote(cidr_ip) + return self.get_status('AuthorizeSecurityGroupIngress', params) + + def revoke_security_group(self, group_name, src_security_group_name=None, + src_security_group_owner_id=None, + ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None): + """ + Remove an existing rule from an existing security group. + You need to pass in either src_security_group_name and + src_security_group_owner_id OR ip_protocol, from_port, to_port, + and cidr_ip. In other words, either you are revoking another + group or you are revoking some ip-based rule. + + :type group_name: string + :param group_name: The name of the security group you are removing + the rule from. + + :type src_security_group_name: string + :param src_security_group_name: The name of the security group you are + revoking access to. + + :type src_security_group_owner_id: string + :param src_security_group_owner_id: The ID of the owner of the security group you are + revoking access to. + + :type ip_protocol: string + :param ip_protocol: Either tcp | udp | icmp + + :type from_port: int + :param from_port: The beginning port number you are disabling + + :type to_port: int + :param to_port: The ending port number you are disabling + + :type to_port: string + :param to_port: The CIDR block you are revoking access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + params = {'GroupName':group_name} + if src_security_group_name: + params['SourceSecurityGroupName'] = src_security_group_name + if src_security_group_owner_id: + params['SourceSecurityGroupOwnerId'] = src_security_group_owner_id + if ip_protocol: + params['IpProtocol'] = ip_protocol + if from_port: + params['FromPort'] = from_port + if to_port: + params['ToPort'] = to_port + if cidr_ip: + params['CidrIp'] = cidr_ip + return self.get_status('RevokeSecurityGroupIngress', params) + + # + # Regions + # + + def get_all_regions(self): + """ + Get all available regions for the EC2 service. + + :rtype: list + :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` + """ + return self.get_list('DescribeRegions', None, [('item', RegionInfo)]) + + # + # Reservation methods + # + + def get_all_reserved_instances_offerings(self, reserved_instances_id=None, + instance_type=None, + availability_zone=None, + product_description=None): + """ + Describes Reserved Instance offerings that are available for purchase. + + :type reserved_instances_id: str + :param reserved_instances_id: Displays Reserved Instances with the specified offering IDs. + + :type instance_type: str + :param instance_type: Displays Reserved Instances of the specified instance type. + + :type availability_zone: str + :param availability_zone: Displays Reserved Instances within the specified Availability Zone. + + :type product_description: str + :param product_description: Displays Reserved Instances with the specified product description. + + :rtype: list + :return: A list of :class:`boto.ec2.reservedinstance.ReservedInstancesOffering` + """ + params = {} + if reserved_instances_id: + params['ReservedInstancesId'] = reserved_instances_id + if instance_type: + params['InstanceType'] = instance_type + if availability_zone: + params['AvailabilityZone'] = availability_zone + if product_description: + params['ProductDescription'] = product_description + + return self.get_list('DescribeReservedInstancesOfferings', + params, [('item', ReservedInstancesOffering)]) + + def get_all_reserved_instances(self, reserved_instances_id=None): + """ + Describes Reserved Instance offerings that are available for purchase. + + :type reserved_instance_ids: list + :param reserved_instance_ids: A list of the reserved instance ids that will be returned. + If not provided, all reserved instances will be returned. + + :rtype: list + :return: A list of :class:`boto.ec2.reservedinstance.ReservedInstance` + """ + params = {} + if reserved_instances_id: + self.build_list_params(params, reserved_instances_id, 'ReservedInstancesId') + return self.get_list('DescribeReservedInstances', + params, [('item', ReservedInstance)]) + + def purchase_reserved_instance_offering(self, reserved_instances_offering_id, + instance_count=1): + """ + Purchase a Reserved Instance for use with your account. + ** CAUTION ** + This request can result in large amounts of money being charged to your + AWS account. Use with caution! + + :type reserved_instances_offering_id: string + :param reserved_instances_offering_id: The offering ID of the Reserved + Instance to purchase + + :type instance_count: int + :param instance_count: The number of Reserved Instances to purchase. + Default value is 1. + + :rtype: :class:`boto.ec2.reservedinstance.ReservedInstance` + :return: The newly created Reserved Instance + """ + params = {'ReservedInstancesOfferingId' : reserved_instances_offering_id, + 'InstanceCount' : instance_count} + return self.get_object('PurchaseReservedInstancesOffering', params, ReservedInstance) + + # + # Monitoring + # + + def monitor_instance(self, instance_id): + """ + Enable CloudWatch monitoring for the supplied instance. + + :type instance_id: string + :param instance_id: The instance id + + :rtype: list + :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` + """ + params = {'InstanceId' : instance_id} + return self.get_list('MonitorInstances', params, [('item', InstanceInfo)]) + + def unmonitor_instance(self, instance_id): + """ + Disable CloudWatch monitoring for the supplied instance. + + :type instance_id: string + :param instance_id: The instance id + + :rtype: list + :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` + """ + params = {'InstanceId' : instance_id} + return self.get_list('UnmonitorInstances', params, [('item', InstanceInfo)]) + diff --git a/storefront/boto/ec2/ec2object.py b/storefront/boto/ec2/ec2object.py new file mode 100644 index 0000000..9ffab5d --- /dev/null +++ b/storefront/boto/ec2/ec2object.py @@ -0,0 +1,41 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Object +""" + +class EC2Object(object): + + def __init__(self, connection=None): + self.connection = connection + if self.connection and hasattr(self.connection, 'region'): + self.region = connection.region + else: + self.region = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + setattr(self, name, value) + + diff --git a/storefront/boto/ec2/elb/__init__.py b/storefront/boto/ec2/elb/__init__.py new file mode 100644 index 0000000..55e846f --- /dev/null +++ b/storefront/boto/ec2/elb/__init__.py @@ -0,0 +1,238 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +load balancing service from AWS. +""" +from boto.connection import AWSQueryConnection +from boto.ec2.instanceinfo import InstanceInfo +from boto.ec2.elb.loadbalancer import LoadBalancer +from boto.ec2.elb.instancestate import InstanceState +from boto.ec2.elb.healthcheck import HealthCheck +import boto + +class ELBConnection(AWSQueryConnection): + + APIVersion = boto.config.get('Boto', 'elb_version', '2009-05-15') + Endpoint = boto.config.get('Boto', 'elb_endpoint', 'elasticloadbalancing.amazonaws.com') + SignatureVersion = '1' + #ResponseError = EC2ResponseError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=Endpoint, debug=0, + https_connection_factory=None, path='/'): + """ + Init method to create a new connection to EC2 Load Balancing Service. + + B{Note:} The host argument is overridden by the host specified in the boto configuration file. + """ + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory, path) + + def build_list_params(self, params, items, label): + if isinstance(items, str): + items = [items] + for i in range(1, len(items)+1): + params[label % i] = items[i-1] + + def get_all_load_balancers(self, load_balancer_name=None): + """ + Retrieve all load balancers associated with your account. + + :type load_balancer_names: str + :param load_balancer_names: An optional filter string to get only one ELB + + :rtype: list + :return: A list of :class:`boto.ec2.elb.loadbalancer.LoadBalancer` + """ + params = {} + if load_balancer_name: + #self.build_list_params(params, load_balancer_names, 'LoadBalancerName.%d') + params['LoadBalancerName'] = load_balancer_name + return self.get_list('DescribeLoadBalancers', params, [('member', LoadBalancer)]) + + + def create_load_balancer(self, name, zones, listeners): + """ + Create a new load balancer for your account. + + :type name: string + :param name: The mnemonic name associated with the new load balancer + + :type zones: List of strings + :param zones: The names of the availability zone(s) to add. + + :type listeners: List of tuples + :param listeners: Each tuple contains three values. + (LoadBalancerPortNumber, InstancePortNumber, Protocol) + where LoadBalancerPortNumber and InstancePortNumber are + integer values between 1 and 65535 and Protocol is a + string containing either 'TCP' or 'HTTP'. + + :rtype: :class:`boto.ec2.elb.loadbalancer.LoadBalancer` + :return: The newly created :class:`boto.ec2.elb.loadbalancer.LoadBalancer` + """ + params = {'LoadBalancerName' : name} + for i in range(0, len(listeners)): + params['Listeners.member.%d.LoadBalancerPort' % (i+1)] = listeners[i][0] + params['Listeners.member.%d.InstancePort' % (i+1)] = listeners[i][1] + params['Listeners.member.%d.Protocol' % (i+1)] = listeners[i][2] + self.build_list_params(params, zones, 'AvailabilityZones.member.%d') + load_balancer = self.get_object('CreateLoadBalancer', params, LoadBalancer) + load_balancer.name = name + load_balancer.listeners = listeners + load_balancer.availability_zones = zones + return load_balancer + + def delete_load_balancer(self, name): + """ + Delete a Load Balancer from your account. + + :type name: string + :param name: The name of the Load Balancer to delete + """ + params = {'LoadBalancerName': name} + return self.get_status('DeleteLoadBalancer', params) + + def enable_availability_zones(self, load_balancer_name, zones_to_add): + """ + Add availability zones to an existing Load Balancer + All zones must be in the same region as the Load Balancer + Adding zones that are already registered with the Load Balancer + has no effect. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type zones: List of strings + :param zones: The name of the zone(s) to add. + + :rtype: List of strings + :return: An updated list of zones for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, zones_to_add, 'AvailabilityZones.member.%d') + return self.get_list('EnableAvailabilityZonesForLoadBalancer', params, None) + + def disable_availability_zones(self, load_balancer_name, zones_to_remove): + """ + Remove availability zones from an existing Load Balancer. + All zones must be in the same region as the Load Balancer. + Removing zones that are not registered with the Load Balancer + has no effect. + You cannot remove all zones from an Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type zones: List of strings + :param zones: The name of the zone(s) to remove. + + :rtype: List of strings + :return: An updated list of zones for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, zones_to_remove, 'AvailabilityZones.member.%d') + return self.get_list('DisableAvailabilityZonesForLoadBalancer', params, None) + + def register_instances(self, load_balancer_name, instances): + """ + Add new Instances to an existing Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type instances: List of strings + :param instances: The instance ID's of the EC2 instances to add. + + :rtype: List of strings + :return: An updated list of instances for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, instances, 'Instances.member.%d.InstanceId') + return self.get_list('RegisterInstancesWithLoadBalancer', params, [('member', InstanceInfo)]) + + def deregister_instances(self, load_balancer_name, instances): + """ + Remove Instances from an existing Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type instances: List of strings + :param instances: The instance ID's of the EC2 instances to remove. + + :rtype: List of strings + :return: An updated list of instances for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, instances, 'Instances.member.%d.InstanceId') + return self.get_list('DeregisterInstancesFromLoadBalancer', params, [('member', InstanceInfo)]) + + def describe_instance_health(self, load_balancer_name, instances=None): + """ + Get current state of all Instances registered to an Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type instances: List of strings + :param instances: The instance ID's of the EC2 instances + to return status for. If not provided, + the state of all instances will be returned. + + :rtype: List of :class:`boto.ec2.elb.instancestate.InstanceState` + :return: list of state info for instances in this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + if instances: + self.build_list_params(params, instances, 'instances.member.%d') + return self.get_list('DescribeInstanceHealth', params, [('member', InstanceState)]) + + def configure_health_check(self, name, health_check): + """ + Define a health check for the EndPoints. + + :type name: string + :param name: The mnemonic name associated with the new access point + + :type health_check: :class:`boto.ec2.elb.healthcheck.HealthCheck` + :param health_check: A HealthCheck object populated with the desired + values. + + :rtype: :class:`boto.ec2.elb.healthcheck.HealthCheck` + :return: The updated :class:`boto.ec2.elb.healthcheck.HealthCheck` + """ + params = {'LoadBalancerName' : name, + 'HealthCheck.Timeout' : health_check.timeout, + 'HealthCheck.Target' : health_check.target, + 'HealthCheck.Interval' : health_check.interval, + 'HealthCheck.UnhealthyThreshold' : health_check.unhealthy_threshold, + 'HealthCheck.HealthyThreshold' : health_check.healthy_threshold} + return self.get_object('ConfigureHealthCheck', params, HealthCheck) diff --git a/storefront/boto/ec2/elb/healthcheck.py b/storefront/boto/ec2/elb/healthcheck.py new file mode 100644 index 0000000..5a3edbc --- /dev/null +++ b/storefront/boto/ec2/elb/healthcheck.py @@ -0,0 +1,68 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class HealthCheck(object): + """ + Represents an EC2 Access Point Health Check + """ + + def __init__(self, access_point=None, interval=30, target=None, + healthy_threshold=3, timeout=5, unhealthy_threshold=5): + self.access_point = access_point + self.interval = interval + self.target = target + self.healthy_threshold = healthy_threshold + self.timeout = timeout + self.unhealthy_threshold = unhealthy_threshold + + def __repr__(self): + return 'HealthCheck:%s' % self.target + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Interval': + self.interval = int(value) + elif name == 'Target': + self.target = value + elif name == 'HealthyThreshold': + self.healthy_threshold = int(value) + elif name == 'Timeout': + self.timeout = int(value) + elif name == 'UnhealthyThreshold': + self.unhealthy_threshold = int(value) + else: + setattr(self, name, value) + + def update(self): + if not self.access_point: + return + + new_hc = self.connection.configure_health_check(self.access_point, + self) + self.interval = new_hc.interval + self.target = new_hc.target + self.healthy_threshold = new_hc.healthy_threshold + self.unhealthy_threshold = new_hc.unhealthy_threshold + self.timeout = new_hc.timeout + + diff --git a/storefront/boto/ec2/elb/instancestate.py b/storefront/boto/ec2/elb/instancestate.py new file mode 100644 index 0000000..4a9b0d4 --- /dev/null +++ b/storefront/boto/ec2/elb/instancestate.py @@ -0,0 +1,54 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class InstanceState(object): + """ + Represents the state of an EC2 Load Balancer Instance + """ + + def __init__(self, load_balancer=None, description=None, + state=None, instance_id=None, reason_code=None): + self.load_balancer = load_balancer + self.description = description + self.state = state + self.instance_id = instance_id + self.reason_code = reason_code + + def __repr__(self): + return 'InstanceState:(%s,%s)' % (self.instance_id, self.state) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Description': + self.description = value + elif name == 'State': + self.state = value + elif name == 'InstanceId': + self.instance_id = value + elif name == 'ReasonCode': + self.reason_code = value + else: + setattr(self, name, value) + + + diff --git a/storefront/boto/ec2/elb/listelement.py b/storefront/boto/ec2/elb/listelement.py new file mode 100644 index 0000000..5be4599 --- /dev/null +++ b/storefront/boto/ec2/elb/listelement.py @@ -0,0 +1,31 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class ListElement(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'member': + self.append(value) + + diff --git a/storefront/boto/ec2/elb/listener.py b/storefront/boto/ec2/elb/listener.py new file mode 100644 index 0000000..ab482c2 --- /dev/null +++ b/storefront/boto/ec2/elb/listener.py @@ -0,0 +1,64 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Listener(object): + """ + Represents an EC2 Load Balancer Listener tuple + """ + + def __init__(self, load_balancer=None, load_balancer_port=0, + instance_port=0, protocol=''): + self.load_balancer = load_balancer + self.load_balancer_port = load_balancer_port + self.instance_port = instance_port + self.protocol = protocol + + def __repr__(self): + return "(%d, %d, '%s')" % (self.load_balancer_port, self.instance_port, self.protocol) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'LoadBalancerPort': + self.load_balancer_port = int(value) + elif name == 'InstancePort': + self.instance_port = int(value) + elif name == 'Protocol': + self.protocol = value + else: + setattr(self, name, value) + + def get_tuple(self): + return self.load_balancer_port, self.instance_port, self.protocol + + def __getitem__(self, key): + if key == 0: + return self.load_balancer_port + if key == 1: + return self.instance_port + if key == 2: + return self.protocol + raise KeyError + + + + diff --git a/storefront/boto/ec2/elb/loadbalancer.py b/storefront/boto/ec2/elb/loadbalancer.py new file mode 100644 index 0000000..2902107 --- /dev/null +++ b/storefront/boto/ec2/elb/loadbalancer.py @@ -0,0 +1,144 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.ec2.elb.healthcheck import HealthCheck +from boto.ec2.elb.instancestate import InstanceState +from boto.ec2.elb.listener import Listener +from boto.ec2.elb.listelement import ListElement +from boto.ec2.zone import Zone +from boto.ec2.instanceinfo import InstanceInfo +from boto.resultset import ResultSet + +class LoadBalancer(object): + """ + Represents an EC2 Load Balancer + """ + + def __init__(self, connection=None, name=None, endpoints=None): + self.connection = connection + self.name = name + self.listeners = None + self.health_check = None + self.dns_name = None + self.created_time = None + self.instances = None + self.availability_zones = ListElement() + + def __repr__(self): + return 'LoadBalancer:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'HealthCheck': + self.health_check = HealthCheck(self) + return self.health_check + elif name == 'Listeners': + self.listeners = ResultSet([('member', Listener)]) + return self.listeners + elif name == 'AvailabilityZones': + return self.availability_zones + elif name == 'Instances': + self.instances = ResultSet([('member', InstanceInfo)]) + return self.instances + else: + return None + + def endElement(self, name, value, connection): + if name == 'LoadBalancerName': + self.name = value + elif name == 'DNSName': + self.dns_name = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'InstanceId': + self.instances.append(value) + else: + setattr(self, name, value) + + def enable_zones(self, zones): + """ + Enable availability zones to this Access Point. + All zones must be in the same region as the Access Point. + + :type zones: string or List of strings + :param zones: The name of the zone(s) to add. + + """ + if isinstance(zones, str) or isinstance(zones, unicode): + zones = [zones] + new_zones = self.connection.enable_availability_zones(self.name, zones) + self.availability_zones = new_zones + + def disable_zones(self, zones): + """ + Disable availability zones from this Access Point. + + :type zones: string or List of strings + :param zones: The name of the zone(s) to add. + + """ + if isinstance(zones, str) or isinstance(zones, unicode): + zones = [zones] + new_zones = self.connection.disable_availability_zones(self.name, zones) + self.availability_zones = new_zones + + def register_instances(self, instances): + """ + Add instances to this Load Balancer + All instances must be in the same region as the Load Balancer. + Adding endpoints that are already registered with the Load Balancer + has no effect. + + :type zones: string or List of instance id's + :param zones: The name of the endpoint(s) to add. + + """ + if isinstance(instances, str) or isinstance(instances, unicode): + instances = [instances] + new_instances = self.connection.register_instances(self.name, instances) + self.instances = new_instances + + def deregister_instances(self, instances): + """ + Remove instances from this Load Balancer. + Removing instances that are not registered with the Load Balancer + has no effect. + + :type zones: string or List of instance id's + :param zones: The name of the endpoint(s) to add. + + """ + if isinstance(instances, str) or isinstance(instances, unicode): + instances = [instances] + new_instances = self.connection.deregister_instances(self.name, instances) + self.instances = new_instances + + def delete(self): + """ + Delete this load balancer + """ + return self.connection.delete_load_balancer(self.name) + + def configure_health_check(self, health_check): + self.connection.configure_health_check(self.name, health_check) + + def get_instance_health(self, instances=None): + self.connection.describe_instance_health(self.name, instances) + diff --git a/storefront/boto/ec2/image.py b/storefront/boto/ec2/image.py new file mode 100644 index 0000000..8ef2513 --- /dev/null +++ b/storefront/boto/ec2/image.py @@ -0,0 +1,243 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.ec2.ec2object import EC2Object +from boto.ec2.blockdevicemapping import BlockDeviceMapping + +class ProductCodes(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'productCode': + self.append(value) + +class Image(EC2Object): + """ + Represents an EC2 Image + """ + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.location = None + self.state = None + self.ownerId = None + self.owner_alias = None + self.is_public = False + self.architecture = None + self.platform = None + self.type = None + self.kernel_id = None + self.ramdisk_id = None + self.name = None + self.description = None + self.product_codes = ProductCodes() + self.block_device_mapping = None + self.root_device_type = None + self.root_device_name = None + + def __repr__(self): + return 'Image:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'blockDeviceMapping': + self.block_device_mapping = BlockDeviceMapping() + return self.block_device_mapping + elif name == 'productCodes': + return self.product_codes + else: + return None + + def endElement(self, name, value, connection): + if name == 'imageId': + self.id = value + elif name == 'imageLocation': + self.location = value + elif name == 'imageState': + self.state = value + elif name == 'imageOwnerId': + self.ownerId = value + elif name == 'isPublic': + if value == 'false': + self.is_public = False + elif value == 'true': + self.is_public = True + else: + raise Exception( + 'Unexpected value of isPublic %s for image %s'%( + value, + self.id + ) + ) + elif name == 'architecture': + self.architecture = value + elif name == 'imageType': + self.type = value + elif name == 'kernelId': + self.kernel_id = value + elif name == 'ramdiskId': + self.ramdisk_id = value + elif name == 'imageOwnerAlias': + self.owner_alias = value + elif name == 'platform': + self.platform = value + elif name == 'name': + self.name = value + elif name == 'description': + self.description = value + elif name == 'rootDeviceType': + self.root_device_type = value + elif name == 'rootDeviceName': + self.root_device_name = value + else: + setattr(self, name, value) + + def run(self, min_count=1, max_count=1, key_name=None, + security_groups=None, user_data=None, + addressing_type=None, instance_type='m1.small', placement=None, + kernel_id=None, ramdisk_id=None, + monitoring_enabled=False, subnet_id=None): + """ + Runs this instance. + + :type min_count: int + :param min_count: The minimum number of instances to start + + :type max_count: int + :param max_count: The maximum number of instances to start + + :type key_name: string + :param key_name: The keypair to run this instance with. + + :type security_groups: + :param security_groups: + + :type user_data: + :param user_data: + + :type addressing_type: + :param daddressing_type: + + :type instance_type: string + :param instance_type: The type of instance to run (m1.small, m1.large, m1.xlarge) + + :type placement: + :param placement: + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the instances + + :type ramdisk_id: string + :param ramdisk_id: The ID of the RAM disk with which to launch the instances + + :type monitoring_enabled: bool + :param monitoring_enabled: Enable CloudWatch monitoring on the instance. + + :type subnet_id: string + :param subnet_id: The subnet ID within which to launch the instances for VPC. + + :rtype: Reservation + :return: The :class:`boto.ec2.instance.Reservation` associated with the request for machines + """ + return self.connection.run_instances(self.id, min_count, max_count, + key_name, security_groups, + user_data, addressing_type, + instance_type, placement, + kernel_id, ramdisk_id, + monitoring_enabled, subnet_id) + + def deregister(self): + return self.connection.deregister_image(self.id) + + def get_launch_permissions(self): + img_attrs = self.connection.get_image_attribute(self.id, + 'launchPermission') + return img_attrs.attrs + + def set_launch_permissions(self, user_ids=None, group_names=None): + return self.connection.modify_image_attribute(self.id, + 'launchPermission', + 'add', + user_ids, + group_names) + + def remove_launch_permissions(self, user_ids=None, group_names=None): + return self.connection.modify_image_attribute(self.id, + 'launchPermission', + 'remove', + user_ids, + group_names) + + def reset_launch_attributes(self): + return self.connection.reset_image_attribute(self.id, + 'launchPermission') + + def get_kernel(self): + img_attrs =self.connection.get_image_attribute(self.id, 'kernel') + return img_attrs.kernel + + def get_ramdisk(self): + img_attrs = self.connection.get_image_attribute(self.id, 'ramdisk') + return img_attrs.ramdisk + +class ImageAttribute: + + def __init__(self, parent=None): + self.name = None + self.kernel = None + self.ramdisk = None + self.attrs = {} + + def startElement(self, name, attrs, connection): + if name == 'blockDeviceMapping': + self.attrs['block_device_mapping'] = BlockDeviceMapping() + return self.attrs['block_device_mapping'] + else: + return None + + def endElement(self, name, value, connection): + if name == 'launchPermission': + self.name = 'launch_permission' + elif name == 'group': + if self.attrs.has_key('groups'): + self.attrs['groups'].append(value) + else: + self.attrs['groups'] = [value] + elif name == 'userId': + if self.attrs.has_key('user_ids'): + self.attrs['user_ids'].append(value) + else: + self.attrs['user_ids'] = [value] + elif name == 'productCode': + if self.attrs.has_key('product_codes'): + self.attrs['product_codes'].append(value) + else: + self.attrs['product_codes'] = [value] + elif name == 'imageId': + self.image_id = value + elif name == 'kernel': + self.kernel = value + elif name == 'ramdisk': + self.ramdisk = value + else: + setattr(self, name, value) diff --git a/storefront/boto/ec2/instance.py b/storefront/boto/ec2/instance.py new file mode 100644 index 0000000..5932c4e --- /dev/null +++ b/storefront/boto/ec2/instance.py @@ -0,0 +1,280 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Instance +""" +import boto +from boto.ec2.ec2object import EC2Object +from boto.resultset import ResultSet +from boto.ec2.address import Address +from boto.ec2.blockdevicemapping import BlockDeviceMapping +from boto.ec2.image import ProductCodes +import base64 + +class Reservation(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.owner_id = None + self.groups = [] + self.instances = [] + + def __repr__(self): + return 'Reservation:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'instancesSet': + self.instances = ResultSet([('item', Instance)]) + return self.instances + elif name == 'groupSet': + self.groups = ResultSet([('item', Group)]) + return self.groups + else: + return None + + def endElement(self, name, value, connection): + if name == 'reservationId': + self.id = value + elif name == 'ownerId': + self.owner_id = value + else: + setattr(self, name, value) + + def stop_all(self): + for instance in self.instances: + instance.stop() + +class Instance(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.dns_name = None + self.public_dns_name = None + self.private_dns_name = None + self.state = None + self.state_code = None + self.key_name = None + self.shutdown_state = None + self.previous_state = None + self.instance_type = None + self.instance_class = None + self.launch_time = None + self.image_id = None + self.placement = None + self.kernel = None + self.ramdisk = None + self.product_codes = ProductCodes() + self.ami_launch_index = None + self.monitored = False + self.instance_class = None + self.spot_instance_request_id = None + self.subnet_id = None + self.vpc_id = None + self.private_ip_address = None + self.ip_address = None + self.requester_id = None + self._in_monitoring_element = False + self.persistent = False + self.root_device_name = None + self.block_device_mapping = None + + def __repr__(self): + return 'Instance:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'monitoring': + self._in_monitoring_element = True + elif name == 'blockDeviceMapping': + self.block_device_mapping = BlockDeviceMapping() + return self.block_device_mapping + elif name == 'productCodes': + return self.product_codes + return None + + def endElement(self, name, value, connection): + if name == 'instanceId': + self.id = value + elif name == 'imageId': + self.image_id = value + elif name == 'dnsName' or name == 'publicDnsName': + self.dns_name = value # backwards compatibility + self.public_dns_name = value + elif name == 'privateDnsName': + self.private_dns_name = value + elif name == 'keyName': + self.key_name = value + elif name == 'amiLaunchIndex': + self.ami_launch_index = value + elif name == 'shutdownState': + self.shutdown_state = value + elif name == 'previousState': + self.previous_state = value + elif name == 'name': + self.state = value + elif name == 'code': + try: + self.state_code = int(value) + except ValueError: + boto.log.warning('Error converting code (%s) to int' % value) + self.state_code = value + elif name == 'instanceType': + self.instance_type = value + elif name == 'instanceClass': + self.instance_class = value + elif name == 'rootDeviceName': + self.root_device_name = value + elif name == 'launchTime': + self.launch_time = value + elif name == 'availabilityZone': + self.placement = value + elif name == 'placement': + pass + elif name == 'kernelId': + self.kernel = value + elif name == 'ramdiskId': + self.ramdisk = value + elif name == 'state': + if self._in_monitoring_element: + if value == 'enabled': + self.monitored = True + self._in_monitoring_element = False + elif name == 'instanceClass': + self.instance_class = value + elif name == 'spotInstanceRequestId': + self.spot_instance_request_id = value + elif name == 'subnetId': + self.subnet_id = value + elif name == 'vpcId': + self.vpc_id = value + elif name == 'privateIpAddress': + self.private_ip_address = value + elif name == 'ipAddress': + self.ip_address = value + elif name == 'requesterId': + self.requester_id = value + elif name == 'persistent': + if value == 'true': + self.persistent = True + else: + self.persistent = False + else: + setattr(self, name, value) + + def _update(self, updated): + self.updated = updated + if hasattr(updated, 'dns_name'): + self.dns_name = updated.dns_name + self.public_dns_name = updated.dns_name + if hasattr(updated, 'private_dns_name'): + self.private_dns_name = updated.private_dns_name + if hasattr(updated, 'ami_launch_index'): + self.ami_launch_index = updated.ami_launch_index + self.shutdown_state = updated.shutdown_state + self.previous_state = updated.previous_state + if hasattr(updated, 'state'): + self.state = updated.state + else: + self.state = None + if hasattr(updated, 'state_code'): + self.state_code = updated.state_code + else: + self.state_code = None + + def update(self): + rs = self.connection.get_all_instances([self.id]) + if len(rs) > 0: + self._update(rs[0].instances[0]) + return self.state + + def stop(self): + rs = self.connection.terminate_instances([self.id]) + self._update(rs[0]) + + def reboot(self): + return self.connection.reboot_instances([self.id]) + + def get_console_output(self): + return self.connection.get_console_output(self.id) + + def confirm_product(self, product_code): + return self.connection.confirm_product_instance(self.id, product_code) + + def use_ip(self, ip_address): + if isinstance(ip_address, Address): + ip_address = ip_address.public_ip + return self.connection.associate_address(self.id, ip_address) + + def monitor(self): + return self.connection.monitor_instance(self.id) + + def unmonitor(self): + return self.connection.unmonitor_instance(self.id) + +class Group: + + def __init__(self, parent=None): + self.id = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'groupId': + self.id = value + else: + setattr(self, name, value) + +class ConsoleOutput: + + def __init__(self, parent=None): + self.parent = parent + self.instance_id = None + self.timestamp = None + self.comment = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'instanceId': + self.instance_id = value + elif name == 'output': + self.output = base64.b64decode(value) + else: + setattr(self, name, value) + +class InstanceAttribute(dict): + + def __init__(self, parent=None): + dict.__init__(self) + self._current_value = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'value': + self._current_value = value + else: + self[name] = self._current_value diff --git a/storefront/boto/ec2/instanceinfo.py b/storefront/boto/ec2/instanceinfo.py new file mode 100644 index 0000000..6efbaed --- /dev/null +++ b/storefront/boto/ec2/instanceinfo.py @@ -0,0 +1,47 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class InstanceInfo(object): + """ + Represents an EC2 Instance status response from CloudWatch + """ + + def __init__(self, connection=None, id=None, state=None): + self.connection = connection + self.id = id + self.state = state + + def __repr__(self): + return 'InstanceInfo:%s' % self.id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'instanceId' or name == 'InstanceId': + self.id = value + elif name == 'state': + self.state = value + else: + setattr(self, name, value) + + + diff --git a/storefront/boto/ec2/keypair.py b/storefront/boto/ec2/keypair.py new file mode 100644 index 0000000..d08e5ce --- /dev/null +++ b/storefront/boto/ec2/keypair.py @@ -0,0 +1,111 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Keypair +""" + +import os +from boto.ec2.ec2object import EC2Object +from boto.exception import BotoClientError + +class KeyPair(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.name = None + self.fingerprint = None + self.material = None + + def __repr__(self): + return 'KeyPair:%s' % self.name + + def endElement(self, name, value, connection): + if name == 'keyName': + self.name = value + elif name == 'keyFingerprint': + self.fingerprint = value + elif name == 'keyMaterial': + self.material = value + else: + setattr(self, name, value) + + def delete(self): + """ + Delete the KeyPair. + + :rtype: bool + :return: True if successful, otherwise False. + """ + return self.connection.delete_key_pair(self.name) + + def save(self, directory_path): + """ + Save the material (the unencrypted PEM encoded RSA private key) + of a newly created KeyPair to a local file. + + :type directory_path: string + :param directory_path: The fully qualified path to the directory + in which the keypair will be saved. The + keypair file will be named using the name + of the keypair as the base name and .pem + for the file extension. If a file of that + name already exists in the directory, an + exception will be raised and the old file + will not be overwritten. + + :rtype: bool + :return: True if successful. + """ + if self.material: + file_path = os.path.join(directory_path, '%s.pem' % self.name) + if os.path.exists(file_path): + raise BotoClientError('%s already exists, it will not be overwritten' % file_path) + fp = open(file_path, 'wb') + fp.write(self.material) + fp.close() + return True + else: + raise BotoClientError('KeyPair contains no material') + + def copy_to_region(self, region): + """ + Create a new key pair of the same new in another region. + Note that the new key pair will use a different ssh + cert than the this key pair. After doing the copy, + you will need to save the material associated with the + new key pair (use the save method) to a local file. + + :type region: :class:`boto.ec2.regioninfo.RegionInfo` + :param region: The region to which this security group will be copied. + + :rtype: :class:`boto.ec2.keypair.KeyPair` + :return: The new key pair + """ + if region.name == self.region: + raise BotoClientError('Unable to copy to the same Region') + conn_params = self.connection.get_params() + rconn = region.connect(**conn_params) + kp = rconn.create_key_pair(self.name) + return kp + + + diff --git a/storefront/boto/ec2/launchspecification.py b/storefront/boto/ec2/launchspecification.py new file mode 100644 index 0000000..a574a38 --- /dev/null +++ b/storefront/boto/ec2/launchspecification.py @@ -0,0 +1,96 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a launch specification for Spot instances. +""" + +from boto.ec2.ec2object import EC2Object +from boto.resultset import ResultSet +from boto.ec2.blockdevicemapping import BlockDeviceMapping +from boto.ec2.instance import Group + +class GroupList(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'groupId': + self.append(value) + +class LaunchSpecification(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.key_name = None + self.instance_type = None + self.image_id = None + self.groups = [] + self.placement = None + self.kernel = None + self.ramdisk = None + self.monitored = False + self.subnet_id = None + self._in_monitoring_element = False + self.block_device_mapping = None + + def __repr__(self): + return 'LaunchSpecification(%s)' % self.image_id + + def startElement(self, name, attrs, connection): + if name == 'groupSet': + self.groups = ResultSet([('item', Group)]) + return self.groups + elif name == 'monitoring': + self._in_monitoring_element = True + elif name == 'blockDeviceMapping': + self.block_device_mapping = BlockDeviceMapping() + return self.block_device_mapping + else: + return None + + def endElement(self, name, value, connection): + if name == 'imageId': + self.image_id = value + elif name == 'keyName': + self.key_name = value + elif name == 'instanceType': + self.instance_type = value + elif name == 'availabilityZone': + self.placement = value + elif name == 'placement': + pass + elif name == 'kernelId': + self.kernel = value + elif name == 'ramdiskId': + self.ramdisk = value + elif name == 'subnetId': + self.subnet_id = value + elif name == 'state': + if self._in_monitoring_element: + if value == 'enabled': + self.monitored = True + self._in_monitoring_element = False + else: + setattr(self, name, value) + + diff --git a/storefront/boto/ec2/regioninfo.py b/storefront/boto/ec2/regioninfo.py new file mode 100644 index 0000000..ab61703 --- /dev/null +++ b/storefront/boto/ec2/regioninfo.py @@ -0,0 +1,60 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class RegionInfo(object): + """ + Represents an EC2 Region + """ + + def __init__(self, connection=None, name=None, endpoint=None): + self.connection = connection + self.name = name + self.endpoint = endpoint + + def __repr__(self): + return 'RegionInfo:%s' % self.name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'regionName': + self.name = value + elif name == 'regionEndpoint': + self.endpoint = value + else: + setattr(self, name, value) + + def connect(self, **kw_params): + """ + Connect to this Region's endpoint. Returns an EC2Connection + object pointing to the endpoint associated with this region. + You may pass any of the arguments accepted by the EC2Connection + object's constructor as keyword arguments and they will be + passed along to the EC2Connection object. + + :rtype: :class:`boto.ec2.connection.EC2Connection` + :return: The connection to this regions endpoint + """ + from boto.ec2.connection import EC2Connection + return EC2Connection(region=self, **kw_params) + + diff --git a/storefront/boto/ec2/reservedinstance.py b/storefront/boto/ec2/reservedinstance.py new file mode 100644 index 0000000..1d35c1d --- /dev/null +++ b/storefront/boto/ec2/reservedinstance.py @@ -0,0 +1,97 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.ec2.ec2object import EC2Object + +class ReservedInstancesOffering(EC2Object): + + def __init__(self, connection=None, id=None, instance_type=None, + availability_zone=None, duration=None, fixed_price=None, + usage_price=None, description=None): + EC2Object.__init__(self, connection) + self.id = id + self.instance_type = instance_type + self.availability_zone = availability_zone + self.duration = duration + self.fixed_price = fixed_price + self.usage_price = usage_price + self.description = description + + def __repr__(self): + return 'ReservedInstanceOffering:%s' % self.id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'reservedInstancesOfferingId': + self.id = value + elif name == 'instanceType': + self.instance_type = value + elif name == 'availabilityZone': + self.availability_zone = value + elif name == 'duration': + self.duration = value + elif name == 'fixedPrice': + self.fixed_price = value + elif name == 'usagePrice': + self.usage_price = value + elif name == 'productDescription': + self.description = value + else: + setattr(self, name, value) + + def describe(self): + print 'ID=%s' % self.id + print '\tInstance Type=%s' % self.instance_type + print '\tZone=%s' % self.availability_zone + print '\tDuration=%s' % self.duration + print '\tFixed Price=%s' % self.fixed_price + print '\tUsage Price=%s' % self.usage_price + print '\tDescription=%s' % self.description + + def purchase(self, instance_count=1): + return self.connection.purchase_reserved_instance_offering(self.id, instance_count) + +class ReservedInstance(ReservedInstancesOffering): + + def __init__(self, connection=None, id=None, instance_type=None, + availability_zone=None, duration=None, fixed_price=None, + usage_price=None, description=None, + instance_count=None, state=None): + ReservedInstancesOffering.__init__(self, connection, id, instance_type, + availability_zone, duration, fixed_price, + usage_price, description) + self.instance_count = instance_count + self.state = state + + def __repr__(self): + return 'ReservedInstance:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'reservedInstancesId': + self.id = value + if name == 'instanceCount': + self.instance_count = int(value) + elif name == 'state': + self.state = value + else: + ReservedInstancesOffering.endElement(self, name, value, connection) diff --git a/storefront/boto/ec2/securitygroup.py b/storefront/boto/ec2/securitygroup.py new file mode 100644 index 0000000..6f17ad3 --- /dev/null +++ b/storefront/boto/ec2/securitygroup.py @@ -0,0 +1,281 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Security Group +""" +from boto.ec2.ec2object import EC2Object + +class SecurityGroup(EC2Object): + + def __init__(self, connection=None, owner_id=None, + name=None, description=None): + EC2Object.__init__(self, connection) + self.owner_id = owner_id + self.name = name + self.description = description + self.rules = [] + + def __repr__(self): + return 'SecurityGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'item': + self.rules.append(IPPermissions(self)) + return self.rules[-1] + else: + return None + + def endElement(self, name, value, connection): + if name == 'ownerId': + self.owner_id = value + elif name == 'groupName': + self.name = value + elif name == 'groupDescription': + self.description = value + elif name == 'ipRanges': + pass + elif name == 'return': + if value == 'false': + self.status = False + elif value == 'true': + self.status = True + else: + raise Exception( + 'Unexpected value of status %s for image %s'%( + value, + self.id + ) + ) + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_security_group(self.name) + + def add_rule(self, ip_protocol, from_port, to_port, + src_group_name, src_group_owner_id, cidr_ip): + rule = IPPermissions(self) + rule.ip_protocol = ip_protocol + rule.from_port = from_port + rule.to_port = to_port + self.rules.append(rule) + rule.add_grant(src_group_name, src_group_owner_id, cidr_ip) + + def remove_rule(self, ip_protocol, from_port, to_port, + src_group_name, src_group_owner_id, cidr_ip): + target_rule = None + for rule in self.rules: + if rule.ip_protocol == ip_protocol: + if rule.from_port == from_port: + if rule.to_port == to_port: + target_rule = rule + target_grant = None + for grant in rule.grants: + if grant.name == src_group_name: + if grant.owner_id == src_group_owner_id: + if grant.cidr_ip == cidr_ip: + target_grant = grant + if target_grant: + rule.grants.remove(target_grant) + if len(rule.grants) == 0: + self.rules.remove(target_rule) + + def authorize(self, ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None, src_group=None): + """ + Add a new rule to this security group. + You need to pass in either src_group_name + OR ip_protocol, from_port, to_port, + and cidr_ip. In other words, either you are authorizing another + group or you are authorizing some ip-based rule. + + :type ip_protocol: string + :param ip_protocol: Either tcp | udp | icmp + + :type from_port: int + :param from_port: The beginning port number you are enabling + + :type to_port: int + :param to_port: The ending port number you are enabling + + :type to_port: string + :param to_port: The CIDR block you are providing access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :type src_group: :class:`boto.ec2.securitygroup.SecurityGroup` or + :class:`boto.ec2.securitygroup.GroupOrCIDR` + + :rtype: bool + :return: True if successful. + """ + if src_group: + from_port = None + to_port = None + cidr_ip = None + ip_protocol = None + src_group_name = src_group.name + src_group_owner_id = src_group.owner_id + else: + src_group_name = None + src_group_owner_id = None + status = self.connection.authorize_security_group(self.name, + src_group_name, + src_group_owner_id, + ip_protocol, + from_port, + to_port, + cidr_ip) + if status: + self.add_rule(ip_protocol, from_port, to_port, src_group_name, + src_group_owner_id, cidr_ip) + return status + + def revoke(self, ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None, src_group=None): + if src_group: + from_port=None + to_port=None + cidr_ip=None + ip_protocol = None + src_group_name = src_group.name + src_group_owner_id = src_group.owner_id + else: + src_group_name = None + src_group_owner_id = None + status = self.connection.revoke_security_group(self.name, + src_group_name, + src_group_owner_id, + ip_protocol, + from_port, + to_port, + cidr_ip) + if status: + self.remove_rule(ip_protocol, from_port, to_port, src_group_name, + src_group_owner_id, cidr_ip) + return status + + def copy_to_region(self, region, name=None): + """ + Create a copy of this security group in another region. + Note that the new security group will be a separate entity + and will not stay in sync automatically after the copy + operation. + + :type region: :class:`boto.ec2.regioninfo.RegionInfo` + :param region: The region to which this security group will be copied. + + :type name: string + :param name: The name of the copy. If not supplied, the copy + will have the same name as this security group. + + :rtype: :class:`boto.ec2.securitygroup.SecurityGroup` + :return: The new security group. + """ + if region.name == self.region: + raise BotoClientError('Unable to copy to the same Region') + conn_params = self.connection.get_params() + rconn = region.connect(**conn_params) + sg = rconn.create_security_group(name or self.name, self.description) + source_groups = [] + for rule in self.rules: + grant = rule.grants[0] + if grant.name: + if grant.name not in source_groups: + source_groups.append(grant.name) + sg.authorize(None, None, None, None, grant) + else: + sg.authorize(rule.ip_protocol, rule.from_port, rule.to_port, + grant.cidr_ip) + return sg + + def instances(self): + instances = [] + rs = self.connection.get_all_instances() + for reservation in rs: + uses_group = [g.id for g in reservation.groups if g.id == self.name] + if uses_group: + instances.extend(reservation.instances) + return instances + +class IPPermissions: + + def __init__(self, parent=None): + self.parent = parent + self.ip_protocol = None + self.from_port = None + self.to_port = None + self.grants = [] + + def __repr__(self): + return 'IPPermissions:%s(%s-%s)' % (self.ip_protocol, + self.from_port, self.to_port) + + def startElement(self, name, attrs, connection): + if name == 'item': + self.grants.append(GroupOrCIDR(self)) + return self.grants[-1] + return None + + def endElement(self, name, value, connection): + if name == 'ipProtocol': + self.ip_protocol = value + elif name == 'fromPort': + self.from_port = value + elif name == 'toPort': + self.to_port = value + else: + setattr(self, name, value) + + def add_grant(self, owner_id=None, name=None, cidr_ip=None): + grant = GroupOrCIDR(self) + grant.owner_id = owner_id + grant.name = name + grant.cidr_ip = cidr_ip + self.grants.append(grant) + return grant + +class GroupOrCIDR: + + def __init__(self, parent=None): + self.owner_id = None + self.name = None + self.cidr_ip = None + + def __repr__(self): + if self.cidr_ip: + return '%s' % self.cidr_ip + else: + return '%s-%s' % (self.name, self.owner_id) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'userId': + self.owner_id = value + elif name == 'groupName': + self.name = value + if name == 'cidrIp': + self.cidr_ip = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/ec2/snapshot.py b/storefront/boto/ec2/snapshot.py new file mode 100644 index 0000000..33b53b0 --- /dev/null +++ b/storefront/boto/ec2/snapshot.py @@ -0,0 +1,124 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Elastic IP Snapshot +""" +from boto.ec2.ec2object import EC2Object + +class Snapshot(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.volume_id = None + self.status = None + self.progress = None + self.start_time = None + self.owner_id = None + self.volume_size = None + self.description = None + + def __repr__(self): + return 'Snapshot:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'snapshotId': + self.id = value + elif name == 'volumeId': + self.volume_id = value + elif name == 'status': + self.status = value + elif name == 'startTime': + self.start_time = value + elif name == 'ownerId': + self.owner_id = value + elif name == 'volumeSize': + self.volume_size = int(value) + elif name == 'description': + self.description = value + else: + setattr(self, name, value) + + def _update(self, updated): + self.progress = updated.progress + self.status = updated.status + + def update(self): + rs = self.connection.get_all_snapshots([self.id]) + if len(rs) > 0: + self._update(rs[0]) + return self.progress + + def delete(self): + return self.connection.delete_snapshot(self.id) + + def get_permissions(self): + attrs = self.connection.get_snapshot_attribute(self.id, + attribute='createVolumePermission') + return attrs.attrs + + def share(self, user_ids=None, groups=None): + return self.connection.modify_snapshot_attribute(self.id, + 'createVolumePermission', + 'add', + user_ids, + groups) + + def unshare(self, user_ids=None, groups=None): + return self.connection.modify_snapshot_attribute(self.id, + 'createVolumePermission', + 'remove', + user_ids, + groups) + + def reset_permissions(self): + return self.connection.reset_snapshot_attribute(self.id, 'createVolumePermission') + +class SnapshotAttribute: + + def __init__(self, parent=None): + self.snapshot_id = None + self.attrs = {} + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'createVolumePermission': + self.name = 'create_volume_permission' + elif name == 'group': + if self.attrs.has_key('groups'): + self.attrs['groups'].append(value) + else: + self.attrs['groups'] = [value] + elif name == 'userId': + if self.attrs.has_key('user_ids'): + self.attrs['user_ids'].append(value) + else: + self.attrs['user_ids'] = [value] + elif name == 'snapshotId': + self.snapshot_id = value + else: + setattr(self, name, value) + + + diff --git a/storefront/boto/ec2/spotdatafeedsubscription.py b/storefront/boto/ec2/spotdatafeedsubscription.py new file mode 100644 index 0000000..9b820a3 --- /dev/null +++ b/storefront/boto/ec2/spotdatafeedsubscription.py @@ -0,0 +1,63 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Spot Instance Datafeed Subscription +""" +from boto.ec2.ec2object import EC2Object +from boto.ec2.spotinstancerequest import SpotInstanceStateFault + +class SpotDatafeedSubscription(EC2Object): + + def __init__(self, connection=None, owner_id=None, + bucket=None, prefix=None, state=None,fault=None): + EC2Object.__init__(self, connection) + self.owner_id = owner_id + self.bucket = bucket + self.prefix = prefix + self.state = state + self.fault = fault + + def __repr__(self): + return 'SpotDatafeedSubscription:%s' % self.bucket + + def startElement(self, name, attrs, connection): + if name == 'fault': + self.fault = SpotInstanceStateFault() + return self.fault + else: + return None + + def endElement(self, name, value, connection): + if name == 'ownerId': + self.owner_id = value + elif name == 'bucket': + self.bucket = value + elif name == 'prefix': + self.prefix = value + elif name == 'state': + self.state = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_spot_datafeed_subscription() + diff --git a/storefront/boto/ec2/spotinstancerequest.py b/storefront/boto/ec2/spotinstancerequest.py new file mode 100644 index 0000000..5b1d7ce --- /dev/null +++ b/storefront/boto/ec2/spotinstancerequest.py @@ -0,0 +1,106 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Spot Instance Request +""" + +from boto.ec2.ec2object import EC2Object +from boto.ec2.launchspecification import LaunchSpecification + +class SpotInstanceStateFault(object): + + def __init__(self, code=None, message=None): + self.code = code + self.message = message + + def __repr__(self): + return '(%s, %s)' % (self.code, self.message) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'code': + self.code = code + elif name == 'message': + self.message = message + setattr(self, name, value) + +class SpotInstanceRequest(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.price = None + self.type = None + self.state = None + self.fault = None + self.valid_from = None + self.valid_until = None + self.launch_group = None + self.product_description = None + self.availability_zone_group = None + self.create_time = None + self.launch_specification = None + + def __repr__(self): + return 'SpotInstanceRequest:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'launchSpecification': + self.launch_specification = LaunchSpecification(connection) + return self.launch_specification + elif name == 'fault': + self.fault = SpotInstanceStateFault() + return self.fault + else: + return None + + def endElement(self, name, value, connection): + if name == 'spotInstanceRequestId': + self.id = value + elif name == 'spotPrice': + self.price = float(value) + elif name == 'type': + self.type = value + elif name == 'state': + self.state = value + elif name == 'productDescription': + self.product_description = value + elif name == 'validFrom': + self.valid_from = value + elif name == 'validUntil': + self.valid_until = value + elif name == 'launchGroup': + self.launch_group = value + elif name == 'availabilityZoneGroup': + self.availability_zone_group = value + elif name == 'createTime': + self.create_time = value + else: + setattr(self, name, value) + + def cancel(self): + self.connection.cancel_spot_instance_requests([self.id]) + + + diff --git a/storefront/boto/ec2/spotpricehistory.py b/storefront/boto/ec2/spotpricehistory.py new file mode 100644 index 0000000..d4e1711 --- /dev/null +++ b/storefront/boto/ec2/spotpricehistory.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Spot Instance Request +""" + +from boto.ec2.ec2object import EC2Object + +class SpotPriceHistory(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.price = 0.0 + self.instance_type = None + self.product_description = None + self.timestamp = None + + def __repr__(self): + return 'SpotPriceHistory(%s):%2f' % (self.instance_type, self.price) + + def endElement(self, name, value, connection): + if name == 'instanceType': + self.instance_type = value + elif name == 'spotPrice': + self.price = float(value) + elif name == 'productDescription': + self.product_description = value + elif name == 'timestamp': + self.timestamp = value + else: + setattr(self, name, value) + + diff --git a/storefront/boto/ec2/volume.py b/storefront/boto/ec2/volume.py new file mode 100644 index 0000000..200ca90 --- /dev/null +++ b/storefront/boto/ec2/volume.py @@ -0,0 +1,229 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Elastic IP Volume +""" +from boto.ec2.ec2object import EC2Object + +class Volume(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.create_time = None + self.status = None + self.size = None + self.snapshot_id = None + self.attach_data = None + self.zone = None + + def __repr__(self): + return 'Volume:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'attachmentSet': + self.attach_data = AttachmentSet() + return self.attach_data + else: + return None + + def endElement(self, name, value, connection): + if name == 'volumeId': + self.id = value + elif name == 'createTime': + self.create_time = value + elif name == 'status': + if value != '': + self.status = value + elif name == 'size': + self.size = int(value) + elif name == 'snapshotId': + self.snapshot_id = value + elif name == 'availabilityZone': + self.zone = value + else: + setattr(self, name, value) + + def _update(self, updated): + self.updated = updated + if hasattr(updated, 'create_time'): + self.create_time = updated.create_time + if hasattr(updated, 'status'): + self.status = updated.status + else: + self.status = None + if hasattr(updated, 'size'): + self.size = updated.size + if hasattr(updated, 'snapshot_id'): + self.snapshot_id = updated.snapshot_id + if hasattr(updated, 'attach_data'): + self.attach_data = updated.attach_data + if hasattr(updated, 'zone'): + self.zone = updated.zone + + def update(self): + rs = self.connection.get_all_volumes([self.id]) + if len(rs) > 0: + self._update(rs[0]) + return self.status + + def delete(self): + """ + Delete this EBS volume. + + :rtype: bool + :return: True if successful + """ + return self.connection.delete_volume(self.id) + + def attach(self, instance_id, device): + """ + Attach this EBS volume to an EC2 instance. + + :type instance_id: str + :param instance_id: The ID of the EC2 instance to which it will + be attached. + + :type device: str + :param device: The device on the instance through which the + volume will be exposted (e.g. /dev/sdh) + + :rtype: bool + :return: True if successful + """ + return self.connection.attach_volume(self.id, instance_id, device) + + def detach(self, force=False): + """ + Detach this EBS volume from an EC2 instance. + + :type force: bool + :param force: Forces detachment if the previous detachment attempt did + not occur cleanly. This option can lead to data loss or + a corrupted file system. Use this option only as a last + resort to detach a volume from a failed instance. The + instance will not have an opportunity to flush file system + caches nor file system meta data. If you use this option, + you must perform file system check and repair procedures. + + :rtype: bool + :return: True if successful + """ + instance_id = None + if self.attach_data: + instance_id = self.attach_data.instance_id + device = None + if self.attach_data: + device = self.attach_data.device + return self.connection.detach_volume(self.id, instance_id, device, force) + + def create_snapshot(self, description=None): + """ + Create a snapshot of this EBS Volume. + + :type description: str + :param description: A description of the snapshot. Limited to 256 characters. + + :rtype: bool + :return: True if successful + """ + return self.connection.create_snapshot(self.id, description) + + def volume_state(self): + """ + Returns the state of the volume. Same value as the status attribute. + """ + return self.status + + def attachment_state(self): + """ + Get the attachmentSet information for the volume. This info is stored + in a dictionary object and contains at least the following info: + + - volumeId + - instanceId + - device + - status + - attachTime + """ + state = None + if self.attach_data: + state = self.attach_data.status + return state + + def snapshots(self, owner=None, restorable_by=None): + """ + Get all snapshots related to this volume. Note that this requires + that all available snapshots for the account be retrieved from EC2 + first and then the list is filtered client-side to contain only + those for this volume. + + :type owner: str + :param owner: If present, only the snapshots owned by the specified user + will be returned. Valid values are: + self | amazon | AWS Account ID + + :type restorable_by: str + :param restorable_by: If present, only the snapshots that are restorable + by the specified account id will be returned. + + :rtype: list of L{boto.ec2.snapshot.Snapshot} + :return: The requested Snapshot objects + + """ + rs = self.connection.get_all_snapshots(owner=owner, + restorable_by=restorable_by) + mine = [] + for snap in rs: + if snap.volume_id == self.id: + mine.append(snap) + return mine + +class AttachmentSet(object): + + def __init__(self): + self.id = None + self.instance_id = None + self.status = None + self.attach_time = None + self.device = None + + def __repr__(self): + return 'AttachmentSet:%s' % self.id + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'volumeId': + self.id = value + elif name == 'instanceId': + self.instance_id = value + elif name == 'status': + self.status = value + elif name == 'attachTime': + self.attach_time = value + elif name == 'device': + self.device = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/ec2/zone.py b/storefront/boto/ec2/zone.py new file mode 100644 index 0000000..aec79b2 --- /dev/null +++ b/storefront/boto/ec2/zone.py @@ -0,0 +1,47 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Availability Zone +""" +from boto.ec2.ec2object import EC2Object + +class Zone(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.name = None + self.state = None + + def __repr__(self): + return 'Zone:%s' % self.name + + def endElement(self, name, value, connection): + if name == 'zoneName': + self.name = value + elif name == 'zoneState': + self.state = value + else: + setattr(self, name, value) + + + + diff --git a/storefront/boto/exception.py b/storefront/boto/exception.py new file mode 100644 index 0000000..ba65694 --- /dev/null +++ b/storefront/boto/exception.py @@ -0,0 +1,284 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Exception classes - Subclassing allows you to check for specific errors +""" + +from boto import handler +from boto.resultset import ResultSet + +import xml.sax + +class BotoClientError(StandardError): + """ + General Boto Client error (error accessing AWS) + """ + + def __init__(self, reason): + self.reason = reason + + def __repr__(self): + return 'S3Error: %s' % self.reason + + def __str__(self): + return 'S3Error: %s' % self.reason + +class SDBPersistenceError(StandardError): + + pass + +class S3PermissionsError(BotoClientError): + """ + Permissions error when accessing a bucket or key on S3. + """ + pass + +class BotoServerError(StandardError): + + def __init__(self, status, reason, body=None): + self.status = status + self.reason = reason + self.body = body or '' + self.request_id = None + self.error_code = None + self.error_message = None + self.box_usage = None + + # Attempt to parse the error response. If body isn't present, + # then just ignore the error response. + if self.body: + try: + h = handler.XmlHandler(self, self) + xml.sax.parseString(self.body, h) + except xml.sax.SAXParseException, pe: + # Go ahead and clean up anything that may have + # managed to get into the error data so we + # don't get partial garbage. + print "Warning: failed to parse error message from AWS: %s" % pe + self._cleanupParsedProperties() + + def __getattr__(self, name): + if name == 'message': + return self.error_message + if name == 'code': + return self.error_code + raise AttributeError + + def __repr__(self): + return '%s: %s %s\n%s' % (self.__class__.__name__, + self.status, self.reason, self.body) + + def __str__(self): + return '%s: %s %s\n%s' % (self.__class__.__name__, + self.status, self.reason, self.body) + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name in ('RequestId', 'RequestID'): + self.request_id = value + elif name == 'Code': + self.error_code = value + elif name == 'Message': + self.error_message = value + elif name == 'BoxUsage': + self.box_usage = value + return None + + def _cleanupParsedProperties(self): + self.request_id = None + self.error_code = None + self.error_message = None + self.box_usage = None + +class ConsoleOutput: + + def __init__(self, parent=None): + self.parent = parent + self.instance_id = None + self.timestamp = None + self.comment = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'instanceId': + self.instance_id = value + elif name == 'output': + self.output = base64.b64decode(value) + else: + setattr(self, name, value) + +class S3CreateError(BotoServerError): + """ + Error creating a bucket or key on S3. + """ + def __init__(self, status, reason, body=None): + self.bucket = None + BotoServerError.__init__(self, status, reason, body) + + def endElement(self, name, value, connection): + if name == 'BucketName': + self.bucket = value + else: + return BotoServerError.endElement(self, name, value, connection) + +class S3CopyError(BotoServerError): + """ + Error copying a key on S3. + """ + pass + +class SQSError(BotoServerError): + """ + General Error on Simple Queue Service. + """ + def __init__(self, status, reason, body=None): + self.detail = None + self.type = None + BotoServerError.__init__(self, status, reason, body) + + def startElement(self, name, attrs, connection): + return BotoServerError.startElement(self, name, attrs, connection) + + def endElement(self, name, value, connection): + if name == 'Detail': + self.detail = value + elif name == 'Type': + self.type = value + else: + return BotoServerError.endElement(self, name, value, connection) + + def _cleanupParsedProperties(self): + BotoServerError._cleanupParsedProperties(self) + for p in ('detail', 'type'): + setattr(self, p, None) + +class SQSDecodeError(BotoClientError): + """ + Error when decoding an SQS message. + """ + def __init__(self, reason, message): + self.reason = reason + self.message = message + + def __repr__(self): + return 'SQSDecodeError: %s' % self.reason + + def __str__(self): + return 'SQSDecodeError: %s' % self.reason + +class S3ResponseError(BotoServerError): + """ + Error in response from S3. + """ + def __init__(self, status, reason, body=None): + self.resource = None + BotoServerError.__init__(self, status, reason, body) + + def startElement(self, name, attrs, connection): + return BotoServerError.startElement(self, name, attrs, connection) + + def endElement(self, name, value, connection): + if name == 'Resource': + self.resource = value + else: + return BotoServerError.endElement(self, name, value, connection) + + def _cleanupParsedProperties(self): + BotoServerError._cleanupParsedProperties(self) + for p in ('resource'): + setattr(self, p, None) + +class EC2ResponseError(BotoServerError): + """ + Error in response from EC2. + """ + + def __init__(self, status, reason, body=None): + self.errors = None + self._errorResultSet = [] + BotoServerError.__init__(self, status, reason, body) + self.errors = [ (e.error_code, e.error_message) \ + for e in self._errorResultSet ] + if len(self.errors): + self.error_code, self.error_message = self.errors[0] + + def startElement(self, name, attrs, connection): + if name == 'Errors': + self._errorResultSet = ResultSet([('Error', _EC2Error)]) + return self._errorResultSet + else: + return None + + def endElement(self, name, value, connection): + if name == 'RequestID': + self.request_id = value + else: + return None # don't call subclass here + + def _cleanupParsedProperties(self): + BotoServerError._cleanupParsedProperties(self) + self._errorResultSet = [] + for p in ('errors'): + setattr(self, p, None) + +class _EC2Error: + + def __init__(self, connection=None): + self.connection = connection + self.error_code = None + self.error_message = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Code': + self.error_code = value + elif name == 'Message': + self.error_message = value + else: + return None + +class SDBResponseError(BotoServerError): + """ + Error in respones from SDB. + """ + pass + +class AWSConnectionError(BotoClientError): + """ + General error connecting to Amazon Web Services. + """ + pass + +class S3DataError(BotoClientError): + """ + Error receiving data from S3. + """ + pass + +class FPSResponseError(BotoServerError): + pass diff --git a/storefront/boto/fps/__init__.py b/storefront/boto/fps/__init__.py new file mode 100644 index 0000000..2f44483 --- /dev/null +++ b/storefront/boto/fps/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2008, Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/storefront/boto/fps/connection.py b/storefront/boto/fps/connection.py new file mode 100644 index 0000000..0f14775 --- /dev/null +++ b/storefront/boto/fps/connection.py @@ -0,0 +1,173 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import urllib +import xml.sax +import uuid +import boto +import boto.utils +import urllib +from boto import handler +from boto.connection import AWSQueryConnection +from boto.resultset import ResultSet +from boto.exception import FPSResponseError + +class FPSConnection(AWSQueryConnection): + + APIVersion = '2007-01-08' + SignatureVersion = '1' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + host='fps.sandbox.amazonaws.com', debug=0, + https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + host, debug, https_connection_factory) + + def install_payment_instruction(self, instruction, token_type="Unrestricted", transaction_id=None): + """ + InstallPaymentInstruction + instruction: The PaymentInstruction to send, for example: + + MyRole=='Caller' orSay 'Roles do not match'; + + token_type: Defaults to "Unrestricted" + transaction_id: Defaults to a new ID + """ + + if(transaction_id == None): + transaction_id = uuid.uuid4() + params = {} + params['PaymentInstruction'] = instruction + params['TokenType'] = token_type + params['CallerReference'] = transaction_id + response = self.make_request("InstallPaymentInstruction", params) + return response + + def install_caller_instruction(self, token_type="Unrestricted", transaction_id=None): + """ + Set us up as a caller + This will install a new caller_token into the FPS section. + This should really only be called to regenerate the caller token. + """ + response = self.install_payment_instruction("MyRole=='Caller';", token_type=token_type, transaction_id=transaction_id) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + caller_token = rs.TokenId + try: + boto.config.save_system_option("FPS", "caller_token", caller_token) + except(IOError): + boto.config.save_user_option("FPS", "caller_token", caller_token) + return caller_token + else: + raise FPSResponseError(response.status, respons.reason, body) + + def install_recipient_instruction(self, token_type="Unrestricted", transaction_id=None): + """ + Set us up as a Recipient + This will install a new caller_token into the FPS section. + This should really only be called to regenerate the recipient token. + """ + response = self.install_payment_instruction("MyRole=='Recipient';", token_type=token_type, transaction_id=transaction_id) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + recipient_token = rs.TokenId + try: + boto.config.save_system_option("FPS", "recipient_token", recipient_token) + except(IOError): + boto.config.save_user_option("FPS", "recipient_token", recipient_token) + + return recipient_token + else: + raise FPSResponseError(response.status, respons.reason, body) + + def make_url(self, returnURL, paymentReason, pipelineName, **params): + """ + Generate the URL with the signature required for a transaction + """ + params['callerKey'] = str(self.aws_access_key_id) + params['returnURL'] = str(returnURL) + params['paymentReason'] = str(paymentReason) + params['pipelineName'] = pipelineName + + if(not params.has_key('callerReference')): + params['callerReference'] = str(uuid.uuid4()) + + url = "" + keys = params.keys() + keys.sort() + for k in keys: + url += "&%s=%s" % (k, urllib.quote_plus(str(params[k]))) + + url = "/cobranded-ui/actions/start?%s" % ( url[1:]) + signature= boto.utils.encode(self.aws_secret_access_key, url, True) + return "https://authorize.payments-sandbox.amazon.com%s&awsSignature=%s" % (url, signature) + + def make_payment(self, amount, sender_token, charge_fee_to="Recipient", reference=None, senderReference=None, recipientReference=None, senderDescription=None, recipientDescription=None, callerDescription=None, metadata=None, transactionDate=None): + """ + Make a payment transaction + You must specify the amount and the sender token. + """ + params = {} + params['RecipientTokenId'] = boto.config.get("FPS", "recipient_token") + params['CallerTokenId'] = boto.config.get("FPS", "caller_token") + params['SenderTokenId'] = sender_token + params['TransactionAmount.Amount'] = str(amount) + params['TransactionAmount.CurrencyCode'] = "USD" + params['ChargeFeeTo'] = charge_fee_to + + if(transactionDate != None): + params['TransactionDate'] = transactionDate + if(senderReference != None): + params['SenderReference'] = senderReference + if(recipientReference != None): + params['RecipientReference'] = recipientReference + if(senderDescription != None): + params['SenderDescription'] = senderDescription + if(recipientDescription != None): + params['RecipientDescription'] = recipientDescription + if(callerDescription != None): + params['CallerDescription'] = callerDescription + if(metadata != None): + params['MetaData'] = metadata + if(transactionDate != None): + params['TransactionDate'] = transactionDate + if(reference == None): + reference = uuid.uuid4() + params['CallerReference'] = reference + + response = self.make_request("Pay", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) diff --git a/storefront/boto/handler.py b/storefront/boto/handler.py new file mode 100644 index 0000000..525f9c9 --- /dev/null +++ b/storefront/boto/handler.py @@ -0,0 +1,46 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax + +class XmlHandler(xml.sax.ContentHandler): + + def __init__(self, root_node, connection): + self.connection = connection + self.nodes = [('root', root_node)] + self.current_text = '' + + def startElement(self, name, attrs): + self.current_text = '' + new_node = self.nodes[-1][1].startElement(name, attrs, self.connection) + if new_node != None: + self.nodes.append((name, new_node)) + + def endElement(self, name): + self.nodes[-1][1].endElement(name, self.current_text, self.connection) + if self.nodes[-1][0] == name: + self.nodes.pop() + self.current_text = '' + + def characters(self, content): + self.current_text += content + + diff --git a/storefront/boto/manage/__init__.py b/storefront/boto/manage/__init__.py new file mode 100644 index 0000000..49d029b --- /dev/null +++ b/storefront/boto/manage/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/storefront/boto/manage/cmdshell.py b/storefront/boto/manage/cmdshell.py new file mode 100644 index 0000000..340b1e2 --- /dev/null +++ b/storefront/boto/manage/cmdshell.py @@ -0,0 +1,165 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.mashups.interactive import interactive_shell +import boto +import os, time, shutil +import StringIO +import paramiko +import socket + +class SSHClient(object): + + def __init__(self, server, host_key_file='~/.ssh/known_hosts', uname='root'): + self.server = server + self.host_key_file = host_key_file + self.uname = uname + self._pkey = paramiko.RSAKey.from_private_key_file(server.ssh_key_file) + self._ssh_client = paramiko.SSHClient() + self._ssh_client.load_system_host_keys() + self._ssh_client.load_host_keys(os.path.expanduser(host_key_file)) + self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.connect() + + def connect(self): + retry = 0 + while retry < 5: + try: + self._ssh_client.connect(self.server.hostname, username=self.uname, pkey=self._pkey) + return + except socket.error, (value,message): + if value == 61: + print 'SSH Connection refused, will retry in 5 seconds' + time.sleep(5) + retry += 1 + else: + raise + except paramiko.BadHostKeyException: + print "%s has an entry in ~/.ssh/known_hosts and it doesn't match" % self.server.hostname + print 'Edit that file to remove the entry and then hit return to try again' + rawinput('Hit Enter when ready') + retry += 1 + except EOFError: + print 'Unexpected Error from SSH Connection, retry in 5 seconds' + time.sleep(5) + retry += 1 + print 'Could not establish SSH connection' + + def get_file(self, src, dst): + sftp_client = self._ssh_client.open_sftp() + sftp_client.get(src, dst) + + def put_file(self, src, dst): + sftp_client = self._ssh_client.open_sftp() + sftp_client.put(src, dst) + + def listdir(self, path): + sftp_client = self._ssh_client.open_sftp() + return sftp_client.listdir(path) + + def open_sftp(self): + return self._ssh_client.open_sftp() + + def isdir(self, path): + status = self.run('[ -d %s ] || echo "FALSE"' % path) + if status[1].startswith('FALSE'): + return 0 + return 1 + + def exists(self, path): + status = self.run('[ -a %s ] || echo "FALSE"' % path) + if status[1].startswith('FALSE'): + return 0 + return 1 + + def shell(self): + channel = self._ssh_client.invoke_shell() + interactive_shell(channel) + + def run(self, command): + boto.log.info('running:%s on %s' % (command, self.server.instance_id)) + log_fp = StringIO.StringIO() + status = 0 + try: + t = self._ssh_client.exec_command(command) + except paramiko.SSHException: + status = 1 + log_fp.write(t[1].read()) + log_fp.write(t[2].read()) + t[0].close() + t[1].close() + t[2].close() + boto.log.info('output: %s' % log_fp.getvalue()) + return (status, log_fp.getvalue()) + + def close(self): + transport = self._ssh_client.get_transport() + transport.close() + self.server.reset_cmdshell() + +class LocalClient(object): + + def __init__(self, server, host_key_file=None, uname='root'): + self.server = server + self.host_key_file = host_key_file + self.uname = uname + + def get_file(self, src, dst): + shutil.copyfile(src, dst) + + def put_file(self, src, dst): + shutil.copyfile(src, dst) + + def listdir(self, path): + return os.listdir(path) + + def isdir(self, path): + return os.path.isdir(path) + + def exists(self, path): + return os.path.exists(path) + + def shell(self): + raise NotImplementedError, 'shell not supported with LocalClient' + + def run(self): + boto.log.info('running:%s' % self.command) + log_fp = StringIO.StringIO() + process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + while process.poll() == None: + time.sleep(1) + t = process.communicate() + log_fp.write(t[0]) + log_fp.write(t[1]) + boto.log.info(log_fp.getvalue()) + boto.log.info('output: %s' % log_fp.getvalue()) + return (process.returncode, log_fp.getvalue()) + + def close(self): + pass + +def start(server): + instance_id = boto.config.get('Instance', 'instance-id', None) + if instance_id == server.instance_id: + return LocalClient(server) + else: + return SSHClient(server) diff --git a/storefront/boto/manage/propget.py b/storefront/boto/manage/propget.py new file mode 100644 index 0000000..172e1aa --- /dev/null +++ b/storefront/boto/manage/propget.py @@ -0,0 +1,66 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import os +from boto.sdb.db.property import * + +def get(prop, choices=None): + prompt = prop.verbose_name + if not prompt: + prompt = prop.name + if choices: + if callable(choices): + choices = choices() + else: + choices = prop.get_choices() + valid = False + while not valid: + if choices: + min = 1 + max = len(choices) + for i in range(min, max+1): + value = choices[i-1] + if isinstance(value, tuple): + value = value[0] + print '[%d] %s' % (i, value) + value = raw_input('%s [%d-%d]: ' % (prompt, min, max)) + try: + int_value = int(value) + value = choices[int_value-1] + if isinstance(value, tuple): + value = value[1] + valid = True + except ValueError: + print '%s is not a valid choice' % value + except IndexError: + print '%s is not within the range[%d-%d]' % (min, max) + else: + value = raw_input('%s: ' % prompt) + try: + value = prop.validate(value) + if prop.empty(value) and prop.required: + print 'A value is required' + else: + valid = True + except: + print 'Invalid value: %s' % value + return value + diff --git a/storefront/boto/manage/server.py b/storefront/boto/manage/server.py new file mode 100644 index 0000000..cc623ef --- /dev/null +++ b/storefront/boto/manage/server.py @@ -0,0 +1,542 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +High-level abstraction of an EC2 server +""" +from __future__ import with_statement +import boto.ec2 +from boto.mashups.iobject import IObject +from boto.pyami.config import BotoConfigPath, Config +from boto.sdb.db.model import Model +from boto.sdb.db.property import * +from boto.manage import propget +from boto.ec2.zone import Zone +from boto.ec2.keypair import KeyPair +import os, time, StringIO +from contextlib import closing +from boto.exception import EC2ResponseError + +InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', + 'c1.medium', 'c1.xlarge', + 'm2.2xlarge', 'm2.4xlarge'] + +class Bundler(object): + + def __init__(self, server, uname='root'): + from boto.manage.cmdshell import SSHClient + self.server = server + self.uname = uname + self.ssh_client = SSHClient(server, uname=uname) + + def copy_x509(self, key_file, cert_file): + print '\tcopying cert and pk over to /mnt directory on server' + sftp_client = self.ssh_client.open_sftp() + path, name = os.path.split(key_file) + self.remote_key_file = '/mnt/%s' % name + self.ssh_client.put_file(key_file, self.remote_key_file) + path, name = os.path.split(cert_file) + self.remote_cert_file = '/mnt/%s' % name + self.ssh_client.put_file(cert_file, self.remote_cert_file) + print '...complete!' + + def bundle_image(self, prefix, size, ssh_key): + command = "" + if self.uname != 'root': + command = "sudo " + command += 'ec2-bundle-vol ' + command += '-c %s -k %s ' % (self.remote_cert_file, self.remote_key_file) + command += '-u %s ' % self.server._reservation.owner_id + command += '-p %s ' % prefix + command += '-s %d ' % size + command += '-d /mnt ' + if self.server.instance_type == 'm1.small' or self.server.instance_type == 'c1.medium': + command += '-r i386' + else: + command += '-r x86_64' + return command + + def upload_bundle(self, bucket, prefix, ssh_key): + command = "" + if self.uname != 'root': + command = "sudo " + command += 'ec2-upload-bundle ' + command += '-m /mnt/%s.manifest.xml ' % prefix + command += '-b %s ' % bucket + command += '-a %s ' % self.server.ec2.aws_access_key_id + command += '-s %s ' % self.server.ec2.aws_secret_access_key + return command + + def bundle(self, bucket=None, prefix=None, key_file=None, cert_file=None, + size=None, ssh_key=None, fp=None, clear_history=True): + iobject = IObject() + if not bucket: + bucket = iobject.get_string('Name of S3 bucket') + if not prefix: + prefix = iobject.get_string('Prefix for AMI file') + if not key_file: + key_file = iobject.get_filename('Path to RSA private key file') + if not cert_file: + cert_file = iobject.get_filename('Path to RSA public cert file') + if not size: + size = iobject.get_int('Size (in MB) of bundled image') + if not ssh_key: + ssh_key = self.server.get_ssh_key_file() + self.copy_x509(key_file, cert_file) + if not fp: + fp = StringIO.StringIO() + fp.write('mv %s /mnt/boto.cfg; ' % BotoConfigPath) + fp.write('mv /root/.ssh/authorized_keys /mnt/authorized_keys; ') + if clear_history: + fp.write('history -c; ') + fp.write(self.bundle_image(prefix, size, ssh_key)) + fp.write('; ') + fp.write(self.upload_bundle(bucket, prefix, ssh_key)) + fp.write('; ') + fp.write('mv /mnt/boto.cfg %s; ' % BotoConfigPath) + fp.write('mv /mnt/authorized_keys /root/.ssh/authorized_keys\n') + command = fp.getvalue() + print 'running the following command on the remote server:' + print command + t = self.ssh_client.run(command) + print '\t%s' % t[0] + print '\t%s' % t[1] + print '...complete!' + print 'registering image...' + self.image_id = self.server.ec2.register_image('%s/%s.manifest.xml' % (bucket, prefix)) + return self.image_id + +class CommandLineGetter(object): + + def get_ami_list(self): + my_amis = [] + for ami in self.ec2.get_all_images(): + # hack alert, need a better way to do this! + if ami.location.find('pyami') >= 0: + my_amis.append((ami.location, ami)) + return my_amis + + def get_region(self, params): + region = params.get('region', None) + if isinstance(region, str) or isinstance(region, unicode): + region = boto.ec2.get_region(region) + params['region'] = region + if not region: + prop = self.cls.find_property('region_name') + params['region'] = propget.get(prop, choices=boto.ec2.regions) + + def get_name(self, params): + if not params.get('name', None): + prop = self.cls.find_property('name') + params['name'] = propget.get(prop) + + def get_description(self, params): + if not params.get('description', None): + prop = self.cls.find_property('description') + params['description'] = propget.get(prop) + + def get_instance_type(self, params): + if not params.get('instance_type', None): + prop = StringProperty(name='instance_type', verbose_name='Instance Type', + choices=InstanceTypes) + params['instance_type'] = propget.get(prop) + + def get_quantity(self, params): + if not params.get('quantity', None): + prop = IntegerProperty(name='quantity', verbose_name='Number of Instances') + params['quantity'] = propget.get(prop) + + def get_zone(self, params): + if not params.get('zone', None): + prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', + choices=self.ec2.get_all_zones) + params['zone'] = propget.get(prop) + + def get_ami_id(self, params): + ami = params.get('ami', None) + if isinstance(ami, str) or isinstance(ami, unicode): + ami_list = self.get_ami_list() + for l,a in ami_list: + if a.id == ami: + ami = a + params['ami'] = a + if not params.get('ami', None): + prop = StringProperty(name='ami', verbose_name='AMI', + choices=self.get_ami_list) + params['ami'] = propget.get(prop) + + def get_group(self, params): + group = params.get('group', None) + if isinstance(group, str) or isinstance(group, unicode): + group_list = self.ec2.get_all_security_groups() + for g in group_list: + if g.name == group: + group = g + params['group'] = g + if not group: + prop = StringProperty(name='group', verbose_name='EC2 Security Group', + choices=self.ec2.get_all_security_groups) + params['group'] = propget.get(prop) + + def get_key(self, params): + keypair = params.get('keypair', None) + if isinstance(keypair, str) or isinstance(keypair, unicode): + key_list = self.ec2.get_all_key_pairs() + for k in key_list: + if k.name == keypair: + keypair = k.name + params['keypair'] = k.name + if not keypair: + prop = StringProperty(name='keypair', verbose_name='EC2 KeyPair', + choices=self.ec2.get_all_key_pairs) + params['keypair'] = propget.get(prop).name + + def get(self, cls, params): + self.cls = cls + self.get_region(params) + self.ec2 = params['region'].connect() + self.get_name(params) + self.get_description(params) + self.get_instance_type(params) + self.get_zone(params) + self.get_quantity(params) + self.get_ami_id(params) + self.get_group(params) + self.get_key(params) + +class Server(Model): + + # + # The properties of this object consists of real properties for data that + # is not already stored in EC2 somewhere (e.g. name, description) plus + # calculated properties for all of the properties that are already in + # EC2 (e.g. hostname, security groups, etc.) + # + name = StringProperty(unique=True, verbose_name="Name") + description = StringProperty(verbose_name="Description") + region_name = StringProperty(verbose_name="EC2 Region Name") + instance_id = StringProperty(verbose_name="EC2 Instance ID") + elastic_ip = StringProperty(verbose_name="EC2 Elastic IP Address") + production = BooleanProperty(verbose_name="Is This Server Production", default=False) + ami_id = CalculatedProperty(verbose_name="AMI ID", calculated_type=str, use_method=True) + zone = CalculatedProperty(verbose_name="Availability Zone Name", calculated_type=str, use_method=True) + hostname = CalculatedProperty(verbose_name="Public DNS Name", calculated_type=str, use_method=True) + private_hostname = CalculatedProperty(verbose_name="Private DNS Name", calculated_type=str, use_method=True) + groups = CalculatedProperty(verbose_name="Security Groups", calculated_type=list, use_method=True) + security_group = CalculatedProperty(verbose_name="Primary Security Group Name", calculated_type=str, use_method=True) + key_name = CalculatedProperty(verbose_name="Key Name", calculated_type=str, use_method=True) + instance_type = CalculatedProperty(verbose_name="Instance Type", calculated_type=str, use_method=True) + status = CalculatedProperty(verbose_name="Current Status", calculated_type=str, use_method=True) + launch_time = CalculatedProperty(verbose_name="Server Launch Time", calculated_type=str, use_method=True) + console_output = CalculatedProperty(verbose_name="Console Output", calculated_type=file, use_method=True) + + packages = [] + plugins = [] + + @classmethod + def add_credentials(cls, cfg, aws_access_key_id, aws_secret_access_key): + if not cfg.has_section('Credentials'): + cfg.add_section('Credentials') + cfg.set('Credentials', 'aws_access_key_id', aws_access_key_id) + cfg.set('Credentials', 'aws_secret_access_key', aws_secret_access_key) + if not cfg.has_section('DB_Server'): + cfg.add_section('DB_Server') + cfg.set('DB_Server', 'db_type', 'SimpleDB') + cfg.set('DB_Server', 'db_name', cls._manager.domain.name) + + ''' + Create a new instance based on the specified configuration file or the specified + configuration and the passed in parameters. + + If the config_file argument is not None, the configuration is read from there. + Otherwise, the cfg argument is used. + + The config file may include other config files with a #import reference. The included + config files must reside in the same directory as the specified file. + + The logical_volume argument, if supplied, will be used to get the current physical + volume ID and use that as an override of the value specified in the config file. This + may be useful for debugging purposes when you want to debug with a production config + file but a test Volume. + + The dictionary argument may be used to override any EC2 configuration values in the + config file. + ''' + @classmethod + def create(cls, config_file=None, logical_volume = None, cfg = None, **params): + if config_file: + cfg = Config(path=config_file) + if cfg.has_section('EC2'): + # include any EC2 configuration values that aren't specified in params: + for option in cfg.options('EC2'): + if option not in params: + params[option] = cfg.get('EC2', option) + getter = CommandLineGetter() + getter.get(cls, params) + region = params.get('region') + ec2 = region.connect() + cls.add_credentials(cfg, ec2.aws_access_key_id, ec2.aws_secret_access_key) + ami = params.get('ami') + kp = params.get('keypair') + group = params.get('group') + zone = params.get('zone') + # deal with possibly passed in logical volume: + if logical_volume != None: + cfg.set('EBS', 'logical_volume_name', logical_volume.name) + cfg_fp = StringIO.StringIO() + cfg.write(cfg_fp) + # deal with the possibility that zone and/or keypair are strings read from the config file: + if isinstance(zone, Zone): + zone = zone.name + if isinstance(kp, KeyPair): + kp = kp.name + reservation = ami.run(min_count=1, + max_count=params.get('quantity', 1), + key_name=kp, + security_groups=[group], + instance_type=params.get('instance_type'), + placement = zone, + user_data = cfg_fp.getvalue()) + l = [] + i = 0 + elastic_ip = params.get('elastic_ip') + instances = reservation.instances + if elastic_ip != None and instances.__len__() > 0: + instance = instances[0] + while instance.update() != 'running': + time.sleep(1) + instance.use_ip(elastic_ip) + print 'set the elastic IP of the first instance to %s' % elastic_ip + for instance in instances: + s = cls() + s.ec2 = ec2 + s.name = params.get('name') + '' if i==0 else str(i) + s.description = params.get('description') + s.region_name = region.name + s.instance_id = instance.id + if elastic_ip and i == 0: + s.elastic_ip = elastic_ip + s.put() + l.append(s) + i += 1 + return l + + @classmethod + def create_from_instance_id(cls, instance_id, name, description=''): + regions = boto.ec2.regions() + for region in regions: + ec2 = region.connect() + try: + rs = ec2.get_all_instances([instance_id]) + except: + rs = [] + if len(rs) == 1: + s = cls() + s.ec2 = ec2 + s.name = name + s.description = description + s.region_name = region.name + s.instance_id = instance_id + s._reservation = rs[0] + for instance in s._reservation.instances: + if instance.id == instance_id: + s._instance = instance + s.put() + return s + return None + + @classmethod + def create_from_current_instances(cls): + servers = [] + regions = boto.ec2.regions() + for region in regions: + ec2 = region.connect() + rs = ec2.get_all_instances() + for reservation in rs: + for instance in reservation.instances: + try: + Server.find(instance_id=instance.id).next() + boto.log.info('Server for %s already exists' % instance.id) + except StopIteration: + s = cls() + s.ec2 = ec2 + s.name = instance.id + s.region_name = region.name + s.instance_id = instance.id + s._reservation = reservation + s.put() + servers.append(s) + return servers + + def __init__(self, id=None, **kw): + Model.__init__(self, id, **kw) + self.ssh_key_file = None + self.ec2 = None + self._cmdshell = None + self._reservation = None + self._instance = None + self._setup_ec2() + + def _setup_ec2(self): + if self.ec2 and self._instance and self._reservation: + return + if self.id: + if self.region_name: + for region in boto.ec2.regions(): + if region.name == self.region_name: + self.ec2 = region.connect() + if self.instance_id and not self._instance: + try: + rs = self.ec2.get_all_instances([self.instance_id]) + if len(rs) >= 1: + for instance in rs[0].instances: + if instance.id == self.instance_id: + self._reservation = rs[0] + self._instance = instance + except EC2ResponseError: + pass + + def _status(self): + status = '' + if self._instance: + self._instance.update() + status = self._instance.state + return status + + def _hostname(self): + hostname = '' + if self._instance: + hostname = self._instance.public_dns_name + return hostname + + def _private_hostname(self): + hostname = '' + if self._instance: + hostname = self._instance.private_dns_name + return hostname + + def _instance_type(self): + it = '' + if self._instance: + it = self._instance.instance_type + return it + + def _launch_time(self): + lt = '' + if self._instance: + lt = self._instance.launch_time + return lt + + def _console_output(self): + co = '' + if self._instance: + co = self._instance.get_console_output() + return co + + def _groups(self): + gn = [] + if self._reservation: + gn = self._reservation.groups + return gn + + def _security_group(self): + groups = self._groups() + if len(groups) >= 1: + return groups[0].id + return "" + + def _zone(self): + zone = None + if self._instance: + zone = self._instance.placement + return zone + + def _key_name(self): + kn = None + if self._instance: + kn = self._instance.key_name + return kn + + def put(self): + Model.put(self) + self._setup_ec2() + + def delete(self): + if self.production: + raise ValueError, "Can't delete a production server" + #self.stop() + Model.delete(self) + + def stop(self): + if self.production: + raise ValueError, "Can't delete a production server" + if self._instance: + self._instance.stop() + + def reboot(self): + if self._instance: + self._instance.reboot() + + def wait(self): + while self.status != 'running': + time.sleep(5) + + def get_ssh_key_file(self): + if not self.ssh_key_file: + ssh_dir = os.path.expanduser('~/.ssh') + if os.path.isdir(ssh_dir): + ssh_file = os.path.join(ssh_dir, '%s.pem' % self.key_name) + if os.path.isfile(ssh_file): + self.ssh_key_file = ssh_file + if not self.ssh_key_file: + iobject = IObject() + self.ssh_key_file = iobject.get_filename('Path to OpenSSH Key file') + return self.ssh_key_file + + def get_cmdshell(self): + if not self._cmdshell: + import cmdshell + self.get_ssh_key_file() + self._cmdshell = cmdshell.start(self) + return self._cmdshell + + def reset_cmdshell(self): + self._cmdshell = None + + def run(self, command): + with closing(self.get_cmdshell()) as cmd: + status = cmd.run(command) + return status + + def get_bundler(self, uname='root'): + ssh_key_file = self.get_ssh_key_file() + return Bundler(self, uname) + + def get_ssh_client(self, uname='root'): + from boto.manage.cmdshell import SSHClient + ssh_key_file = self.get_ssh_key_file() + return SSHClient(self, uname=uname) + + def install(self, pkg): + return self.run('apt-get -y install %s' % pkg) + + + diff --git a/storefront/boto/manage/task.py b/storefront/boto/manage/task.py new file mode 100644 index 0000000..5fb234d --- /dev/null +++ b/storefront/boto/manage/task.py @@ -0,0 +1,173 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto +from boto.sdb.db.property import * +from boto.sdb.db.model import Model +import datetime, subprocess, StringIO, time + +def check_hour(val): + if val == '*': + return + if int(val) < 0 or int(val) > 23: + raise ValueError + +class Task(Model): + + """ + A scheduled, repeating task that can be executed by any participating servers. + The scheduling is similar to cron jobs. Each task has an hour attribute. + The allowable values for hour are [0-23|*]. + + To keep the operation reasonably efficient and not cause excessive polling, + the minimum granularity of a Task is hourly. Some examples: + + hour='*' - the task would be executed each hour + hour='3' - the task would be executed at 3AM GMT each day. + + """ + name = StringProperty() + hour = StringProperty(required=True, validator=check_hour, default='*') + command = StringProperty(required=True) + last_executed = DateTimeProperty() + last_status = IntegerProperty() + last_output = StringProperty() + message_id = StringProperty() + + @classmethod + def start_all(cls, queue_name): + for task in cls.all(): + task.start(queue_name) + + def __init__(self, id=None, **kw): + Model.__init__(self, id, **kw) + self.hourly = self.hour == '*' + self.daily = self.hour != '*' + self.now = datetime.datetime.utcnow() + + def check(self): + """ + Determine how long until the next scheduled time for a Task. + Returns the number of seconds until the next scheduled time or zero + if the task needs to be run immediately. + If it's an hourly task and it's never been run, run it now. + If it's a daily task and it's never been run and the hour is right, run it now. + """ + need_to_run = False + boto.log.info('checking Task[%s]-now=%s, last=%s' % (self.name, self.now, self.last_executed)) + + if self.hourly and not self.last_executed: + return 0 + + if self.daily and not self.last_executed: + if int(self.hour) == self.now.hour: + return 0 + else: + return max((int(self.hour) - self.now.hour),0)*60*60 + + delta = self.now - self.last_executed + if self.hourly: + if delta.seconds >= 60*60: + return 0 + else: + return 60*60 - delta.seconds + else: + if delta.days >= 1: + return 0 + else: + return min(60*60*24-delta.seconds, 43200) + + def _run(self, msg, vtimeout): + boto.log.info('Task[%s] - running:%s' % (self.name, self.command)) + log_fp = StringIO.StringIO() + process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + nsecs = 5 + current_timeout = vtimeout + while process.poll() == None: + boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout)) + if nsecs >= current_timeout: + current_timeout += vtimeout + boto.log.info('Task[%s] - setting timeout to %d seconds' % (self.name, current_timeout)) + if msg: + msg.change_visibility(current_timeout) + time.sleep(5) + nsecs += 5 + t = process.communicate() + log_fp.write(t[0]) + log_fp.write(t[1]) + boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue())) + self.last_executed = self.now + self.last_status = process.returncode + self.last_output = log_fp.getvalue()[0:1023] + + def run(self, msg, vtimeout=60): + delay = self.check() + boto.log.info('Task[%s] - delay=%s seconds' % (self.name, delay)) + if delay == 0: + self._run(msg, vtimeout) + queue = msg.queue + new_msg = queue.new_message(self.id) + new_msg = queue.write(msg) + self.message_id = new_msg.id + self.put() + boto.log.info('Task[%s] - new message id=%s' % (self.name, new_msg.id)) + msg.delete() + boto.log.info('Task[%s] - deleted message %s' % (self.name, msg.id)) + else: + boto.log.info('new_vtimeout: %d' % delay) + msg.change_visibility(delay) + + def start(self, queue_name): + boto.log.info('Task[%s] - starting with queue: %s' % (self.name, queue_name)) + queue = boto.lookup('sqs', queue_name) + msg = queue.new_message(self.id) + msg = queue.write(msg) + self.message_id = msg.id + self.put() + boto.log.info('Task[%s] - start successful' % self.name) + +class TaskPoller(object): + + def __init__(self, queue_name): + self.sqs = boto.connect_sqs() + self.queue = self.sqs.lookup(queue_name) + + def poll(self, wait=60, vtimeout=60): + while 1: + m = self.queue.read(vtimeout) + if m: + task = Task.get_by_id(m.get_body()) + if task: + if not task.message_id or m.id == task.message_id: + boto.log.info('Task[%s] - read message %s' % (task.name, m.id)) + task.run(m, vtimeout) + else: + boto.log.info('Task[%s] - found extraneous message, ignoring' % task.name) + else: + time.sleep(wait) + + + + + + diff --git a/storefront/boto/manage/test_manage.py b/storefront/boto/manage/test_manage.py new file mode 100644 index 0000000..e0b032a --- /dev/null +++ b/storefront/boto/manage/test_manage.py @@ -0,0 +1,34 @@ +from boto.manage.server import Server +from boto.manage.volume import Volume +import time + +print '--> Creating New Volume' +volume = Volume.create() +print volume + +print '--> Creating New Server' +server_list = Server.create() +server = server_list[0] +print server + +print '----> Waiting for Server to start up' +while server.status != 'running': + print '*' + time.sleep(10) +print '----> Server is running' + +print '--> Run "df -k" on Server' +status = server.run('df -k') +print status[1] + +print '--> Now run volume.make_ready to make the volume ready to use on server' +volume.make_ready(server) + +print '--> Run "df -k" on Server' +status = server.run('df -k') +print status[1] + +print '--> Do an "ls -al" on the new filesystem' +status = server.run('ls -al %s' % volume.mount_point) +print status[1] + diff --git a/storefront/boto/manage/volume.py b/storefront/boto/manage/volume.py new file mode 100644 index 0000000..bed5594 --- /dev/null +++ b/storefront/boto/manage/volume.py @@ -0,0 +1,417 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from __future__ import with_statement +from boto.sdb.db.model import Model +from boto.sdb.db.property import * +from boto.manage.server import Server +from boto.manage import propget +import boto.ec2 +import time, traceback +from contextlib import closing +import dateutil.parser + +class CommandLineGetter(object): + + def get_region(self, params): + if not params.get('region', None): + prop = self.cls.find_property('region_name') + params['region'] = propget.get(prop, choices=boto.ec2.regions) + + def get_zone(self, params): + if not params.get('zone', None): + prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', + choices=self.ec2.get_all_zones) + params['zone'] = propget.get(prop) + + def get_name(self, params): + if not params.get('name', None): + prop = self.cls.find_property('name') + params['name'] = propget.get(prop) + + def get_size(self, params): + if not params.get('size', None): + prop = IntegerProperty(name='size', verbose_name='Size (GB)') + params['size'] = propget.get(prop) + + def get_mount_point(self, params): + if not params.get('mount_point', None): + prop = self.cls.find_property('mount_point') + params['mount_point'] = propget.get(prop) + + def get_device(self, params): + if not params.get('device', None): + prop = self.cls.find_property('device') + params['device'] = propget.get(prop) + + def get(self, cls, params): + self.cls = cls + self.get_region(params) + self.ec2 = params['region'].connect() + self.get_zone(params) + self.get_name(params) + self.get_size(params) + self.get_mount_point(params) + self.get_device(params) + +class Volume(Model): + + name = StringProperty(required=True, unique=True, verbose_name='Name') + region_name = StringProperty(required=True, verbose_name='EC2 Region') + zone_name = StringProperty(required=True, verbose_name='EC2 Zone') + mount_point = StringProperty(verbose_name='Mount Point') + device = StringProperty(verbose_name="Device Name", default='/dev/sdp') + volume_id = StringProperty(required=True) + past_volume_ids = ListProperty(item_type=str) + server = ReferenceProperty(Server, collection_name='volumes', + verbose_name='Server Attached To') + volume_state = CalculatedProperty(verbose_name="Volume State", + calculated_type=str, use_method=True) + attachment_state = CalculatedProperty(verbose_name="Attachment State", + calculated_type=str, use_method=True) + size = CalculatedProperty(verbose_name="Size (GB)", + calculated_type=int, use_method=True) + + @classmethod + def create(cls, **params): + getter = CommandLineGetter() + getter.get(cls, params) + region = params.get('region') + ec2 = region.connect() + zone = params.get('zone') + size = params.get('size') + ebs_volume = ec2.create_volume(size, zone.name) + v = cls() + v.ec2 = ec2 + v.volume_id = ebs_volume.id + v.name = params.get('name') + v.mount_point = params.get('mount_point') + v.device = params.get('device') + v.region_name = region.name + v.zone_name = zone.name + v.put() + return v + + @classmethod + def create_from_volume_id(cls, region_name, volume_id, name): + vol = None + ec2 = boto.ec2.connect_to_region(region_name) + rs = ec2.get_all_volumes([volume_id]) + if len(rs) == 1: + v = rs[0] + vol = cls() + vol.volume_id = v.id + vol.name = name + vol.region_name = v.region.name + vol.zone_name = v.zone + vol.put() + return vol + + def create_from_latest_snapshot(self, name, size=None): + snapshot = self.get_snapshots()[-1] + return self.create_from_snapshot(name, snapshot, size) + + def create_from_snapshot(self, name, snapshot, size=None): + if size < self.size: + size = self.size + ec2 = self.get_ec2_connection() + if self.zone_name == None or self.zone_name == '': + # deal with the migration case where the zone is not set in the logical volume: + current_volume = ec2.get_all_volumes([self.volume_id])[0] + self.zone_name = current_volume.zone + ebs_volume = ec2.create_volume(size, self.zone_name, snapshot) + v = Volume() + v.ec2 = self.ec2 + v.volume_id = ebs_volume.id + v.name = name + v.mount_point = self.mount_point + v.device = self.device + v.region_name = self.region_name + v.zone_name = self.zone_name + v.put() + return v + + def get_ec2_connection(self): + if self.server: + return self.server.ec2 + if not hasattr(self, 'ec2') or self.ec2 == None: + self.ec2 = boto.ec2.connect_to_region(self.region_name) + return self.ec2 + + def _volume_state(self): + ec2 = self.get_ec2_connection() + rs = ec2.get_all_volumes([self.volume_id]) + return rs[0].volume_state() + + def _attachment_state(self): + ec2 = self.get_ec2_connection() + rs = ec2.get_all_volumes([self.volume_id]) + return rs[0].attachment_state() + + def _size(self): + if not hasattr(self, '__size'): + ec2 = self.get_ec2_connection() + rs = ec2.get_all_volumes([self.volume_id]) + self.__size = rs[0].size + return self.__size + + def install_xfs(self): + if self.server: + self.server.install('xfsprogs xfsdump') + + def get_snapshots(self): + """ + Returns a list of all completed snapshots for this volume ID. + """ + ec2 = self.get_ec2_connection() + rs = ec2.get_all_snapshots() + all_vols = [self.volume_id] + self.past_volume_ids + snaps = [] + for snapshot in rs: + if snapshot.volume_id in all_vols: + if snapshot.progress == '100%': + snapshot.date = dateutil.parser.parse(snapshot.start_time) + snapshot.keep = True + snaps.append(snapshot) + snaps.sort(cmp=lambda x,y: cmp(x.date, y.date)) + return snaps + + def attach(self, server=None): + if self.attachment_state == 'attached': + print 'already attached' + return None + if server: + self.server = server + self.put() + ec2 = self.get_ec2_connection() + ec2.attach_volume(self.volume_id, self.server.instance_id, self.device) + + def detach(self, force=False): + state = self.attachment_state + if state == 'available' or state == None or state == 'detaching': + print 'already detached' + return None + ec2 = self.get_ec2_connection() + ec2.detach_volume(self.volume_id, self.server.instance_id, self.device, force) + self.server = None + self.put() + + def checkfs(self, use_cmd=None): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + # detemine state of file system on volume, only works if attached + if use_cmd: + cmd = use_cmd + else: + cmd = self.server.get_cmdshell() + status = cmd.run('xfs_check %s' % self.device) + if not use_cmd: + cmd.close() + if status[1].startswith('bad superblock magic number 0'): + return False + return True + + def wait(self): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + with closing(self.server.get_cmdshell()) as cmd: + # wait for the volume device to appear + cmd = self.server.get_cmdshell() + while not cmd.exists(self.device): + boto.log.info('%s still does not exist, waiting 10 seconds' % self.device) + time.sleep(10) + + def format(self): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + status = None + with closing(self.server.get_cmdshell()) as cmd: + if not self.checkfs(cmd): + boto.log.info('make_fs...') + status = cmd.run('mkfs -t xfs %s' % self.device) + return status + + def mount(self): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + boto.log.info('handle_mount_point') + with closing(self.server.get_cmdshell()) as cmd: + cmd = self.server.get_cmdshell() + if not cmd.isdir(self.mount_point): + boto.log.info('making directory') + # mount directory doesn't exist so create it + cmd.run("mkdir %s" % self.mount_point) + else: + boto.log.info('directory exists already') + status = cmd.run('mount -l') + lines = status[1].split('\n') + for line in lines: + t = line.split() + if t and t[2] == self.mount_point: + # something is already mounted at the mount point + # unmount that and mount it as /tmp + if t[0] != self.device: + cmd.run('umount %s' % self.mount_point) + cmd.run('mount %s /tmp' % t[0]) + cmd.run('chmod 777 /tmp') + break + # Mount up our new EBS volume onto mount_point + cmd.run("mount %s %s" % (self.device, self.mount_point)) + cmd.run('xfs_growfs %s' % self.mount_point) + + def make_ready(self, server): + self.server = server + self.put() + self.install_xfs() + self.attach() + self.wait() + self.format() + self.mount() + + def freeze(self): + if self.server: + return self.server.run("/usr/sbin/xfs_freeze -f %s" % self.mount_point) + + def unfreeze(self): + if self.server: + return self.server.run("/usr/sbin/xfs_freeze -u %s" % self.mount_point) + + def snapshot(self): + # if this volume is attached to a server + # we need to freeze the XFS file system + try: + self.freeze() + if self.server == None: + snapshot = self.get_ec2_connection().create_snapshot(self.volume_id) + else: + snapshot = self.server.ec2.create_snapshot(self.volume_id) + boto.log.info('Snapshot of Volume %s created: %s' % (self.name, snapshot)) + except Exception, e: + boto.log.info('Snapshot error') + boto.log.info(traceback.format_exc()) + finally: + status = self.unfreeze() + return status + + def get_snapshot_range(self, snaps, start_date=None, end_date=None): + l = [] + for snap in snaps: + if start_date and end_date: + if snap.date >= start_date and snap.date <= end_date: + l.append(snap) + elif start_date: + if snap.date >= start_date: + l.append(snap) + elif end_date: + if snap.date <= end_date: + l.append(snap) + else: + l.append(snap) + return l + + def trim_snapshots(self, delete=False): + """ + Trim the number of snapshots for this volume. This method always + keeps the oldest snapshot. It then uses the parameters passed in + to determine how many others should be kept. + + The algorithm is to keep all snapshots from the current day. Then + it will keep the first snapshot of the day for the previous seven days. + Then, it will keep the first snapshot of the week for the previous + four weeks. After than, it will keep the first snapshot of the month + for as many months as there are. + + """ + snaps = self.get_snapshots() + # Always keep the oldest and the newest + if len(snaps) <= 2: + return snaps + snaps = snaps[1:-1] + now = datetime.datetime.now(snaps[0].date.tzinfo) + midnight = datetime.datetime(year=now.year, month=now.month, + day=now.day, tzinfo=now.tzinfo) + # Keep the first snapshot from each day of the previous week + one_week = datetime.timedelta(days=7, seconds=60*60) + print midnight-one_week, midnight + previous_week = self.get_snapshot_range(snaps, midnight-one_week, midnight) + print previous_week + if not previous_week: + return snaps + current_day = None + for snap in previous_week: + if current_day and current_day == snap.date.day: + snap.keep = False + else: + current_day = snap.date.day + # Get ourselves onto the next full week boundary + if previous_week: + week_boundary = previous_week[0].date + if week_boundary.weekday() != 0: + delta = datetime.timedelta(days=week_boundary.weekday()) + week_boundary = week_boundary - delta + # Keep one within this partial week + partial_week = self.get_snapshot_range(snaps, week_boundary, previous_week[0].date) + if len(partial_week) > 1: + for snap in partial_week[1:]: + snap.keep = False + # Keep the first snapshot of each week for the previous 4 weeks + for i in range(0,4): + weeks_worth = self.get_snapshot_range(snaps, week_boundary-one_week, week_boundary) + if len(weeks_worth) > 1: + for snap in weeks_worth[1:]: + snap.keep = False + week_boundary = week_boundary - one_week + # Now look through all remaining snaps and keep one per month + remainder = self.get_snapshot_range(snaps, end_date=week_boundary) + current_month = None + for snap in remainder: + if current_month and current_month == snap.date.month: + snap.keep = False + else: + current_month = snap.date.month + if delete: + for snap in snaps: + if not snap.keep: + boto.log.info('Deleting %s(%s) for %s' % (snap, snap.date, self.name)) + snap.delete() + return snaps + + def grow(self, size): + pass + + def copy(self, snapshot): + pass + + def get_snapshot_from_date(self, date): + pass + + def delete(self, delete_ebs_volume=False): + if delete_ebs_volume: + self.detach() + ec2 = self.get_ec2_connection() + ec2.delete_volume(self.volume_id) + Model.delete(self) + + def archive(self): + # snapshot volume, trim snaps, delete volume-id + pass + + diff --git a/storefront/boto/mapreduce/__init__.py b/storefront/boto/mapreduce/__init__.py new file mode 100644 index 0000000..ac3ddc4 --- /dev/null +++ b/storefront/boto/mapreduce/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/storefront/boto/mapreduce/lqs.py b/storefront/boto/mapreduce/lqs.py new file mode 100644 index 0000000..fc76e50 --- /dev/null +++ b/storefront/boto/mapreduce/lqs.py @@ -0,0 +1,152 @@ +import SocketServer, os, datetime, sys, random, time +import simplejson + +class LQSCommand: + + def __init__(self, line): + self.raw_line = line + self.line = self.raw_line.strip() + l = self.line.split(' ') + self.name = l[0] + if len(l) > 1: + self.args = [arg for arg in l[1:] if arg] + else: + self.args = [] + +class LQSMessage(dict): + + def __init__(self, item=None, args=None, jsonvalue=None): + dict.__init__(self) + if jsonvalue: + self.decode(jsonvalue) + else: + self['id'] = '%d_%d' % (int(time.time()), int(random.random()*1000000)) + self['item'] = item + self['args'] = args + + def encode(self): + return simplejson.dumps(self) + + def decode(self, value): + self.update(simplejson.loads(value)) + + def is_empty(self): + if self['item'] == None: + return True + return False + +class LQSServer(SocketServer.UDPServer): + + PORT = 5151 + TIMEOUT = 30 + MAXSIZE = 8192 + + def __init__(self, server_address, RequestHandlerClass, iterator, args=None): + server_address = (server_address, self.PORT) + SocketServer.UDPServer.__init__(self, server_address, RequestHandlerClass) + self.count = 0 + self.iterator = iterator + self.args = args + self.start = datetime.datetime.now() + self.end = None + self.extant = [] + +class LQSHandler(SocketServer.DatagramRequestHandler): + + def get_cmd(self): + return LQSCommand(self.rfile.readline()) + + def build_msg(self): + if not self.server.iterator: + return LQSMessage(None) + try: + item = self.server.iterator.next() + msg = LQSMessage(item, self.server.args) + return msg + except StopIteration: + self.server.iterator = None + return LQSMessage(None) + + def respond(self, msg): + self.wfile.write(msg.encode()) + + def check_extant(self): + if len(self.server.extant) == 0 and not self.server.iterator: + self.server.end = datetime.datetime.now() + delta = self.server.end - self.server.start + print 'Total Processing Time: %s' % delta + print 'Total Messages Processed: %d' % self.server.count + + def do_debug(self, cmd): + args = {'extant' : self.server.extant, + 'count' : self.server.count} + msg = LQSMessage('debug', args) + self.respond(msg) + + def do_next(self, cmd): + out_msg = self.build_msg() + if not out_msg.is_empty(): + self.server.count += 1 + self.server.extant.append(out_msg['id']) + self.respond(out_msg) + + def do_delete(self, cmd): + if len(cmd.args) != 1: + self.error(cmd, 'delete command requires message id') + else: + mid = cmd.args[0] + try: + self.server.extant.remove(mid) + except ValueError: + self.error(cmd, 'message id not found') + args = {'deleted' : True} + msg = LQSMessage(mid, args) + self.respond(msg) + self.check_extant() + + def error(self, cmd, error_msg=None): + args = {'error_msg' : error_msg, + 'cmd_name' : cmd.name, + 'cmd_args' : cmd.args} + msg = LQSMessage('error', args) + self.respond(msg) + + def do_stop(self, cmd): + sys.exit(0) + + def handle(self): + cmd = self.get_cmd() + if hasattr(self, 'do_%s' % cmd.name): + method = getattr(self, 'do_%s' % cmd.name) + method(cmd) + else: + self.error(cmd, 'unrecognized command') + +class PersistHandler(LQSHandler): + + def build_msg(self): + if not self.server.iterator: + return LQSMessage(None) + try: + obj = self.server.iterator.next() + msg = LQSMessage(obj.id, self.server.args) + return msg + except StopIteration: + self.server.iterator = None + return LQSMessage(None) + +def test_file(path, args=None): + l = os.listdir(path) + if not args: + args = {} + args['path'] = path + s = LQSServer('', LQSHandler, iter(l), args) + print "Awaiting UDP messages on port %d" % s.PORT + s.serve_forever() + +def test_simple(n): + l = range(0, n) + s = LQSServer('', LQSHandler, iter(l), None) + print "Awaiting UDP messages on port %d" % s.PORT + s.serve_forever() + diff --git a/storefront/boto/mapreduce/partitiondb.py b/storefront/boto/mapreduce/partitiondb.py new file mode 100644 index 0000000..c5b0475 --- /dev/null +++ b/storefront/boto/mapreduce/partitiondb.py @@ -0,0 +1,172 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import random, time, os, datetime +import boto +from boto.sdb.persist.object import SDBObject +from boto.sdb.persist.property import * + +class Identifier(object): + + _hex_digits = '0123456789abcdef' + + @classmethod + def gen(cls, prefix): + suffix = '' + for i in range(0,8): + suffix += random.choice(cls._hex_digits) + return ts + '-' + suffix + +class Version(SDBObject): + + name = StringProperty() + pdb = ObjectProperty(ref_class=SDBObject) + date = DateTimeProperty() + + def __init__(self, id=None, manager=None): + SDBObject.__init__(self, id, manager) + if id == None: + self.name = Identifier.gen('v') + self.date = datetime.datetime.now() + print 'created Version %s' % self.name + + def partitions(self): + """ + Return an iterator containing all Partition objects related to this Version. + + :rtype: iterator of :class:`boto.mapreduce.partitiondb.Partition` + :return: The Partitions in this Version + """ + return self.get_related_objects('version', Partition) + + def add_partition(self, name=None): + """ + Add a new Partition to this Version. + + :type name: string + :param name: The name of the new Partition (optional) + + :rtype: :class:`boto.mapreduce.partitiondb.Partition` + :return: The new Partition object + """ + p = Partition(manager=self.manager, name=name) + p.version = self + p.pdb = self.pdb + p.save() + return p + + def get_s3_prefix(self): + if not self.pdb: + raise ValueError, 'pdb attribute must be set to compute S3 prefix' + return self.pdb.get_s3_prefix() + self.name + '/' + +class PartitionDB(SDBObject): + + name = StringProperty() + bucket_name = StringProperty() + versions = ObjectListProperty(ref_class=Version) + + def __init__(self, id=None, manager=None, name='', bucket_name=''): + SDBObject.__init__(self, id, manager) + if id == None: + self.name = name + self.bucket_name = bucket_name + + def get_s3_prefix(self): + return self.name + '/' + + def add_version(self): + """ + Add a new Version to this PartitionDB. The newly added version becomes the + current version. + + :rtype: :class:`boto.mapreduce.partitiondb.Version` + :return: The newly created Version object. + """ + v = Version() + v.pdb = self + v.save() + self.versions.append(v) + return v + + def revert(self): + """ + Revert to the previous version of this PartitionDB. The current version is removed from the + list of Versions and the Version immediately preceeding it becomes the current version. + Note that this method does not delete the Version object or any Partitions related to the + Version object. + + :rtype: :class:`boto.mapreduce.partitiondb.Version` + :return: The previous current Version object. + """ + v = self.current_version() + if v: + self.versions.remove(v) + return v + + def current_version(self): + """ + Get the currently active Version of this PartitionDB object. + + :rtype: :class:`boto.mapreduce.partitiondb.Version` + :return: The current Version object or None if there are no Versions associated + with this PartitionDB object. + """ + if self.versions: + if len(self.versions) > 0: + return self.versions[-1] + return None + +class Partition(SDBObject): + + def __init__(self, id=None, manager=None, name=None): + SDBObject.__init__(self, id, manager) + if id == None: + self.name = name + + name = StringProperty() + version = ObjectProperty(ref_class=Version) + pdb = ObjectProperty(ref_class=PartitionDB) + data = S3KeyProperty() + + def get_key_name(self): + return self.version.get_s3_prefix() + self.name + + def upload(self, path, bucket_name=None): + if not bucket_name: + bucket_name = self.version.pdb.bucket_name + s3 = self.manager.get_s3_connection() + bucket = s3.lookup(bucket_name) + directory, filename = os.path.split(path) + self.name = filename + key = bucket.new_key(self.get_key_name()) + key.set_contents_from_filename(path) + self.data = key + self.save() + + def delete(self): + if self.data: + self.data.delete() + SDBObject.delete(self) + + + diff --git a/storefront/boto/mapreduce/pdb_delete b/storefront/boto/mapreduce/pdb_delete new file mode 100644 index 0000000..b7af9cc --- /dev/null +++ b/storefront/boto/mapreduce/pdb_delete @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import queuetools, os, signal, sys +import subprocess +import time +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from lqs import LQSServer, PersistHandler +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager, revive_object_from_id + +USAGE = """ + SYNOPSIS + %prog [options] [command] + DESCRIPTION + Delete a PartitionDB and all related data in SimpleDB and S3. +""" +class Client: + + def __init__(self, queue_name): + self.q = queuetools.get_queue(queue_name) + self.q.connect() + self.manager = get_manager() + self.process() + + def process(self): + m = self.q.get() + while m['item']: + print 'Deleting: %s' % m['item'] + obj = revive_object_from_id(m['item'], manager=self.manager) + obj.delete() + self.q.delete(m) + m = self.q.get() + print 'client processing complete' + +class Server: + + def __init__(self, pdb_name, domain_name=None): + self.pdb_name = pdb_name + self.manager = get_manager(domain_name) + self.pdb = PartitionDB.get(name=self.pdb_name) + self.serve() + + def serve(self): + args = {'pdb_id' : self.pdb.id} + rs = self.pdb.get_related_objects('pdb') + self.pdb.delete() + s = LQSServer('', PersistHandler, rs, args) + s.serve_forever() + +class Delete: + + Commands = {'client' : 'Start a Delete client', + 'server' : 'Start a Delete server'} + + def __init__(self): + self.parser = OptionParser(usage=USAGE) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-processes', action='store', type='int', dest='num_processes', + help='the number of client processes launched') + self.parser.set_defaults(num_processes=5) + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB in which to store files (will create if necessary)') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_server(self): + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + s = Server(self.options.pdb_name, self.options.domain_name) + + def do_client(self): + c = Client('localhost') + + def main(self): + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) == 0: + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + server_command = '%s -p %s ' % (self.prog_name, self.options.pdb_name) + server_command += ' server' + client_command = '%s client' % self.prog_name + server = subprocess.Popen(server_command, shell=True) + print 'server pid: %s' % server.pid + time.sleep(5) + clients = [] + for i in range(0, self.options.num_processes): + client = subprocess.Popen(client_command, shell=True) + clients.append(client) + print 'waiting for clients to finish' + for client in clients: + client.wait() + os.kill(server.pid, signal.SIGTERM) + elif len(self.args) == 1: + self.command = self.args[0] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + else: + self.parser.error('unrecognized commands') + +if __name__ == "__main__": + delete = Delete() + delete.main() diff --git a/storefront/boto/mapreduce/pdb_describe b/storefront/boto/mapreduce/pdb_describe new file mode 100755 index 0000000..d0fa86c --- /dev/null +++ b/storefront/boto/mapreduce/pdb_describe @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import sys +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager, get_domain + +USAGE = """ + SYNOPSIS + %prog [options] + DESCRIPTION + List and describe your PartitionDBs. + Called with no options, all PartitionDB objects defined in your default + domain (as specified in the "default_domain" option in the "[Persist]" + section of your boto config file) will be listed. + When called with a particular PartitionDB name (using -p option) all + Version objects of that PartitionDB object will be listed. + When called with the -p option and a particular Version name specified + (using the -v option) all Partitions in that Version object will be listed. +""" +class Describe: + + def __init__(self): + self.parser = OptionParser(usage=USAGE) + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-entries', action='store', type='int', + help='maximum number of entries to print (default 100)') + self.parser.set_defaults(num_entries=100) + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB to describe') + self.parser.add_option('-v', '--version-name', action='store', type='string', + help='name of the PDB Version to describe') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def describe_all(self): + print 'Using SimpleDB Domain: %s' % get_domain() + print 'PDBs:' + rs = PartitionDB.list() + i = 0 + for pdb in rs: + print '%s\t%s\t%s' % (pdb.id, pdb.name, pdb.bucket_name) + i += 1 + if i == self.options.num_entries: + break + + def describe_pdb(self, pdb_name): + print 'Using SimpleDB Domain: %s' % get_domain() + print 'PDB: %s' % pdb_name + print 'Versions:' + try: + pdb = PartitionDB.get(name=pdb_name) + i = 0 + for v in pdb.versions: + if v.date: + ds = v.date.isoformat() + else: + ds = 'unknown' + print '%s\t%s\t%s' % (v.id, v.name, ds) + i += 1 + if i == self.options.num_entries: + break + cv = pdb.current_version() + if cv: + print 'Current Version: %s' % cv.name + else: + print 'Current Version: None' + except SDBPersistenceError: + self.parser.error('pdb_name (%s) unknown' % pdb_name) + + def describe_version(self, pdb_name, version_name): + print 'Using SimpleDB Domain: %s' % get_domain() + print 'PDB: %s' % pdb_name + print 'Version: %s' % version_name + print 'Partitions:' + try: + pdb = PartitionDB.get(name=pdb_name) + for v in pdb.versions: + if v.name == version_name: + i = 0 + for p in v.partitions(): + print '%s\t%s' % (p.id, p.name) + i += 1 + if i == self.options.num_entries: + break + except SDBPersistenceError: + self.parser.error('pdb_name (%s) unknown' % pdb_name) + + def main(self): + self.options, self.args = self.parser.parse_args() + self.manager = get_manager(self.options.domain_name) + + if self.options.pdb_name: + if self.options.version_name: + self.describe_version(self.options.pdb_name, self.options.version_name) + else: + self.describe_pdb(self.options.pdb_name) + else: + self.describe_all() + +if __name__ == "__main__": + describe = Describe() + describe.main() diff --git a/storefront/boto/mapreduce/pdb_revert b/storefront/boto/mapreduce/pdb_revert new file mode 100755 index 0000000..daffeef --- /dev/null +++ b/storefront/boto/mapreduce/pdb_revert @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import queuetools, os, signal, sys +import subprocess +import time +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from lqs import LQSServer, PersistHandler +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager + +USAGE = """ + SYNOPSIS + %prog [options] [command] + DESCRIPTION + Revert to the previous Version in a PartitionDB. +""" +class Client: + + def __init__(self, queue_name): + self.q = queuetools.get_queue(queue_name) + self.q.connect() + self.manager = get_manager() + self.process() + + def process(self): + m = self.q.get() + while m['item']: + print 'Deleting: %s' % m['item'] + p = Partition(id=m['item'], manager=self.manager) + p.delete() + self.q.delete(m) + m = self.q.get() + print 'client processing complete' + +class Server: + + def __init__(self, pdb_name, domain_name=None): + self.pdb_name = pdb_name + self.manager = get_manager(domain_name) + self.pdb = PartitionDB.get(name=self.pdb_name) + self.serve() + + def serve(self): + v = self.pdb.revert() + args = {'v_id' : v.id} + rs = v.partitions() + s = LQSServer('', PersistHandler, rs, args) + s.serve_forever() + +class Revert: + + Commands = {'client' : 'Start a Revert client', + 'server' : 'Start a Revert server'} + + def __init__(self): + self.parser = OptionParser(usage=USAGE) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-processes', action='store', type='int', dest='num_processes', + help='the number of client processes launched') + self.parser.set_defaults(num_processes=5) + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB in which to store files (will create if necessary)') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_server(self): + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + s = Server(self.options.pdb_name, self.options.domain_name) + + def do_client(self): + c = Client('localhost') + + def main(self): + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) == 0: + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + server_command = '%s -p %s ' % (self.prog_name, self.options.pdb_name) + server_command += ' server' + client_command = '%s client' % self.prog_name + server = subprocess.Popen(server_command, shell=True) + print 'server pid: %s' % server.pid + time.sleep(5) + clients = [] + for i in range(0, self.options.num_processes): + client = subprocess.Popen(client_command, shell=True) + clients.append(client) + print 'waiting for clients to finish' + for client in clients: + client.wait() + os.kill(server.pid, signal.SIGTERM) + elif len(self.args) == 1: + self.command = self.args[0] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + else: + self.parser.error('unrecognized commands') + +if __name__ == "__main__": + revert = Revert() + revert.main() diff --git a/storefront/boto/mapreduce/pdb_upload b/storefront/boto/mapreduce/pdb_upload new file mode 100755 index 0000000..1ca2b6d --- /dev/null +++ b/storefront/boto/mapreduce/pdb_upload @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import queuetools, os, signal, sys +import subprocess +import time +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from lqs import LQSServer, LQSHandler +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager + +USAGE = """ + SYNOPSIS + %prog [options] + DESCRIPTION + Upload partition files to a PartitionDB. + Called with no options, all PartitionDB objects defined in your default + domain (as specified in the "default_domain" option in the "[Persist]" + section of your boto config file) will be listed. + When called with a particular PartitionDB name (using -p option) all + Version objects of that PartitionDB object will be listed. + When called with the -p option and a particular Version name specified + (using the -v option) all Partitions in that Version object will be listed. +""" +class Client: + + def __init__(self, queue_name): + self.q = queuetools.get_queue(queue_name) + self.q.connect() + self.manager = get_manager() + self.process() + + def process(self): + m = self.q.get() + if m['item']: + v = Version(m['args']['v_id'], self.manager) + bucket_name = v.pdb.bucket_name + while m['item']: + print 'Uploading: %s' % m['item'] + p = v.add_partition(name=m['item']) + p.upload(os.path.join(m['args']['path'], m['item']), bucket_name) + self.q.delete(m) + m = self.q.get() + print 'client processing complete' + +class Server: + + def __init__(self, path, pdb_name, bucket_name=None, domain_name=None): + self.path = path + self.pdb_name = pdb_name + self.bucket_name = bucket_name + self.manager = get_manager(domain_name) + self.get_pdb() + self.serve() + + def get_pdb(self): + try: + self.pdb = PartitionDB.get(name=self.pdb_name) + except SDBPersistenceError: + self.pdb = PartitionDB(manager=self.manager, name=self.pdb_name, bucket_name=self.bucket_name) + self.pdb.save() + + def serve(self): + v = self.pdb.add_version() + args = {'path' : self.path, + 'v_id' : v.id} + l = os.listdir(self.path) + s = LQSServer('', LQSHandler, iter(l), args) + s.serve_forever() + +class Upload: + + Usage = "usage: %prog [options] command" + + Commands = {'client' : 'Start an Upload client', + 'server' : 'Start an Upload server'} + + def __init__(self): + self.parser = OptionParser(usage=self.Usage) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-processes', action='store', type='int', dest='num_processes', + help='the number of client processes launched') + self.parser.set_defaults(num_processes=2) + self.parser.add_option('-i', '--input-path', action='store', type='string', + help='the path to directory to upload') + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB in which to store files (will create if necessary)') + self.parser.add_option('-b', '--bucket-name', action='store', type='string', + help='name of S3 bucket (only needed if creating new PDB)') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_server(self): + if not self.options.input_path: + self.parser.error('No path provided') + if not os.path.isdir(self.options.input_path): + self.parser.error('Invalid path (%s)' % self.options.input_path) + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + s = Server(self.options.input_path, self.options.pdb_name, + self.options.bucket_name, self.options.domain_name) + + def do_client(self): + c = Client('localhost') + + def main(self): + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) == 0: + if not self.options.input_path: + self.parser.error('No path provided') + if not os.path.isdir(self.options.input_path): + self.parser.error('Invalid path (%s)' % self.options.input_path) + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + server_command = '%s -p %s -i %s' % (self.prog_name, self.options.pdb_name, self.options.input_path) + if self.options.bucket_name: + server_command += ' -b %s' % self.options.bucket_name + server_command += ' server' + client_command = '%s client' % self.prog_name + server = subprocess.Popen(server_command, shell=True) + print 'server pid: %s' % server.pid + time.sleep(5) + clients = [] + for i in range(0, self.options.num_processes): + client = subprocess.Popen(client_command, shell=True) + clients.append(client) + print 'waiting for clients to finish' + for client in clients: + client.wait() + os.kill(server.pid, signal.SIGTERM) + elif len(self.args) == 1: + self.command = self.args[0] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + else: + self.parser.error('unrecognized commands') + +if __name__ == "__main__": + upload = Upload() + upload.main() diff --git a/storefront/boto/mapreduce/queuetools.py b/storefront/boto/mapreduce/queuetools.py new file mode 100644 index 0000000..3e08a10 --- /dev/null +++ b/storefront/boto/mapreduce/queuetools.py @@ -0,0 +1,66 @@ +#!/usr/bin/python +import socket, sys +from lqs import LQSServer, LQSMessage +import boto +from boto.sqs.jsonmessage import JSONMessage + +class LQSClient: + + def __init__(self, host): + self.host = host + self.port = LQSServer.PORT + self.timeout = LQSServer.TIMEOUT + self.max_len = LQSServer.MAXSIZE + self.sock = None + + def connect(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.sock.settimeout(self.timeout) + self.sock.connect((self.host, self.port)) + + def decode(self, jsonstr): + return LQSMessage(jsonvalue=jsonstr) + + def get(self): + self.sock.send('next') + try: + jsonstr = self.sock.recv(self.max_len) + msg = LQSMessage(jsonvalue=jsonstr) + return msg + except: + print "recv from %s failed" % self.host + + def delete(self, msg): + self.sock.send('delete %s' % msg['id']) + try: + jsonstr = self.sock.recv(self.max_len) + msg = LQSMessage(jsonvalue=jsonstr) + return msg + except: + print "recv from %s failed" % self.host + + def close(self): + self.sock.close() + +class SQSClient: + + def __init__(self, queue_name): + self.queue_name = queue_name + + def connect(self): + self.queue = boto.lookup('sqs', self.queue_name) + self.queue.set_mesasge_class(JSONMessage) + + def get(self): + m = self.queue.read() + return m.get_body() + + def close(self): + pass + +def get_queue(name): + if name == 'localhost': + return LQSClient(name) + else: + return SQSClient(name) + diff --git a/storefront/boto/mashups/__init__.py b/storefront/boto/mashups/__init__.py new file mode 100644 index 0000000..449bd16 --- /dev/null +++ b/storefront/boto/mashups/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/storefront/boto/mashups/iobject.py b/storefront/boto/mashups/iobject.py new file mode 100644 index 0000000..a226b5c --- /dev/null +++ b/storefront/boto/mashups/iobject.py @@ -0,0 +1,115 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import os + +def int_val_fn(v): + try: + int(v) + return True + except: + return False + +class IObject(object): + + def choose_from_list(self, item_list, search_str='', + prompt='Enter Selection'): + if not item_list: + print 'No Choices Available' + return + choice = None + while not choice: + n = 1 + choices = [] + for item in item_list: + if isinstance(item, str): + print '[%d] %s' % (n, item) + choices.append(item) + n += 1 + else: + obj, id, desc = item + if desc: + if desc.find(search_str) >= 0: + print '[%d] %s - %s' % (n, id, desc) + choices.append(obj) + n += 1 + else: + if id.find(search_str) >= 0: + print '[%d] %s' % (n, id) + choices.append(obj) + n += 1 + if choices: + val = raw_input('%s[1-%d]: ' % (prompt, len(choices))) + if val.startswith('/'): + search_str = val[1:] + else: + try: + int_val = int(val) + if int_val == 0: + return None + choice = choices[int_val-1] + except ValueError: + print '%s is not a valid choice' % val + except IndexError: + print '%s is not within the range[1-%d]' % (val, + len(choices)) + else: + print "No objects matched your pattern" + search_str = '' + return choice + + def get_string(self, prompt, validation_fn=None): + okay = False + while not okay: + val = raw_input('%s: ' % prompt) + if validation_fn: + okay = validation_fn(val) + if not okay: + print 'Invalid value: %s' % val + else: + okay = True + return val + + def get_filename(self, prompt): + okay = False + val = '' + while not okay: + val = raw_input('%s: %s' % (prompt, val)) + val = os.path.expanduser(val) + if os.path.isfile(val): + okay = True + elif os.path.isdir(val): + path = val + val = self.choose_from_list(os.listdir(path)) + if val: + val = os.path.join(path, val) + okay = True + else: + val = '' + else: + print 'Invalid value: %s' % val + val = '' + return val + + def get_int(self, prompt): + s = self.get_string(prompt, int_val_fn) + return int(s) + diff --git a/storefront/boto/mashups/order.py b/storefront/boto/mashups/order.py new file mode 100644 index 0000000..6efdc3e --- /dev/null +++ b/storefront/boto/mashups/order.py @@ -0,0 +1,211 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +High-level abstraction of an EC2 order for servers +""" + +import boto +import boto.ec2 +from boto.mashups.server import Server, ServerSet +from boto.mashups.iobject import IObject +from boto.pyami.config import Config +from boto.sdb.persist import get_domain, set_domain +import time, StringIO + +InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', 'c1.medium', 'c1.xlarge'] + +class Item(IObject): + + def __init__(self): + self.region = None + self.name = None + self.instance_type = None + self.quantity = 0 + self.zone = None + self.ami = None + self.groups = [] + self.key = None + self.ec2 = None + self.config = None + + def set_userdata(self, key, value): + self.userdata[key] = value + + def get_userdata(self, key): + return self.userdata[key] + + def set_region(self, region=None): + if region: + self.region = region + else: + l = [(r, r.name, r.endpoint) for r in boto.ec2.regions()] + self.region = self.choose_from_list(l, prompt='Choose Region') + + def set_name(self, name=None): + if name: + self.name = name + else: + self.name = self.get_string('Name') + + def set_instance_type(self, instance_type=None): + if instance_type: + self.instance_type = instance_type + else: + self.instance_type = self.choose_from_list(InstanceTypes, 'Instance Type') + + def set_quantity(self, n=0): + if n > 0: + self.quantity = n + else: + self.quantity = self.get_int('Quantity') + + def set_zone(self, zone=None): + if zone: + self.zone = zone + else: + l = [(z, z.name, z.state) for z in self.ec2.get_all_zones()] + self.zone = self.choose_from_list(l, prompt='Choose Availability Zone') + + def set_ami(self, ami=None): + if ami: + self.ami = ami + else: + l = [(a, a.id, a.location) for a in self.ec2.get_all_images()] + self.ami = self.choose_from_list(l, prompt='Choose AMI') + + def add_group(self, group=None): + if group: + self.groups.append(group) + else: + l = [(s, s.name, s.description) for s in self.ec2.get_all_security_groups()] + self.groups.append(self.choose_from_list(l, prompt='Choose Security Group')) + + def set_key(self, key=None): + if key: + self.key = key + else: + l = [(k, k.name, '') for k in self.ec2.get_all_key_pairs()] + self.key = self.choose_from_list(l, prompt='Choose Keypair') + + def update_config(self): + if not self.config.has_section('Credentials'): + self.config.add_section('Credentials') + self.config.set('Credentials', 'aws_access_key_id', self.ec2.aws_access_key_id) + self.config.set('Credentials', 'aws_secret_access_key', self.ec2.aws_secret_access_key) + if not self.config.has_section('Pyami'): + self.config.add_section('Pyami') + sdb_domain = get_domain() + if sdb_domain: + self.config.set('Pyami', 'server_sdb_domain', sdb_domain) + self.config.set('Pyami', 'server_sdb_name', self.name) + + def set_config(self, config_path=None): + if not config_path: + config_path = self.get_filename('Specify Config file') + self.config = Config(path=config_path) + + def get_userdata_string(self): + s = StringIO.StringIO() + self.config.write(s) + return s.getvalue() + + def enter(self, **params): + self.region = params.get('region', self.region) + if not self.region: + self.set_region() + self.ec2 = self.region.connect() + self.name = params.get('name', self.name) + if not self.name: + self.set_name() + self.instance_type = params.get('instance_type', self.instance_type) + if not self.instance_type: + self.set_instance_type() + self.zone = params.get('zone', self.zone) + if not self.zone: + self.set_zone() + self.quantity = params.get('quantity', self.quantity) + if not self.quantity: + self.set_quantity() + self.ami = params.get('ami', self.ami) + if not self.ami: + self.set_ami() + self.groups = params.get('groups', self.groups) + if not self.groups: + self.add_group() + self.key = params.get('key', self.key) + if not self.key: + self.set_key() + self.config = params.get('config', self.config) + if not self.config: + self.set_config() + self.update_config() + +class Order(IObject): + + def __init__(self): + self.items = [] + self.reservation = None + + def add_item(self, **params): + item = Item() + item.enter(**params) + self.items.append(item) + + def display(self): + print 'This Order consists of the following items' + print + print 'QTY\tNAME\tTYPE\nAMI\t\tGroups\t\t\tKeyPair' + for item in self.items: + print '%s\t%s\t%s\t%s\t%s\t%s' % (item.quantity, item.name, item.instance_type, + item.ami.id, item.groups, item.key.name) + + def place(self, block=True): + if get_domain() == None: + print 'SDB Persistence Domain not set' + domain_name = self.get_string('Specify SDB Domain') + set_domain(domain_name) + s = ServerSet() + for item in self.items: + r = item.ami.run(min_count=1, max_count=item.quantity, + key_name=item.key.name, user_data=item.get_userdata_string(), + security_groups=item.groups, instance_type=item.instance_type, + placement=item.zone.name) + if block: + states = [i.state for i in r.instances] + if states.count('running') != len(states): + print states + time.sleep(15) + states = [i.update() for i in r.instances] + for i in r.instances: + server = Server() + server.name = item.name + server.instance_id = i.id + server.reservation = r + server.save() + s.append(server) + if len(s) == 1: + return s[0] + else: + return s + + + diff --git a/storefront/boto/mashups/server.py b/storefront/boto/mashups/server.py new file mode 100644 index 0000000..48f637b --- /dev/null +++ b/storefront/boto/mashups/server.py @@ -0,0 +1,388 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +High-level abstraction of an EC2 server +""" +import boto, boto.utils +from boto.mashups.iobject import IObject +from boto.pyami.config import Config, BotoConfigPath +from boto.mashups.interactive import interactive_shell +from boto.sdb.db.model import Model +from boto.sdb.db.property import * +import os +import StringIO + +class ServerSet(list): + + def __getattr__(self, name): + results = [] + is_callable = False + for server in self: + try: + val = getattr(server, name) + if callable(val): + is_callable = True + results.append(val) + except: + results.append(None) + if is_callable: + self.map_list = results + return self.map + return results + + def map(self, *args): + results = [] + for fn in self.map_list: + results.append(fn(*args)) + return results + +class Server(Model): + + ec2 = boto.connect_ec2() + + @classmethod + def Inventory(cls): + """ + Returns a list of Server instances, one for each Server object + persisted in the db + """ + l = ServerSet() + rs = cls.find() + for server in rs: + l.append(server) + return l + + @classmethod + def Register(cls, name, instance_id, description=''): + s = cls() + s.name = name + s.instance_id = instance_id + s.description = description + s.save() + return s + + def __init__(self, id=None, **kw): + Model.__init__(self, id, **kw) + self._reservation = None + self._instance = None + self._ssh_client = None + self._pkey = None + self._config = None + + name = StringProperty(unique=True, verbose_name="Name") + instance_id = StringProperty(verbose_name="Instance ID") + config_uri = StringProperty() + ami_id = StringProperty(verbose_name="AMI ID") + zone = StringProperty(verbose_name="Availability Zone") + security_group = StringProperty(verbose_name="Security Group", default="default") + key_name = StringProperty(verbose_name="Key Name") + elastic_ip = StringProperty(verbose_name="Elastic IP") + instance_type = StringProperty(verbose_name="Instance Type") + description = StringProperty(verbose_name="Description") + log = StringProperty() + + def setReadOnly(self, value): + raise AttributeError + + def getInstance(self): + if not self._instance: + if self.instance_id: + try: + rs = self.ec2.get_all_instances([self.instance_id]) + except: + return None + if len(rs) > 0: + self._reservation = rs[0] + self._instance = self._reservation.instances[0] + return self._instance + + instance = property(getInstance, setReadOnly, None, 'The Instance for the server') + + def getAMI(self): + if self.instance: + return self.instance.image_id + + ami = property(getAMI, setReadOnly, None, 'The AMI for the server') + + def getStatus(self): + if self.instance: + self.instance.update() + return self.instance.state + + status = property(getStatus, setReadOnly, None, + 'The status of the server') + + def getHostname(self): + if self.instance: + return self.instance.public_dns_name + + hostname = property(getHostname, setReadOnly, None, + 'The public DNS name of the server') + + def getPrivateHostname(self): + if self.instance: + return self.instance.private_dns_name + + private_hostname = property(getPrivateHostname, setReadOnly, None, + 'The private DNS name of the server') + + def getLaunchTime(self): + if self.instance: + return self.instance.launch_time + + launch_time = property(getLaunchTime, setReadOnly, None, + 'The time the Server was started') + + def getConsoleOutput(self): + if self.instance: + return self.instance.get_console_output() + + console_output = property(getConsoleOutput, setReadOnly, None, + 'Retrieve the console output for server') + + def getGroups(self): + if self._reservation: + return self._reservation.groups + else: + return None + + groups = property(getGroups, setReadOnly, None, + 'The Security Groups controlling access to this server') + + def getConfig(self): + if not self._config: + remote_file = BotoConfigPath + local_file = '%s.ini' % self.instance.id + self.get_file(remote_file, local_file) + self._config = Config(local_file) + return self._config + + def setConfig(self, config): + local_file = '%s.ini' % self.instance.id + fp = open(local_file) + config.write(fp) + fp.close() + self.put_file(local_file, BotoConfigPath) + self._config = config + + config = property(getConfig, setConfig, None, + 'The instance data for this server') + + def set_config(self, config): + """ + Set SDB based config + """ + self._config = config + self._config.dump_to_sdb("botoConfigs", self.id) + + def load_config(self): + self._config = Config(do_load=False) + self._config.load_from_sdb("botoConfigs", self.id) + + def stop(self): + if self.instance: + self.instance.stop() + + def start(self): + self.stop() + ec2 = boto.connect_ec2() + ami = ec2.get_all_images(image_ids = [str(self.ami_id)])[0] + groups = ec2.get_all_security_groups(groupnames=[str(self.security_group)]) + if not self._config: + self.load_config() + if not self._config.has_section("Credentials"): + self._config.add_section("Credentials") + self._config.set("Credentials", "aws_access_key_id", ec2.aws_access_key_id) + self._config.set("Credentials", "aws_secret_access_key", ec2.aws_secret_access_key) + + if not self._config.has_section("Pyami"): + self._config.add_section("Pyami") + + if self._manager.domain: + self._config.set('Pyami', 'server_sdb_domain', self._manager.domain.name) + self._config.set("Pyami", 'server_sdb_name', self.name) + + cfg = StringIO.StringIO() + self._config.write(cfg) + cfg = cfg.getvalue() + r = ami.run(min_count=1, + max_count=1, + key_name=self.key_name, + security_groups = groups, + instance_type = self.instance_type, + placement = self.zone, + user_data = cfg) + i = r.instances[0] + self.instance_id = i.id + self.put() + if self.elastic_ip: + ec2.associate_address(self.instance_id, self.elastic_ip) + + def reboot(self): + if self.instance: + self.instance.reboot() + + def get_ssh_client(self, key_file=None, host_key_file='~/.ssh/known_hosts', + uname='root'): + import paramiko + if not self.instance: + print 'No instance yet!' + return + if not self._ssh_client: + if not key_file: + iobject = IObject() + key_file = iobject.get_filename('Path to OpenSSH Key file') + self._pkey = paramiko.RSAKey.from_private_key_file(key_file) + self._ssh_client = paramiko.SSHClient() + self._ssh_client.load_system_host_keys() + self._ssh_client.load_host_keys(os.path.expanduser(host_key_file)) + self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self._ssh_client.connect(self.instance.public_dns_name, + username=uname, pkey=self._pkey) + return self._ssh_client + + def get_file(self, remotepath, localpath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + sftp_client.get(remotepath, localpath) + + def put_file(self, localpath, remotepath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + sftp_client.put(localpath, remotepath) + + def listdir(self, remotepath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + return sftp_client.listdir(remotepath) + + def shell(self, key_file=None): + ssh_client = self.get_ssh_client(key_file) + channel = ssh_client.invoke_shell() + interactive_shell(channel) + + def bundle_image(self, prefix, key_file, cert_file, size): + print 'bundling image...' + print '\tcopying cert and pk over to /mnt directory on server' + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + path, name = os.path.split(key_file) + remote_key_file = '/mnt/%s' % name + self.put_file(key_file, remote_key_file) + path, name = os.path.split(cert_file) + remote_cert_file = '/mnt/%s' % name + self.put_file(cert_file, remote_cert_file) + print '\tdeleting %s' % BotoConfigPath + # delete the metadata.ini file if it exists + try: + sftp_client.remove(BotoConfigPath) + except: + pass + command = 'ec2-bundle-vol ' + command += '-c %s -k %s ' % (remote_cert_file, remote_key_file) + command += '-u %s ' % self._reservation.owner_id + command += '-p %s ' % prefix + command += '-s %d ' % size + command += '-d /mnt ' + if self.instance.instance_type == 'm1.small' or self.instance_type == 'c1.medium': + command += '-r i386' + else: + command += '-r x86_64' + print '\t%s' % command + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' + + def upload_bundle(self, bucket, prefix): + print 'uploading bundle...' + command = 'ec2-upload-bundle ' + command += '-m /mnt/%s.manifest.xml ' % prefix + command += '-b %s ' % bucket + command += '-a %s ' % self.ec2.aws_access_key_id + command += '-s %s ' % self.ec2.aws_secret_access_key + print '\t%s' % command + ssh_client = self.get_ssh_client() + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' + + def create_image(self, bucket=None, prefix=None, key_file=None, cert_file=None, size=None): + iobject = IObject() + if not bucket: + bucket = iobject.get_string('Name of S3 bucket') + if not prefix: + prefix = iobject.get_string('Prefix for AMI file') + if not key_file: + key_file = iobject.get_filename('Path to RSA private key file') + if not cert_file: + cert_file = iobject.get_filename('Path to RSA public cert file') + if not size: + size = iobject.get_int('Size (in MB) of bundled image') + self.bundle_image(prefix, key_file, cert_file, size) + self.upload_bundle(bucket, prefix) + print 'registering image...' + self.image_id = self.ec2.register_image('%s/%s.manifest.xml' % (bucket, prefix)) + return self.image_id + + def attach_volume(self, volume, device="/dev/sdp"): + """ + Attach an EBS volume to this server + + :param volume: EBS Volume to attach + :type volume: boto.ec2.volume.Volume + + :param device: Device to attach to (default to /dev/sdp) + :type device: string + """ + if hasattr(volume, "id"): + volume_id = volume.id + else: + volume_id = volume + return self.ec2.attach_volume(volume_id=volume_id, instance_id=self.instance_id, device=device) + + def detach_volume(self, volume): + """ + Detach an EBS volume from this server + + :param volume: EBS Volume to detach + :type volume: boto.ec2.volume.Volume + """ + if hasattr(volume, "id"): + volume_id = volume.id + else: + volume_id = volume + return self.ec2.detach_volume(volume_id=volume_id, instance_id=self.instance_id) + + def install_package(self, package_name): + print 'installing %s...' % package_name + command = 'yum -y install %s' % package_name + print '\t%s' % command + ssh_client = self.get_ssh_client() + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' diff --git a/storefront/boto/mturk/__init__.py b/storefront/boto/mturk/__init__.py new file mode 100644 index 0000000..449bd16 --- /dev/null +++ b/storefront/boto/mturk/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/storefront/boto/mturk/connection.py b/storefront/boto/mturk/connection.py new file mode 100644 index 0000000..261e2a7 --- /dev/null +++ b/storefront/boto/mturk/connection.py @@ -0,0 +1,504 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax +import datetime + +from boto import handler +from boto.mturk.price import Price +import boto.mturk.notification +from boto.connection import AWSQueryConnection +from boto.exception import EC2ResponseError +from boto.resultset import ResultSet + +class MTurkConnection(AWSQueryConnection): + + APIVersion = '2006-10-31' + SignatureVersion = '1' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host='mechanicalturk.amazonaws.com', debug=0, + https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory) + + def get_account_balance(self): + """ + """ + params = {} + return self._process_request('GetAccountBalance', params, [('AvailableBalance', Price), + ('OnHoldBalance', Price)]) + + def register_hit_type(self, title, description, reward, duration, + keywords=None, approval_delay=None, qual_req=None): + """ + Register a new HIT Type + \ttitle, description are strings + \treward is a Price object + \tduration can be an integer or string + """ + params = {'Title' : title, + 'Description' : description, + 'AssignmentDurationInSeconds' : duration} + params.update(MTurkConnection.get_price_as_price(reward).get_as_params('Reward')) + + if keywords: + params['Keywords'] = keywords + + if approval_delay is not None: + params['AutoApprovalDelayInSeconds']= approval_delay + + return self._process_request('RegisterHITType', params) + + def set_email_notification(self, hit_type, email, event_types=None): + """ + Performs a SetHITTypeNotification operation to set email notification for a specified HIT type + """ + return self._set_notification(hit_type, 'Email', email, event_types) + + def set_rest_notification(self, hit_type, url, event_types=None): + """ + Performs a SetHITTypeNotification operation to set REST notification for a specified HIT type + """ + return self._set_notification(hit_type, 'REST', url, event_types) + + def _set_notification(self, hit_type, transport, destination, event_types=None): + """ + Common SetHITTypeNotification operation to set notification for a specified HIT type + """ + assert type(hit_type) is str, "hit_type argument should be a string." + + params = {'HITTypeId': hit_type} + + # from the Developer Guide: + # The 'Active' parameter is optional. If omitted, the active status of the HIT type's + # notification specification is unchanged. All HIT types begin with their + # notification specifications in the "inactive" status. + notification_params = {'Destination': destination, + 'Transport': transport, + 'Version': boto.mturk.notification.NotificationMessage.NOTIFICATION_VERSION, + 'Active': True, + } + + # add specific event types if required + if event_types: + self.build_list_params(notification_params, event_types, 'EventType') + + # Set up dict of 'Notification.1.Transport' etc. values + notification_rest_params = {} + num = 1 + for key in notification_params: + notification_rest_params['Notification.%d.%s' % (num, key)] = notification_params[key] + + # Update main params dict + params.update(notification_rest_params) + + # Execute operation + return self._process_request('SetHITTypeNotification', params) + + def create_hit(self, hit_type=None, question=None, lifetime=60*60*24*7, max_assignments=1, + title=None, description=None, keywords=None, reward=None, + duration=60*60*24*7, approval_delay=None, annotation=None, qual_req=None, + questions=None, qualifications=None, response_groups=None): + """ + Creates a new HIT. + Returns a ResultSet + See: http://docs.amazonwebservices.com/AWSMechanicalTurkRequester/2006-10-31/ApiReference_CreateHITOperation.html + """ + + # handle single or multiple questions + if question is not None and questions is not None: + raise ValueError("Must specify either question (single Question instance) or questions (list), but not both") + if question is not None and questions is None: + questions = [question] + + + # Handle basic required arguments and set up params dict + params = {'Question': question.get_as_xml(), + 'LifetimeInSeconds' : lifetime, + 'MaxAssignments' : max_assignments, + } + + # if hit type specified then add it + # else add the additional required parameters + if hit_type: + params['HITTypeId'] = hit_type + else: + # Handle keywords + final_keywords = MTurkConnection.get_keywords_as_string(keywords) + + # Handle price argument + final_price = MTurkConnection.get_price_as_price(reward) + + additional_params = {'Title': title, + 'Description' : description, + 'Keywords': final_keywords, + 'AssignmentDurationInSeconds' : duration, + } + additional_params.update(final_price.get_as_params('Reward')) + + if approval_delay is not None: + additional_params['AutoApprovalDelayInSeconds'] = approval_delay + + # add these params to the others + params.update(additional_params) + + # add the annotation if specified + if annotation is not None: + params['RequesterAnnotation'] = annotation + + # Add the Qualifications if specified + if qualifications is not None: + params.update(qualifications.get_as_params()) + + # Handle optional response groups argument + if response_groups: + self.build_list_params(params, response_groups, 'ResponseGroup') + + # Submit + return self._process_request('CreateHIT', params, [('HIT', HIT),]) + + def get_reviewable_hits(self, hit_type=None, status='Reviewable', + sort_by='Expiration', sort_direction='Ascending', + page_size=10, page_number=1): + """ + Retrieve the HITs that have a status of Reviewable, or HITs that + have a status of Reviewing, and that belong to the Requester calling the operation. + """ + params = {'Status' : status, + 'SortProperty' : sort_by, + 'SortDirection' : sort_direction, + 'PageSize' : page_size, + 'PageNumber' : page_number} + + # Handle optional hit_type argument + if hit_type is not None: + params.update({'HITTypeId': hit_type}) + + return self._process_request('GetReviewableHITs', params, [('HIT', HIT),]) + + def search_hits(self, sort_by='CreationTime', sort_direction='Ascending', + page_size=10, page_number=1): + """ + Return all of a Requester's HITs, on behalf of the Requester. + The operation returns HITs of any status, except for HITs that have been disposed + with the DisposeHIT operation. + Note: + The SearchHITs operation does not accept any search parameters that filter the results. + """ + params = {'SortProperty' : sort_by, + 'SortDirection' : sort_direction, + 'PageSize' : page_size, + 'PageNumber' : page_number} + + return self._process_request('SearchHITs', params, [('HIT', HIT),]) + + def get_assignments(self, hit_id, status=None, + sort_by='SubmitTime', sort_direction='Ascending', + page_size=10, page_number=1): + """ + Retrieves completed assignments for a HIT. + Use this operation to retrieve the results for a HIT. + + The returned ResultSet will have the following attributes: + + NumResults + The number of assignments on the page in the filtered results list, + equivalent to the number of assignments being returned by this call. + A non-negative integer + PageNumber + The number of the page in the filtered results list being returned. + A positive integer + TotalNumResults + The total number of HITs in the filtered results list based on this call. + A non-negative integer + + The ResultSet will contain zero or more Assignment objects + + """ + params = {'HITId' : hit_id, + 'SortProperty' : sort_by, + 'SortDirection' : sort_direction, + 'PageSize' : page_size, + 'PageNumber' : page_number} + + if status is not None: + params['AssignmentStatus'] = status + + return self._process_request('GetAssignmentsForHIT', params, [('Assignment', Assignment),]) + + def approve_assignment(self, assignment_id, feedback=None): + """ + """ + params = {'AssignmentId' : assignment_id,} + if feedback: + params['RequesterFeedback'] = feedback + return self._process_request('ApproveAssignment', params) + + def reject_assignment(self, assignment_id, feedback=None): + """ + """ + params = {'AssignmentId' : assignment_id,} + if feedback: + params['RequesterFeedback'] = feedback + return self._process_request('RejectAssignment', params) + + def get_hit(self, hit_id): + """ + """ + params = {'HITId' : hit_id,} + return self._process_request('GetHIT', params, [('HIT', HIT),]) + + def set_reviewing(self, hit_id, revert=None): + """ + Update a HIT with a status of Reviewable to have a status of Reviewing, + or reverts a Reviewing HIT back to the Reviewable status. + + Only HITs with a status of Reviewable can be updated with a status of Reviewing. + Similarly, only Reviewing HITs can be reverted back to a status of Reviewable. + """ + params = {'HITId' : hit_id,} + if revert: + params['Revert'] = revert + return self._process_request('SetHITAsReviewing', params) + + def disable_hit(self, hit_id): + """ + Remove a HIT from the Mechanical Turk marketplace, approves all submitted assignments + that have not already been approved or rejected, and disposes of the HIT and all + assignment data. + + Assignments for the HIT that have already been submitted, but not yet approved or rejected, will be + automatically approved. Assignments in progress at the time of the call to DisableHIT will be + approved once the assignments are submitted. You will be charged for approval of these assignments. + DisableHIT completely disposes of the HIT and all submitted assignment data. Assignment results + data cannot be retrieved for a HIT that has been disposed. + + It is not possible to re-enable a HIT once it has been disabled. To make the work from a disabled HIT + available again, create a new HIT. + """ + params = {'HITId' : hit_id,} + return self._process_request('DisableHIT', params) + + def dispose_hit(self, hit_id): + """ + Dispose of a HIT that is no longer needed. + + Only HITs in the "reviewable" state, with all submitted assignments approved or rejected, + can be disposed. A Requester can call GetReviewableHITs to determine which HITs are + reviewable, then call GetAssignmentsForHIT to retrieve the assignments. + Disposing of a HIT removes the HIT from the results of a call to GetReviewableHITs. + """ + params = {'HITId' : hit_id,} + return self._process_request('DisposeHIT', params) + + def expire_hit(self, hit_id): + + """ + Expire a HIT that is no longer needed. + + The effect is identical to the HIT expiring on its own. The HIT no longer appears on the + Mechanical Turk web site, and no new Workers are allowed to accept the HIT. Workers who + have accepted the HIT prior to expiration are allowed to complete it or return it, or + allow the assignment duration to elapse (abandon the HIT). Once all remaining assignments + have been submitted, the expired HIT becomes "reviewable", and will be returned by a call + to GetReviewableHITs. + """ + params = {'HITId' : hit_id,} + return self._process_request('ForceExpireHIT', params) + + def extend_hit(self, hit_id, assignments_increment=None, expiration_increment=None): + """ + Increase the maximum number of assignments, or extend the expiration date, of an existing HIT. + + NOTE: If a HIT has a status of Reviewable and the HIT is extended to make it Available, the + HIT will not be returned by GetReviewableHITs, and its submitted assignments will not + be returned by GetAssignmentsForHIT, until the HIT is Reviewable again. + Assignment auto-approval will still happen on its original schedule, even if the HIT has + been extended. Be sure to retrieve and approve (or reject) submitted assignments before + extending the HIT, if so desired. + """ + # must provide assignment *or* expiration increment + if (assignments_increment is None and expiration_increment is None) or \ + (assignments_increment is not None and expiration_increment is not None): + raise ValueError("Must specify either assignments_increment or expiration_increment, but not both") + + params = {'HITId' : hit_id,} + if assignments_increment: + params['MaxAssignmentsIncrement'] = assignments_increment + if expiration_increment: + params['ExpirationIncrementInSeconds'] = expiration_increment + + return self._process_request('ExtendHIT', params) + + def get_help(self, about, help_type='Operation'): + """ + Return information about the Mechanical Turk Service operations and response group + NOTE - this is basically useless as it just returns the URL of the documentation + + help_type: either 'Operation' or 'ResponseGroup' + """ + params = {'About': about, 'HelpType': help_type,} + return self._process_request('Help', params) + + def grant_bonus(self, worker_id, assignment_id, bonus_price, reason): + """ + Issues a payment of money from your account to a Worker. + To be eligible for a bonus, the Worker must have submitted results for one of your + HITs, and have had those results approved or rejected. This payment happens separately + from the reward you pay to the Worker when you approve the Worker's assignment. + The Bonus must be passed in as an instance of the Price object. + """ + params = bonus_price.get_as_params('BonusAmount', 1) + params['WorkerId'] = worker_id + params['AssignmentId'] = assignment_id + params['Reason'] = reason + + return self._process_request('GrantBonus', params) + + def _process_request(self, request_type, params, marker_elems=None): + """ + Helper to process the xml response from AWS + """ + response = self.make_request(request_type, params) + return self._process_response(response, marker_elems) + + def _process_response(self, response, marker_elems=None): + """ + Helper to process the xml response from AWS + """ + body = response.read() + #print body + if '' not in body: + rs = ResultSet(marker_elems) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise EC2ResponseError(response.status, response.reason, body) + + @staticmethod + def get_keywords_as_string(keywords): + """ + Returns a comma+space-separated string of keywords from either a list or a string + """ + if type(keywords) is list: + final_keywords = ', '.join(keywords) + elif type(keywords) is str: + final_keywords = keywords + elif type(keywords) is unicode: + final_keywords = keywords.encode('utf-8') + elif keywords is None: + final_keywords = "" + else: + raise TypeError("keywords argument must be a string or a list of strings; got a %s" % type(keywords)) + return final_keywords + + @staticmethod + def get_price_as_price(reward): + """ + Returns a Price data structure from either a float or a Price + """ + if isinstance(reward, Price): + final_price = reward + else: + final_price = Price(reward) + return final_price + +class BaseAutoResultElement: + """ + Base class to automatically add attributes when parsing XML + """ + def __init__(self, connection): + self.connection = connection + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + setattr(self, name, value) + +class HIT(BaseAutoResultElement): + """ + Class to extract a HIT structure from a response (used in ResultSet) + + Will have attributes named as per the Developer Guide, + e.g. HITId, HITTypeId, CreationTime + """ + + # property helper to determine if HIT has expired + def _has_expired(self): + """ Has this HIT expired yet? """ + expired = False + if hasattr(self, 'Expiration'): + now = datetime.datetime.utcnow() + expiration = datetime.datetime.strptime(self.Expiration, '%Y-%m-%dT%H:%M:%SZ') + expired = (now >= expiration) + else: + raise ValueError("ERROR: Request for expired property, but no Expiration in HIT!") + return expired + + # are we there yet? + expired = property(_has_expired) + +class Assignment(BaseAutoResultElement): + """ + Class to extract an Assignment structure from a response (used in ResultSet) + + Will have attributes named as per the Developer Guide, + e.g. AssignmentId, WorkerId, HITId, Answer, etc + """ + + def __init__(self, connection): + BaseAutoResultElement.__init__(self, connection) + self.answers = [] + + def endElement(self, name, value, connection): + # the answer consists of embedded XML, so it needs to be parsed independantly + if name == 'Answer': + answer_rs = ResultSet([('Answer', QuestionFormAnswer),]) + h = handler.XmlHandler(answer_rs, connection) + value = self.connection.get_utf8_value(value) + xml.sax.parseString(value, h) + self.answers.append(answer_rs) + else: + BaseAutoResultElement.endElement(self, name, value, connection) + +class QuestionFormAnswer(BaseAutoResultElement): + """ + Class to extract Answers from inside the embedded XML QuestionFormAnswers element inside the + Answer element which is part of the Assignment structure + + A QuestionFormAnswers element contains an Answer element for each question in the HIT or + Qualification test for which the Worker provided an answer. Each Answer contains a + QuestionIdentifier element whose value corresponds to the QuestionIdentifier of a + Question in the QuestionForm. See the QuestionForm data structure for more information about + questions and answer specifications. + + If the question expects a free-text answer, the Answer element contains a FreeText element. This + element contains the Worker's answer + + *NOTE* - currently really only supports free-text answers + """ + + pass diff --git a/storefront/boto/mturk/notification.py b/storefront/boto/mturk/notification.py new file mode 100644 index 0000000..4904a99 --- /dev/null +++ b/storefront/boto/mturk/notification.py @@ -0,0 +1,95 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Provides NotificationMessage and Event classes, with utility methods, for +implementations of the Mechanical Turk Notification API. +""" + +import hmac +try: + from hashlib import sha1 as sha +except ImportError: + import sha +import base64 +import re + +class NotificationMessage: + + NOTIFICATION_WSDL = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurk/2006-05-05/AWSMechanicalTurkRequesterNotification.wsdl" + NOTIFICATION_VERSION = '2006-05-05' + + SERVICE_NAME = "AWSMechanicalTurkRequesterNotification" + OPERATION_NAME = "Notify" + + EVENT_PATTERN = r"Event\.(?P\d+)\.(?P\w+)" + EVENT_RE = re.compile(EVENT_PATTERN) + + def __init__(self, d): + """ + Constructor; expects parameter d to be a dict of string parameters from a REST transport notification message + """ + self.signature = d['Signature'] # vH6ZbE0NhkF/hfNyxz2OgmzXYKs= + self.timestamp = d['Timestamp'] # 2006-05-23T23:22:30Z + self.version = d['Version'] # 2006-05-05 + assert d['method'] == NotificationMessage.OPERATION_NAME, "Method should be '%s'" % NotificationMessage.OPERATION_NAME + + # Build Events + self.events = [] + events_dict = {} + if 'Event' in d: + # TurboGears surprised me by 'doing the right thing' and making { 'Event': { '1': { 'EventType': ... } } } etc. + events_dict = d['Event'] + else: + for k in d: + v = d[k] + if k.startswith('Event.'): + ed = NotificationMessage.EVENT_RE.search(k).groupdict() + n = int(ed['n']) + param = str(ed['param']) + if n not in events_dict: + events_dict[n] = {} + events_dict[n][param] = v + for n in events_dict: + self.events.append(Event(events_dict[n])) + + def verify(self, secret_key): + """ + Verifies the authenticity of a notification message. + """ + verification_input = NotificationMessage.SERVICE_NAME + NotificationMessage.OPERATION_NAME + self.timestamp + h = hmac.new(key=secret_key, digestmod=sha) + h.update(verification_input) + signature_calc = base64.b64encode(h.digest()) + return self.signature == signature_calc + +class Event: + def __init__(self, d): + self.event_type = d['EventType'] + self.event_time_str = d['EventTime'] + self.hit_type = d['HITTypeId'] + self.hit_id = d['HITId'] + self.assignment_id = d['AssignmentId'] + + #TODO: build self.event_time datetime from string self.event_time_str + + def __repr__(self): + return "" % (self.event_type, self.hit_id) diff --git a/storefront/boto/mturk/price.py b/storefront/boto/mturk/price.py new file mode 100644 index 0000000..3c88a96 --- /dev/null +++ b/storefront/boto/mturk/price.py @@ -0,0 +1,48 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Price: + + def __init__(self, amount=0.0, currency_code='USD'): + self.amount = amount + self.currency_code = currency_code + self.formatted_price = '' + + def __repr__(self): + if self.formatted_price: + return self.formatted_price + else: + return str(self.amount) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Amount': + self.amount = float(value) + elif name == 'CurrencyCode': + self.currency_code = value + elif name == 'FormattedPrice': + self.formatted_price = value + + def get_as_params(self, label, ord=1): + return {'%s.%d.Amount'%(label, ord) : str(self.amount), + '%s.%d.CurrencyCode'%(label, ord) : self.currency_code} diff --git a/storefront/boto/mturk/qualification.py b/storefront/boto/mturk/qualification.py new file mode 100644 index 0000000..ed02087 --- /dev/null +++ b/storefront/boto/mturk/qualification.py @@ -0,0 +1,118 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Qualifications: + + def __init__(self, requirements = []): + self.requirements = requirements + + def add(self, req): + self.requirements.append(req) + + def get_as_params(self): + params = {} + assert(len(self.requirements) <= 10) + for n, req in enumerate(self.requirements): + reqparams = req.get_as_params() + for rp in reqparams: + params['QualificationRequirement.%s.%s' % ((n+1),rp) ] = reqparams[rp] + return params + + +class Requirement(object): + """ + Representation of a single requirement + """ + + def __init__(self, qualification_type_id, comparator, integer_value, required_to_preview=False): + self.qualification_type_id = qualification_type_id + self.comparator = comparator + self.integer_value = integer_value + self.required_to_preview = required_to_preview + + def get_as_params(self): + params = { + "QualificationTypeId": self.qualification_type_id, + "Comparator": self.comparator, + "IntegerValue": self.integer_value, + } + if self.required_to_preview: + params['RequiredToPreview'] = "true" + return params + +class PercentAssignmentsSubmittedRequirement(Requirement): + """ + The percentage of assignments the Worker has submitted, over all assignments the Worker has accepted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="00000000000000000000", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsAbandonedRequirement(Requirement): + """ + The percentage of assignments the Worker has abandoned (allowed the deadline to elapse), over all assignments the Worker has accepted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="00000000000000000070", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsReturnedRequirement(Requirement): + """ + The percentage of assignments the Worker has returned, over all assignments the Worker has accepted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="000000000000000000E0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsApprovedRequirement(Requirement): + """ + The percentage of assignments the Worker has submitted that were subsequently approved by the Requester, over all assignments the Worker has submitted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="000000000000000000L0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsRejectedRequirement(Requirement): + """ + The percentage of assignments the Worker has submitted that were subsequently rejected by the Requester, over all assignments the Worker has submitted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="000000000000000000S0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class LocaleRequirement(Requirement): + """ + A Qualification requirement based on the Worker's location. The Worker's location is specified by the Worker to Mechanical Turk when the Worker creates his account. + """ + + def __init__(self, comparator, locale, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="00000000000000000071", comparator=comparator, integer_value=None, required_to_preview=required_to_preview) + self.locale = locale + + def get_as_params(self): + params = { + "QualificationTypeId": self.qualification_type_id, + "Comparator": self.comparator, + 'LocaleValue.Country': self.locale, + } + if self.required_to_preview: + params['RequiredToPreview'] = "true" + return params diff --git a/storefront/boto/mturk/question.py b/storefront/boto/mturk/question.py new file mode 100644 index 0000000..89f1a45 --- /dev/null +++ b/storefront/boto/mturk/question.py @@ -0,0 +1,353 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Question(object): + + QUESTION_XML_TEMPLATE = """%s%s%s%s%s""" + DISPLAY_NAME_XML_TEMPLATE = """%s""" + + def __init__(self, identifier, content, answer_spec, is_required=False, display_name=None): #amount=0.0, currency_code='USD'): + self.identifier = identifier + self.content = content + self.answer_spec = answer_spec + self.is_required = is_required + self.display_name = display_name + + def get_as_params(self, label='Question', identifier=None): + + if identifier is None: + raise ValueError("identifier (QuestionIdentifier) is required per MTurk spec.") + + return { label : self.get_as_xml() } + + def get_as_xml(self): + # add the display name if required + display_name_xml = '' + if self.display_name: + display_name_xml = self.DISPLAY_NAME_XML_TEMPLATE %(self.display_name) + + ret = Question.QUESTION_XML_TEMPLATE % (self.identifier, + display_name_xml, + str(self.is_required).lower(), + self.content.get_as_xml(), + self.answer_spec.get_as_xml()) + + return ret + +class ExternalQuestion(object): + + EXTERNAL_QUESTIONFORM_SCHEMA_LOCATION = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd" + EXTERNAL_QUESTION_XML_TEMPLATE = """%s%s""" + + def __init__(self, external_url, frame_height): + self.external_url = external_url + self.frame_height = frame_height + + def get_as_params(self, label='ExternalQuestion'): + return { label : self.get_as_xml() } + + def get_as_xml(self): + ret = ExternalQuestion.EXTERNAL_QUESTION_XML_TEMPLATE % (ExternalQuestion.EXTERNAL_QUESTIONFORM_SCHEMA_LOCATION, + self.external_url, + self.frame_height) + return ret + +class OrderedContent(object): + def __init__(self): + self.items = [] + + def append(self, field, value): + "Expects field type and value" + self.items.append((field, value)) + + def get_binary_xml(self, field, value): + return """ + + + %s + %s + + %s + %s +""" % (value['binary_type'], + value['binary_subtype'], + value['binary'], + value['binary_alttext']) + + def get_application_xml(self, field, value): + raise NotImplementedError("Application question content is not yet supported.") + + def get_as_xml(self): + default_handler = lambda f,v: '<%s>%s' % (f,v,f) + bulleted_list_handler = lambda _,list: '%s' % ''.join([('%s' % item) for item in list]) + formatted_content_handler = lambda _,content: "" % content + application_handler = self.get_application_xml + binary_handler = self.get_binary_xml + + children = '' + for (field,value) in self.items: + handler = default_handler + if field == 'List': + handler = bulleted_list_handler + elif field == 'Application': + handler = application_handler + elif field == 'Binary': + handler = binary_handler + elif field == 'FormattedContent': + handler = formatted_content_handler + children = children + handler(field, value) + + return children + +class Overview(object): + OVERVIEW_XML_TEMPLATE = """%s""" + + def __init__(self): + self.ordered_content = OrderedContent() + + def append(self, field, value): + self.ordered_content.append(field,value) + + def get_as_params(self, label='Overview'): + return { label : self.get_as_xml() } + + def get_as_xml(self): + ret = Overview.OVERVIEW_XML_TEMPLATE % (self.ordered_content.get_as_xml()) + + return ret + + +class QuestionForm(object): + + QUESTIONFORM_SCHEMA_LOCATION = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionForm.xsd" + QUESTIONFORM_XML_TEMPLATE = """%s""" # % (ns, questions_xml) + + def __init__(self, questions=None, overview=None): + if questions is None or type(questions) is not list: + raise ValueError("Must pass a list of Question instances to QuestionForm constructor") + else: + self.questions = questions + self.overview = overview + + + def get_as_xml(self): + if self.overview: + overview_xml = self.overview.get_as_xml() + questions_xml = "".join([q.get_as_xml() for q in self.questions]) + qf_xml = overview_xml + questions_xml + return QuestionForm.QUESTIONFORM_XML_TEMPLATE % (QuestionForm.QUESTIONFORM_SCHEMA_LOCATION, qf_xml) + + #def startElement(self, name, attrs, connection): + # return None + # + #def endElement(self, name, value, connection): + # + # #if name == 'Amount': + # # self.amount = float(value) + # #elif name == 'CurrencyCode': + # # self.currency_code = value + # #elif name == 'FormattedPrice': + # # self.formatted_price = value + # + # pass # What's this method for? I don't get it. + +class QuestionContent(object): + QUESTIONCONTENT_XML_TEMPLATE = """%s""" + + def __init__(self): + self.ordered_content = OrderedContent() + + def append(self, field, value): + self.ordered_content.append(field,value) + + def get_as_xml(self): + ret = QuestionContent.QUESTIONCONTENT_XML_TEMPLATE % (self.ordered_content.get_as_xml()) + + return ret + + +class AnswerSpecification(object): + + ANSWERSPECIFICATION_XML_TEMPLATE = """%s""" + + def __init__(self, spec): + self.spec = spec + def get_as_xml(self): + values = () # TODO + return AnswerSpecification.ANSWERSPECIFICATION_XML_TEMPLATE % self.spec.get_as_xml() + +class FreeTextAnswer(object): + + FREETEXTANSWER_XML_TEMPLATE = """%s%s""" # (constraints, default) + FREETEXTANSWER_CONSTRAINTS_XML_TEMPLATE = """%s%s%s""" # (is_numeric_xml, length_xml, regex_xml) + FREETEXTANSWER_LENGTH_XML_TEMPLATE = """""" # (min_length_attr, max_length_attr) + FREETEXTANSWER_ISNUMERIC_XML_TEMPLATE = """""" # (min_value_attr, max_value_attr) + FREETEXTANSWER_DEFAULTTEXT_XML_TEMPLATE = """%s""" # (default) + + def __init__(self, default=None, min_length=None, max_length=None, is_numeric=False, min_value=None, max_value=None, format_regex=None): + self.default = default + self.min_length = min_length + self.max_length = max_length + self.is_numeric = is_numeric + self.min_value = min_value + self.max_value = max_value + self.format_regex = format_regex + + def get_as_xml(self): + is_numeric_xml = "" + if self.is_numeric: + min_value_attr = "" + max_value_attr = "" + if self.min_value: + min_value_attr = """minValue="%d" """ % self.min_value + if self.max_value: + max_value_attr = """maxValue="%d" """ % self.max_value + is_numeric_xml = FreeTextAnswer.FREETEXTANSWER_ISNUMERIC_XML_TEMPLATE % (min_value_attr, max_value_attr) + + length_xml = "" + if self.min_length or self.max_length: + min_length_attr = "" + max_length_attr = "" + if self.min_length: + min_length_attr = """minLength="%d" """ + if self.max_length: + max_length_attr = """maxLength="%d" """ + length_xml = FreeTextAnswer.FREETEXTANSWER_LENGTH_XML_TEMPLATE % (min_length_attr, max_length_attr) + + regex_xml = "" + if self.format_regex: + format_regex_attribs = '''regex="%s"''' %self.format_regex['regex'] + + error_text = self.format_regex.get('error_text', None) + if error_text: + format_regex_attribs += ' errorText="%s"' %error_text + + flags = self.format_regex.get('flags', None) + if flags: + format_regex_attribs += ' flags="%s"' %flags + + regex_xml = """""" %format_regex_attribs + + constraints_xml = "" + if is_numeric_xml or length_xml or regex_xml: + constraints_xml = FreeTextAnswer.FREETEXTANSWER_CONSTRAINTS_XML_TEMPLATE % (is_numeric_xml, length_xml, regex_xml) + + default_xml = "" + if self.default is not None: + default_xml = FreeTextAnswer.FREETEXTANSWER_DEFAULTTEXT_XML_TEMPLATE % self.default + + return FreeTextAnswer.FREETEXTANSWER_XML_TEMPLATE % (constraints_xml, default_xml) + +class FileUploadAnswer(object): + FILEUPLOADANSWER_XML_TEMLPATE = """%d%d""" # (min, max) + DEFAULT_MIN_SIZE = 1024 # 1K (completely arbitrary!) + DEFAULT_MAX_SIZE = 5 * 1024 * 1024 # 5MB (completely arbitrary!) + + def __init__(self, min=None, max=None): + self.min = min + self.max = max + if self.min is None: + self.min = FileUploadAnswer.DEFAULT_MIN_SIZE + if self.max is None: + self.max = FileUploadAnswer.DEFAULT_MAX_SIZE + + def get_as_xml(self): + return FileUploadAnswer.FILEUPLOADANSWER_XML_TEMLPATE % (self.min, self.max) + +class SelectionAnswer(object): + """ + A class to generate SelectionAnswer XML data structures. + Does not yet implement Binary selection options. + """ + SELECTIONANSWER_XML_TEMPLATE = """%s%s%s""" # % (count_xml, style_xml, selections_xml) + SELECTION_XML_TEMPLATE = """%s%s""" # (identifier, value_xml) + SELECTION_VALUE_XML_TEMPLATE = """<%s>%s""" # (type, value, type) + STYLE_XML_TEMPLATE = """%s""" # (style) + MIN_SELECTION_COUNT_XML_TEMPLATE = """%s""" # count + MAX_SELECTION_COUNT_XML_TEMPLATE = """%s""" # count + ACCEPTED_STYLES = ['radiobutton', 'dropdown', 'checkbox', 'list', 'combobox', 'multichooser'] + OTHER_SELECTION_ELEMENT_NAME = 'OtherSelection' + + def __init__(self, min=1, max=1, style=None, selections=None, type='text', other=False): + + if style is not None: + if style in SelectionAnswer.ACCEPTED_STYLES: + self.style_suggestion = style + else: + raise ValueError("style '%s' not recognized; should be one of %s" % (style, ', '.join(SelectionAnswer.ACCEPTED_STYLES))) + else: + self.style_suggestion = None + + if selections is None: + raise ValueError("SelectionAnswer.__init__(): selections must be a non-empty list of (content, identifier) tuples") + else: + self.selections = selections + + self.min_selections = min + self.max_selections = max + + assert len(selections) >= self.min_selections, "# of selections is less than minimum of %d" % self.min_selections + #assert len(selections) <= self.max_selections, "# of selections exceeds maximum of %d" % self.max_selections + + self.type = type + + self.other = other + + def get_as_xml(self): + xml = "" + if self.type == 'text': + TYPE_TAG = "Text" + elif self.type == 'binary': + TYPE_TAG = "Binary" + else: + raise ValueError("illegal type: %s; must be either 'text' or 'binary'" % str(self.type)) + + # build list of elements + selections_xml = "" + for tpl in self.selections: + value_xml = SelectionAnswer.SELECTION_VALUE_XML_TEMPLATE % (TYPE_TAG, tpl[0], TYPE_TAG) + selection_xml = SelectionAnswer.SELECTION_XML_TEMPLATE % (tpl[1], value_xml) + selections_xml += selection_xml + + if self.other: + # add OtherSelection element as xml if available + if hasattr(self.other, 'get_as_xml'): + assert type(self.other) == FreeTextAnswer, 'OtherSelection can only be a FreeTextAnswer' + selections_xml += self.other.get_as_xml().replace('FreeTextAnswer', 'OtherSelection') + else: + selections_xml += "" + + if self.style_suggestion is not None: + style_xml = SelectionAnswer.STYLE_XML_TEMPLATE % self.style_suggestion + else: + style_xml = "" + + if self.style_suggestion != 'radiobutton': + count_xml = SelectionAnswer.MIN_SELECTION_COUNT_XML_TEMPLATE %self.min_selections + count_xml += SelectionAnswer.MAX_SELECTION_COUNT_XML_TEMPLATE %self.max_selections + else: + count_xml = "" + + ret = SelectionAnswer.SELECTIONANSWER_XML_TEMPLATE % (count_xml, style_xml, selections_xml) + + # return XML + return ret + diff --git a/storefront/boto/mturk/test/all_tests.py b/storefront/boto/mturk/test/all_tests.py new file mode 100644 index 0000000..a8f291a --- /dev/null +++ b/storefront/boto/mturk/test/all_tests.py @@ -0,0 +1,8 @@ +import doctest + +# doctest.testfile("create_hit.doctest") +# doctest.testfile("create_hit_binary.doctest") +doctest.testfile("create_free_text_question_regex.doctest") +# doctest.testfile("create_hit_from_hit_type.doctest") +# doctest.testfile("search_hits.doctest") +# doctest.testfile("reviewable_hits.doctest") diff --git a/storefront/boto/mturk/test/cleanup_tests.py b/storefront/boto/mturk/test/cleanup_tests.py new file mode 100644 index 0000000..7bdff90 --- /dev/null +++ b/storefront/boto/mturk/test/cleanup_tests.py @@ -0,0 +1,67 @@ +from boto.mturk.connection import MTurkConnection + +def cleanup(): + """Remove any boto test related HIT's""" + + conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + current_page = 1 + page_size = 10 + total_disabled = 0 + ignored = [] + + while True: + # reset the total for this loop + disabled_count = 0 + + # search all the hits in the sandbox + search_rs = conn.search_hits(page_size=page_size, page_number=current_page) + + # success? + if search_rs.status: + for hit in search_rs: + # delete any with Boto in the description + print 'hit id:%s Status:%s, desc:%s' %(hit.HITId, hit.HITStatus, hit.Description) + if hit.Description.find('Boto') != -1: + if hit.HITStatus != 'Reviewable': + print 'Disabling hit id:%s %s' %(hit.HITId, hit.Description) + disable_rs = conn.disable_hit(hit.HITId) + if disable_rs.status: + disabled_count += 1 + # update the running total + total_disabled += 1 + else: + print 'Error when disabling, code:%s, message:%s' %(disable_rs.Code, disable_rs.Message) + else: + print 'Disposing hit id:%s %s' %(hit.HITId, hit.Description) + dispose_rs = conn.dispose_hit(hit.HITId) + if dispose_rs.status: + disabled_count += 1 + # update the running total + total_disabled += 1 + else: + print 'Error when disposing, code:%s, message:%s' %(dispose_rs.Code, dispose_rs.Message) + + else: + if hit.HITId not in ignored: + print 'ignored:%s' %hit.HITId + ignored.append(hit.HITId) + + # any more results? + if int(search_rs.TotalNumResults) > current_page*page_size: + # if we have disabled any HITs on this page + # then we don't need to go to a new page + # otherwise we do + if not disabled_count: + current_page += 1 + else: + # no, we're done + break + else: + print 'Error performing search, code:%s, message:%s' %(search_rs.Code, search_rs.Message) + break + + total_ignored = len(ignored) + print 'Processed: %d HITs, disabled/disposed: %d, ignored: %d' %(total_ignored + total_disabled, total_disabled, total_ignored) + +if __name__ == '__main__': + cleanup() diff --git a/storefront/boto/mturk/test/create_free_text_question_regex.doctest b/storefront/boto/mturk/test/create_free_text_question_regex.doctest new file mode 100644 index 0000000..a10b7ed --- /dev/null +++ b/storefront/boto/mturk/test/create_free_text_question_regex.doctest @@ -0,0 +1,92 @@ +>>> import uuid +>>> import datetime +>>> from boto.mturk.connection import MTurkConnection +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer + +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + +# create content for a question +>>> qn_content = QuestionContent(title='Boto no hit type question content', +... text='What is a boto no hit type?') + +# create a free text answer that is not quite so free! +>>> ft_answer = FreeTextAnswer(format_regex=dict(regex="^[12][0-9]{3}-[01]?\d-[0-3]?\d$", +... error_text="You must enter a date with the format yyyy-mm-dd.", +... flags="i"), +... default="This is not a valid format") + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(ft_answer)) + +# now, create the actual HIT for the question without using a HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> keywords=['boto', 'test', 'doctest'] +>>> create_hit_rs = conn.create_hit(question=qn, +... lifetime=60*65, +... max_assignments=2, +... title='Boto create_hit title', +... description='Boto create_hit description', +... keywords=keywords, +... reward=0.23, +... duration=60*6, +... approval_delay=60*60, +... annotation='An annotation from boto create_hit test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +# for the requested hit type id +# the HIT Type Id is a unicode string +>>> hit_type_id = create_hit_rs.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +>>> create_hit_rs.MaxAssignments +u'2' + +>>> create_hit_rs.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime in seconds +>>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) +>>> expiration_datetime = datetime.datetime.strptime(create_hit_rs.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> delta.seconds < 5 +True + +# duration is as specified for the HIT type +>>> create_hit_rs.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly (allow for float error here) +>>> int(create_hit_rs[0].amount * 100) +23 + +>>> create_hit_rs[0].formatted_price +u'$0.23' + +# only US currency supported at present +>>> create_hit_rs[0].currency_code +u'USD' + +# title is the HIT type title +>>> create_hit_rs.Title +u'Boto create_hit title' + +# title is the HIT type description +>>> create_hit_rs.Description +u'Boto create_hit description' + +# annotation is correct +>>> create_hit_rs.RequesterAnnotation +u'An annotation from boto create_hit test' + +>>> create_hit_rs.HITReviewStatus +u'NotReviewed' diff --git a/storefront/boto/mturk/test/create_hit.doctest b/storefront/boto/mturk/test/create_hit.doctest new file mode 100644 index 0000000..22209d6 --- /dev/null +++ b/storefront/boto/mturk/test/create_hit.doctest @@ -0,0 +1,86 @@ +>>> import uuid +>>> import datetime +>>> from boto.mturk.connection import MTurkConnection +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer + +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + +# create content for a question +>>> qn_content = QuestionContent(title='Boto no hit type question content', +... text='What is a boto no hit type?') + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(FreeTextAnswer())) + +# now, create the actual HIT for the question without using a HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> keywords=['boto', 'test', 'doctest'] +>>> create_hit_rs = conn.create_hit(question=qn, +... lifetime=60*65, +... max_assignments=2, +... title='Boto create_hit title', +... description='Boto create_hit description', +... keywords=keywords, +... reward=0.23, +... duration=60*6, +... approval_delay=60*60, +... annotation='An annotation from boto create_hit test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +# for the requested hit type id +# the HIT Type Id is a unicode string +>>> hit_type_id = create_hit_rs.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +>>> create_hit_rs.MaxAssignments +u'2' + +>>> create_hit_rs.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime in seconds +>>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) +>>> expiration_datetime = datetime.datetime.strptime(create_hit_rs.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> delta.seconds < 5 +True + +# duration is as specified for the HIT type +>>> create_hit_rs.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly (allow for float error here) +>>> int(create_hit_rs[0].amount * 100) +23 + +>>> create_hit_rs[0].formatted_price +u'$0.23' + +# only US currency supported at present +>>> create_hit_rs[0].currency_code +u'USD' + +# title is the HIT type title +>>> create_hit_rs.Title +u'Boto create_hit title' + +# title is the HIT type description +>>> create_hit_rs.Description +u'Boto create_hit description' + +# annotation is correct +>>> create_hit_rs.RequesterAnnotation +u'An annotation from boto create_hit test' + +>>> create_hit_rs.HITReviewStatus +u'NotReviewed' diff --git a/storefront/boto/mturk/test/create_hit_binary.doctest b/storefront/boto/mturk/test/create_hit_binary.doctest new file mode 100644 index 0000000..3096083 --- /dev/null +++ b/storefront/boto/mturk/test/create_hit_binary.doctest @@ -0,0 +1,87 @@ +>>> import uuid +>>> import datetime +>>> from boto.mturk.connection import MTurkConnection +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer + +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + +# create content for a question +>>> qn_content = QuestionContent(title='Boto no hit type question content', +... text='What is a boto no hit type?', +... binary='http://www.example.com/test1.jpg') + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(FreeTextAnswer())) + +# now, create the actual HIT for the question without using a HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> keywords=['boto', 'test', 'doctest'] +>>> create_hit_rs = conn.create_hit(question=qn, +... lifetime=60*65, +... max_assignments=2, +... title='Boto create_hit title', +... description='Boto create_hit description', +... keywords=keywords, +... reward=0.23, +... duration=60*6, +... approval_delay=60*60, +... annotation='An annotation from boto create_hit test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +# for the requested hit type id +# the HIT Type Id is a unicode string +>>> hit_type_id = create_hit_rs.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +>>> create_hit_rs.MaxAssignments +u'2' + +>>> create_hit_rs.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime in seconds +>>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) +>>> expiration_datetime = datetime.datetime.strptime(create_hit_rs.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> delta.seconds < 5 +True + +# duration is as specified for the HIT type +>>> create_hit_rs.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly (allow for float error here) +>>> int(create_hit_rs[0].amount * 100) +23 + +>>> create_hit_rs[0].formatted_price +u'$0.23' + +# only US currency supported at present +>>> create_hit_rs[0].currency_code +u'USD' + +# title is the HIT type title +>>> create_hit_rs.Title +u'Boto create_hit title' + +# title is the HIT type description +>>> create_hit_rs.Description +u'Boto create_hit description' + +# annotation is correct +>>> create_hit_rs.RequesterAnnotation +u'An annotation from boto create_hit test' + +>>> create_hit_rs.HITReviewStatus +u'NotReviewed' diff --git a/storefront/boto/mturk/test/create_hit_external.py b/storefront/boto/mturk/test/create_hit_external.py new file mode 100644 index 0000000..e7425d6 --- /dev/null +++ b/storefront/boto/mturk/test/create_hit_external.py @@ -0,0 +1,14 @@ +import uuid +import datetime +from boto.mturk.connection import MTurkConnection +from boto.mturk.question import ExternalQuestion + +def test(): + q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800) + conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + keywords=['boto', 'test', 'doctest'] + create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', response_groups=['Minimal','HITDetail','HITQuestion','HITAssignmentSummary',]) + assert(create_hit_rs.status == True) + +if __name__ == "__main__": + test() diff --git a/storefront/boto/mturk/test/create_hit_from_hit_type.doctest b/storefront/boto/mturk/test/create_hit_from_hit_type.doctest new file mode 100644 index 0000000..144a677 --- /dev/null +++ b/storefront/boto/mturk/test/create_hit_from_hit_type.doctest @@ -0,0 +1,97 @@ +>>> import uuid +>>> import datetime +>>> from boto.mturk.connection import MTurkConnection +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer +>>> +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') +>>> keywords=['boto', 'test', 'doctest'] +>>> hit_type_rs = conn.register_hit_type('Boto Test HIT type', +... 'HIT Type for testing Boto', +... 0.12, +... 60*6, +... keywords=keywords, +... approval_delay=60*60) + +# this was a valid request +>>> hit_type_rs.status +True + +# the HIT Type Id is a unicode string +>>> hit_type_id = hit_type_rs.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +# create content for a question +>>> qn_content = QuestionContent(title='Boto question content create_hit_from_hit_type', +... text='What is a boto create_hit_from_hit_type?') + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(FreeTextAnswer())) + +# now, create the actual HIT for the question using the HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> create_hit_rs = conn.create_hit(hit_type=hit_type_rs.HITTypeId, +... question=qn, +... lifetime=60*65, +... max_assignments=2, +... annotation='An annotation from boto create_hit_from_hit_type test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +# for the requested hit type id +>>> create_hit_rs.HITTypeId == hit_type_id +True + +# with the correct number of maximum assignments +>>> create_hit_rs.MaxAssignments +u'2' + +# and the approval delay +>>> create_hit_rs.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime in seconds +>>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) +>>> expiration_datetime = datetime.datetime.strptime(create_hit_rs.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> delta.seconds < 5 +True + +# duration is as specified for the HIT type +>>> create_hit_rs.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly +>>> create_hit_rs[0].amount +0.12 + +>>> create_hit_rs[0].formatted_price +u'$0.12' + +# only US currency supported at present +>>> create_hit_rs[0].currency_code +u'USD' + +# title is the HIT type title +>>> create_hit_rs.Title +u'Boto Test HIT type' + +# title is the HIT type description +>>> create_hit_rs.Description +u'HIT Type for testing Boto' + +# annotation is correct +>>> create_hit_rs.RequesterAnnotation +u'An annotation from boto create_hit_from_hit_type test' + +# not reviewed yet +>>> create_hit_rs.HITReviewStatus +u'NotReviewed' diff --git a/storefront/boto/mturk/test/create_hit_with_qualifications.py b/storefront/boto/mturk/test/create_hit_with_qualifications.py new file mode 100644 index 0000000..f2149ee --- /dev/null +++ b/storefront/boto/mturk/test/create_hit_with_qualifications.py @@ -0,0 +1,18 @@ +import uuid +import datetime +from boto.mturk.connection import MTurkConnection +from boto.mturk.question import ExternalQuestion +from boto.mturk.qualification import Qualifications, PercentAssignmentsApprovedRequirement + +def test(): + q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800) + conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + keywords=['boto', 'test', 'doctest'] + qualifications = Qualifications() + qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95")) + create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications) + assert(create_hit_rs.status == True) + print create_hit_rs.HITTypeId + +if __name__ == "__main__": + test() diff --git a/storefront/boto/mturk/test/reviewable_hits.doctest b/storefront/boto/mturk/test/reviewable_hits.doctest new file mode 100644 index 0000000..0305901 --- /dev/null +++ b/storefront/boto/mturk/test/reviewable_hits.doctest @@ -0,0 +1,71 @@ +>>> from boto.mturk.connection import MTurkConnection +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + +# should have some reviewable HIT's returned, especially if returning all HIT type's +# NOTE: but only if your account has existing HIT's in the reviewable state +>>> reviewable_rs = conn.get_reviewable_hits() + +# this is a valid request +>>> reviewable_rs.status +True + +>>> len(reviewable_rs) > 1 +True + +# should contain at least one HIT object +>>> reviewable_rs # doctest: +ELLIPSIS +[>> hit_id = reviewable_rs[0].HITId + +# check that we can retrieve the assignments for a HIT +>>> assignments_rs = conn.get_assignments(hit_id) + +# this is a valid request +>>> assignments_rs.status +True + +>>> assignments_rs.NumResults >= 1 +True + +>>> len(assignments_rs) == int(assignments_rs.NumResults) +True + +>>> assignments_rs.PageNumber +u'1' + +>>> assignments_rs.TotalNumResults >= 1 +True + +# should contain at least one Assignment object +>>> assignments_rs # doctest: +ELLIPSIS +[>> assignment = assignments_rs[0] + +>>> assignment.HITId == hit_id +True + +# should have a valid status +>>> assignment.AssignmentStatus in ['Submitted', 'Approved', 'Rejected'] +True + +# should have returned at least one answer +>>> len(assignment.answers) > 0 +True + +# should contain at least one set of QuestionFormAnswer objects +>>> assignment.answers # doctest: +ELLIPSIS +[[>> answer = assignment.answers[0][0] + +# answer should be a FreeTextAnswer +>>> answer.FreeText # doctest: +ELLIPSIS +u'...' + +# question identifier should be a unicode string +>>> answer.QuestionIdentifier # doctest: +ELLIPSIS +u'...' + diff --git a/storefront/boto/mturk/test/search_hits.doctest b/storefront/boto/mturk/test/search_hits.doctest new file mode 100644 index 0000000..a2547ea --- /dev/null +++ b/storefront/boto/mturk/test/search_hits.doctest @@ -0,0 +1,16 @@ +>>> from boto.mturk.connection import MTurkConnection +>>> conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + +# should have some HIT's returned by a search (but only if your account has existing HIT's) +>>> search_rs = conn.search_hits() + +# this is a valid request +>>> search_rs.status +True + +>>> len(search_rs) > 1 +True + +>>> search_rs # doctest: +ELLIPSIS +[= 0: + method, version = update.split(':') + version = '-r%s' % version + else: + version = '-rHEAD' + location = boto.config.get('Boto', 'boto_location', '/usr/local/boto') + self.run('svn update %s %s' % (version, location)) + else: + # first remove the symlink needed when running from subversion + self.run('rm /usr/local/lib/python2.5/site-packages/boto') + self.run('easy_install %s' % update) + + def fetch_s3_file(self, s3_file): + try: + if s3_file.startswith('s3:'): + bucket_name, key_name = s3_file[len('s3:'):].split('/') + c = boto.connect_s3() + bucket = c.get_bucket(bucket_name) + key = bucket.get_key(key_name) + boto.log.info('Fetching %s/%s' % (bucket.name, key.name)) + path = os.path.join(self.working_dir, key.name) + key.get_contents_to_filename(path) + except: + boto.log.exception('Problem Retrieving file: %s' % s3_file) + path = None + return path + + def load_packages(self): + package_str = boto.config.get('Pyami', 'packages') + if package_str: + packages = package_str.split(',') + for package in packages: + package = package.strip() + if package.startswith('s3:'): + package = self.fetch_s3_file(package) + if package: + # if the "package" is really a .py file, it doesn't have to + # be installed, just being in the working dir is enough + if not package.endswith('.py'): + self.run('easy_install -Z %s' % package, exit_on_error=False) + + def main(self): + self.create_working_dir() + self.load_boto() + self.load_packages() + self.notify('Bootstrap Completed for %s' % boto.config.get_instance('instance-id')) + +if __name__ == "__main__": + # because bootstrap starts before any logging configuration can be loaded from + # the boto config files, we will manually enable logging to /var/log/boto.log + boto.set_file_logger('bootstrap', '/var/log/boto.log') + bs = Bootstrap() + bs.main() diff --git a/storefront/boto/pyami/config.py b/storefront/boto/pyami/config.py new file mode 100644 index 0000000..831acc4 --- /dev/null +++ b/storefront/boto/pyami/config.py @@ -0,0 +1,204 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import StringIO, os, re +import ConfigParser +import boto + +BotoConfigPath = '/etc/boto.cfg' +BotoConfigLocations = [BotoConfigPath] +if 'HOME' in os.environ: + UserConfigPath = os.path.expanduser('~/.boto') + BotoConfigLocations.append(UserConfigPath) +else: + UserConfigPath = None +if 'BOTO_CONFIG' in os.environ: + BotoConfigLocations.append(os.path.expanduser(os.environ['BOTO_CONFIG'])) + +class Config(ConfigParser.SafeConfigParser): + + def __init__(self, path=None, fp=None, do_load=True): + ConfigParser.SafeConfigParser.__init__(self, {'working_dir' : '/mnt/pyami', + 'debug' : '0'}) + if do_load: + if path: + self.load_from_path(path) + elif fp: + self.readfp(fp) + else: + self.read(BotoConfigLocations) + if "AWS_CREDENTIAL_FILE" in os.environ: + self.load_credential_file(os.path.expanduser(os.environ['AWS_CREDENTIAL_FILE'])) + + def load_credential_file(self, path): + """Load a credential file as is setup like the Java utilities""" + config = ConfigParser.ConfigParser() + c_data = StringIO.StringIO() + c_data.write("[Credentials]\n") + for line in open(path, "r").readlines(): + c_data.write(line.replace("AWSAccessKeyId", "aws_access_key_id").replace("AWSSecretKey", "aws_secret_access_key")) + c_data.seek(0) + self.readfp(c_data) + + def load_from_path(self, path): + file = open(path) + for line in file.readlines(): + match = re.match("^#import[\s\t]*([^\s^\t]*)[\s\t]*$", line) + if match: + extended_file = match.group(1) + (dir, file) = os.path.split(path) + self.load_from_path(os.path.join(dir, extended_file)) + self.read(path) + + def save_option(self, path, section, option, value): + """ + Write the specified Section.Option to the config file specified by path. + Replace any previous value. If the path doesn't exist, create it. + Also add the option the the in-memory config. + """ + config = ConfigParser.SafeConfigParser() + config.read(path) + if not config.has_section(section): + config.add_section(section) + config.set(section, option, value) + fp = open(path, 'w') + config.write(fp) + fp.close() + if not self.has_section(section): + self.add_section(section) + self.set(section, option, value) + + def save_user_option(self, section, option, value): + self.save_option(UserConfigPath, section, option, value) + + def save_system_option(self, section, option, value): + self.save_option(BotoConfigPath, section, option, value) + + def get_instance(self, name, default=None): + try: + val = self.get('Instance', name) + except: + val = default + return val + + def get_user(self, name, default=None): + try: + val = self.get('User', name) + except: + val = default + return val + + def getint_user(self, name, default=0): + try: + val = self.getint('User', name) + except: + val = default + return val + + def get_value(self, section, name, default=None): + return self.get(section, name, default) + + def get(self, section, name, default=None): + try: + val = ConfigParser.SafeConfigParser.get(self, section, name) + except: + val = default + return val + + def getint(self, section, name, default=0): + try: + val = ConfigParser.SafeConfigParser.getint(self, section, name) + except: + val = int(default) + return val + + def getfloat(self, section, name, default=0.0): + try: + val = ConfigParser.SafeConfigParser.getfloat(self, section, name) + except: + val = float(default) + return val + + def getbool(self, section, name, default=False): + if self.has_option(section, name): + val = self.get(section, name) + if val.lower() == 'true': + val = True + else: + val = False + else: + val = default + return val + + def setbool(self, section, name, value): + if value: + self.set(section, name, 'true') + else: + self.set(section, name, 'false') + + def dump(self): + s = StringIO.StringIO() + self.write(s) + print s.getvalue() + + def dump_safe(self, fp=None): + if not fp: + fp = StringIO.StringIO() + for section in self.sections(): + fp.write('[%s]\n' % section) + for option in self.options(section): + if option == 'aws_secret_access_key': + fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option) + else: + fp.write('%s = %s\n' % (option, self.get(section, option))) + + def dump_to_sdb(self, domain_name, item_name): + import simplejson + sdb = boto.connect_sdb() + domain = sdb.lookup(domain_name) + if not domain: + domain = sdb.create_domain(domain_name) + item = domain.new_item(item_name) + item.active = False + for section in self.sections(): + d = {} + for option in self.options(section): + d[option] = self.get(section, option) + item[section] = simplejson.dumps(d) + item.save() + + def load_from_sdb(self, domain_name, item_name): + import simplejson + sdb = boto.connect_sdb() + domain = sdb.lookup(domain_name) + item = domain.get_item(item_name) + for section in item.keys(): + if not self.has_section(section): + self.add_section(section) + d = simplejson.loads(item[section]) + for attr_name in d.keys(): + attr_value = d[attr_name] + if attr_value == None: + attr_value = 'None' + if isinstance(attr_value, bool): + self.setbool(section, attr_name, attr_value) + else: + self.set(section, attr_name, attr_value) diff --git a/storefront/boto/pyami/copybot.cfg b/storefront/boto/pyami/copybot.cfg new file mode 100644 index 0000000..cbfdc5a --- /dev/null +++ b/storefront/boto/pyami/copybot.cfg @@ -0,0 +1,60 @@ +# +# Your AWS Credentials +# +[Credentials] +aws_access_key_id = +aws_secret_access_key = + +# +# If you want to use a separate set of credentials when writing +# to the destination bucket, put them here +#dest_aws_access_key_id = +#dest_aws_secret_access_key = + +# +# Fill out this section if you want emails from CopyBot +# when it starts and stops +# +[Notification] +#smtp_host = +#smtp_user = +#smtp_pass = +#smtp_from = +#smtp_to = + +# +# If you leave this section as is, it will automatically +# update boto from subversion upon start up. +# If you don't want that to happen, comment this out +# +[Boto] +boto_location = /usr/local/boto +boto_update = svn:HEAD + +# +# This tells the Pyami code in boto what scripts +# to run during startup +# +[Pyami] +scripts = boto.pyami.copybot.CopyBot + +# +# Source bucket and Destination Bucket, obviously. +# If the Destination bucket does not exist, it will +# attempt to create it. +# If exit_on_completion is false, the instance +# will keep running after the copy operation is +# complete which might be handy for debugging. +# If copy_acls is false, the ACL's will not be +# copied with the objects to the new bucket. +# If replace_dst is false, copybot will not +# will only store the source file in the dest if +# that file does not already exist. If it's true +# it will replace it even if it does exist. +# +[CopyBot] +src_bucket = +dst_bucket = +exit_on_completion = true +copy_acls = true +replace_dst = true diff --git a/storefront/boto/pyami/copybot.py b/storefront/boto/pyami/copybot.py new file mode 100644 index 0000000..ed397cb --- /dev/null +++ b/storefront/boto/pyami/copybot.py @@ -0,0 +1,97 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import boto +from boto.pyami.scriptbase import ScriptBase +import os, StringIO + +class CopyBot(ScriptBase): + + def __init__(self): + ScriptBase.__init__(self) + self.wdir = boto.config.get('Pyami', 'working_dir') + self.log_file = '%s.log' % self.instance_id + self.log_path = os.path.join(self.wdir, self.log_file) + boto.set_file_logger(self.name, self.log_path) + self.src_name = boto.config.get(self.name, 'src_bucket') + self.dst_name = boto.config.get(self.name, 'dst_bucket') + self.replace = boto.config.getbool(self.name, 'replace_dst', True) + s3 = boto.connect_s3() + self.src = s3.lookup(self.src_name) + if not self.src: + boto.log.error('Source bucket does not exist: %s' % self.src_name) + dest_access_key = boto.config.get(self.name, 'dest_aws_access_key_id', None) + if dest_access_key: + dest_secret_key = boto.config.get(self.name, 'dest_aws_secret_access_key', None) + s3 = boto.connect(dest_access_key, dest_secret_key) + self.dst = s3.lookup(self.dst_name) + if not self.dst: + self.dst = s3.create_bucket(self.dst_name) + + def copy_bucket_acl(self): + if boto.config.get(self.name, 'copy_acls', True): + acl = self.src.get_xml_acl() + self.dst.set_xml_acl(acl) + + def copy_key_acl(self, src, dst): + if boto.config.get(self.name, 'copy_acls', True): + acl = src.get_xml_acl() + dst.set_xml_acl(acl) + + def copy_keys(self): + boto.log.info('src=%s' % self.src.name) + boto.log.info('dst=%s' % self.dst.name) + try: + for key in self.src: + if not self.replace: + exists = self.dst.lookup(key.name) + if exists: + boto.log.info('key=%s already exists in %s, skipping' % (key.name, self.dst.name)) + continue + boto.log.info('copying %d bytes from key=%s' % (key.size, key.name)) + prefix, base = os.path.split(key.name) + path = os.path.join(self.wdir, base) + key.get_contents_to_filename(path) + new_key = self.dst.new_key(key.name) + new_key.set_contents_from_filename(path) + self.copy_key_acl(key, new_key) + os.unlink(path) + except: + boto.log.exception('Error copying key: %s' % key.name) + + def copy_log(self): + key = self.dst.new_key(self.log_file) + key.set_contents_from_filename(self.log_path) + + def main(self): + fp = StringIO.StringIO() + boto.config.dump_safe(fp) + self.notify('%s (%s) Starting' % (self.name, self.instance_id), fp.getvalue()) + if self.src and self.dst: + self.copy_keys() + if self.dst: + self.copy_log() + self.notify('%s (%s) Stopping' % (self.name, self.instance_id), + 'Copy Operation Complete') + if boto.config.getbool(self.name, 'exit_on_completion', True): + ec2 = boto.connect_ec2() + ec2.terminate_instances([self.instance_id]) + diff --git a/storefront/boto/pyami/helloworld.py b/storefront/boto/pyami/helloworld.py new file mode 100644 index 0000000..680873c --- /dev/null +++ b/storefront/boto/pyami/helloworld.py @@ -0,0 +1,28 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.scriptbase import ScriptBase + +class HelloWorld(ScriptBase): + + def main(self): + self.log('Hello World!!!') + diff --git a/storefront/boto/pyami/installers/__init__.py b/storefront/boto/pyami/installers/__init__.py new file mode 100644 index 0000000..4dcf2f4 --- /dev/null +++ b/storefront/boto/pyami/installers/__init__.py @@ -0,0 +1,64 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.scriptbase import ScriptBase +import os + +class Installer(ScriptBase): + """ + Abstract base class for installers + """ + + def add_cron(self, name, minute, hour, mday, month, wday, who, command, env=None): + """ + Add an entry to the system crontab. + """ + raise NotImplimented() + + def add_init_script(self, file): + """ + Add this file to the init.d directory + """ + + def add_env(self, key, value): + """ + Add an environemnt variable + """ + raise NotImplimented() + + def stop(self, service_name): + """ + Stop a service. + """ + raise NotImplimented() + + def start(self, service_name): + """ + Start a service. + """ + raise NotImplimented() + + def install(self): + """ + Do whatever is necessary to "install" the package. + """ + raise NotImplimented() + diff --git a/storefront/boto/pyami/installers/ubuntu/__init__.py b/storefront/boto/pyami/installers/ubuntu/__init__.py new file mode 100644 index 0000000..60ee658 --- /dev/null +++ b/storefront/boto/pyami/installers/ubuntu/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/storefront/boto/pyami/installers/ubuntu/apache.py b/storefront/boto/pyami/installers/ubuntu/apache.py new file mode 100644 index 0000000..febc2df --- /dev/null +++ b/storefront/boto/pyami/installers/ubuntu/apache.py @@ -0,0 +1,43 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.installers.ubuntu.installer import Installer + +class Apache(Installer): + """ + Install apache2, mod_python, and libapache2-svn + """ + + def install(self): + self.run("apt-get update") + self.run('apt-get -y install apache2', notify=True, exit_on_error=True) + self.run('apt-get -y install libapache2-mod-python', notify=True, exit_on_error=True) + self.run('a2enmod rewrite', notify=True, exit_on_error=True) + self.run('a2enmod ssl', notify=True, exit_on_error=True) + self.run('a2enmod proxy', notify=True, exit_on_error=True) + self.run('a2enmod proxy_ajp', notify=True, exit_on_error=True) + + # Hard reboot the apache2 server to enable these module + self.stop("apache2") + self.start("apache2") + + def main(self): + self.install() diff --git a/storefront/boto/pyami/installers/ubuntu/ebs.py b/storefront/boto/pyami/installers/ubuntu/ebs.py new file mode 100644 index 0000000..2cf0f22 --- /dev/null +++ b/storefront/boto/pyami/installers/ubuntu/ebs.py @@ -0,0 +1,203 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +Automated installer to attach, format and mount an EBS volume. +This installer assumes that you want the volume formatted as +an XFS file system. To drive this installer, you need the +following section in the boto config passed to the new instance. +You also need to install dateutil by listing python-dateutil +in the list of packages to be installed in the Pyami seciont +of your boto config file. + +If there is already a device mounted at the specified mount point, +the installer assumes that it is the ephemeral drive and unmounts +it, remounts it as /tmp and chmods it to 777. + +Config file section:: + + [EBS] + volume_id = + logical_volume_name = + device = + mount_point = + +""" +import boto +from boto.manage.volume import Volume +import os, time +from boto.pyami.installers.ubuntu.installer import Installer +from string import Template + +BackupScriptTemplate = """#!/usr/bin/env python +# Backup EBS volume +import boto +from boto.pyami.scriptbase import ScriptBase +import traceback + +class Backup(ScriptBase): + + def main(self): + try: + ec2 = boto.connect_ec2() + self.run("/usr/sbin/xfs_freeze -f ${mount_point}") + snapshot = ec2.create_snapshot('${volume_id}') + boto.log.info("Snapshot created: %s " % snapshot) + except Exception, e: + self.notify(subject="${instance_id} Backup Failed", body=traceback.format_exc()) + boto.log.info("Snapshot created: ${volume_id}") + except Exception, e: + self.notify(subject="${instance_id} Backup Failed", body=traceback.format_exc()) + finally: + self.run("/usr/sbin/xfs_freeze -u ${mount_point}") + +if __name__ == "__main__": + b = Backup() + b.main() +""" + +BackupCleanupScript= """#!/usr/bin/env python +# Cleans Backups of EBS volumes + +for v in Volume.all(): + v.trim_snapshot(True) +""" + +class EBSInstaller(Installer): + """ + Set up the EBS stuff + """ + + def __init__(self, config_file=None): + Installer.__init__(self, config_file) + self.instance_id = boto.config.get('Instance', 'instance-id') + self.device = boto.config.get('EBS', 'device', '/dev/sdp') + self.volume_id = boto.config.get('EBS', 'volume_id') + self.logical_volume_name = boto.config.get('EBS', 'logical_volume_name') + self.mount_point = boto.config.get('EBS', 'mount_point', '/ebs') + + def attach(self): + ec2 = boto.connect_ec2() + if self.logical_volume_name: + # if a logical volume was specified, override the specified volume_id + # (if there was one) with the current AWS volume for the logical volume: + logical_volume = Volume.find(name = self.logical_volume_name).next() + self.volume_id = logical_volume._volume_id + volume = ec2.get_all_volumes([self.volume_id])[0] + # wait for the volume to be available. The volume may still be being created + # from a snapshot. + while volume.update() != 'available': + boto.log.info('Volume %s not yet available. Current status = %s.' % (volume.id, volume.status)) + time.sleep(5) + ec2.attach_volume(self.volume_id, self.instance_id, self.device) + # now wait for the volume device to appear + while not os.path.exists(self.device): + boto.log.info('%s still does not exist, waiting 10 seconds' % self.device) + time.sleep(10) + + def make_fs(self): + boto.log.info('make_fs...') + has_fs = self.run('fsck %s' % self.device) + if has_fs != 0: + self.run('mkfs -t xfs %s' % self.device) + + def create_backup_script(self): + t = Template(BackupScriptTemplate) + s = t.substitute(volume_id=self.volume_id, instance_id=self.instance_id, + mount_point=self.mount_point) + fp = open('/usr/local/bin/ebs_backup', 'w') + fp.write(s) + fp.close() + self.run('chmod +x /usr/local/bin/ebs_backup') + + def create_backup_cleanup_script(self): + fp = open('/usr/local/bin/ebs_backup_cleanup', 'w') + fp.write(BackupCleanupScript) + fp.close() + self.run('chmod +x /usr/local/bin/ebs_backup_cleanup') + + def handle_mount_point(self): + boto.log.info('handle_mount_point') + if not os.path.isdir(self.mount_point): + boto.log.info('making directory') + # mount directory doesn't exist so create it + self.run("mkdir %s" % self.mount_point) + else: + boto.log.info('directory exists already') + self.run('mount -l') + lines = self.last_command.output.split('\n') + for line in lines: + t = line.split() + if t and t[2] == self.mount_point: + # something is already mounted at the mount point + # unmount that and mount it as /tmp + if t[0] != self.device: + self.run('umount %s' % self.mount_point) + self.run('mount %s /tmp' % t[0]) + self.run('chmod 777 /tmp') + break + # Mount up our new EBS volume onto mount_point + self.run("mount %s %s" % (self.device, self.mount_point)) + self.run('xfs_growfs %s' % self.mount_point) + + def update_fstab(self): + f = open("/etc/fstab", "a") + f.write('%s\t%s\txfs\tdefaults 0 0\n' % (self.mount_point, self.device)) + f.close() + + def install(self): + # First, find and attach the volume + self.attach() + + # Install the xfs tools + self.run('apt-get -y install xfsprogs xfsdump') + + # Check to see if the filesystem was created or not + self.make_fs() + + # create the /ebs directory for mounting + self.handle_mount_point() + + # create the backup script + self.create_backup_script() + + # Set up the backup script + minute = boto.config.get('EBS', 'backup_cron_minute', '0') + hour = boto.config.get('EBS', 'backup_cron_hour', '4,16') + self.add_cron("ebs_backup", "/usr/local/bin/ebs_backup", minute=minute, hour=hour) + + # Set up the backup cleanup script + minute = boto.config.get('EBS', 'backup_cleanup_cron_minute') + hour = boto.config.get('EBS', 'backup_cleanup_cron_hour') + if (minute != None) and (hour != None): + self.create_backup_cleanup_script(); + self.add_cron("ebs_backup_cleanup", "/usr/local/bin/ebs_backup_cleanup", minute=minute, hour=hour) + + # Set up the fstab + self.update_fstab() + + def main(self): + if not os.path.exists(self.device): + self.install() + else: + boto.log.info("Device %s is already attached, skipping EBS Installer" % self.device) diff --git a/storefront/boto/pyami/installers/ubuntu/installer.py b/storefront/boto/pyami/installers/ubuntu/installer.py new file mode 100644 index 0000000..0169950 --- /dev/null +++ b/storefront/boto/pyami/installers/ubuntu/installer.py @@ -0,0 +1,96 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import boto.pyami.installers +import os +import os.path +import stat +import boto +import random +from pwd import getpwnam + +class Installer(boto.pyami.installers.Installer): + """ + Base Installer class for Ubuntu-based AMI's + """ + def add_cron(self, name, command, minute="*", hour="*", mday="*", month="*", wday="*", who="root", env=None): + """ + Write a file to /etc/cron.d to schedule a command + env is a dict containing environment variables you want to set in the file + name will be used as the name of the file + """ + if minute == 'random': + minute = str(random.randrange(60)) + if hour == 'random': + hour = str(random.randrange(24)) + fp = open('/etc/cron.d/%s' % name, "w") + if env: + for key, value in env.items(): + fp.write('%s=%s\n' % (key, value)) + fp.write('%s %s %s %s %s %s %s\n' % (minute, hour, mday, month, wday, who, command)) + fp.close() + + def add_init_script(self, file, name): + """ + Add this file to the init.d directory + """ + f_path = os.path.join("/etc/init.d", name) + f = open(f_path, "w") + f.write(file) + f.close() + os.chmod(f_path, stat.S_IREAD| stat.S_IWRITE | stat.S_IEXEC) + self.run("/usr/sbin/update-rc.d %s defaults" % name) + + def add_env(self, key, value): + """ + Add an environemnt variable + For Ubuntu, the best place is /etc/environment. Values placed here do + not need to be exported. + """ + boto.log.info('Adding env variable: %s=%s' % (key, value)) + if not os.path.exists("/etc/environment.orig"): + self.run('cp /etc/environment /etc/environment.orig', notify=False, exit_on_error=False) + fp = open('/etc/environment', 'a') + fp.write('\n%s="%s"' % (key, value)) + fp.close() + os.environ[key] = value + + def stop(self, service_name): + self.run('/etc/init.d/%s stop' % service_name) + + def start(self, service_name): + self.run('/etc/init.d/%s start' % service_name) + + def create_user(self, user): + """ + Create a user on the local system + """ + self.run("useradd -m %s" % user) + usr = getpwnam(user) + return usr + + + def install(self): + """ + This is the only method you need to override + """ + raise NotImplimented() + diff --git a/storefront/boto/pyami/installers/ubuntu/mysql.py b/storefront/boto/pyami/installers/ubuntu/mysql.py new file mode 100644 index 0000000..490e5db --- /dev/null +++ b/storefront/boto/pyami/installers/ubuntu/mysql.py @@ -0,0 +1,109 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This installer will install mysql-server on an Ubuntu machine. +In addition to the normal installation done by apt-get, it will +also configure the new MySQL server to store it's data files in +a different location. By default, this is /mnt but that can be +configured in the [MySQL] section of the boto config file passed +to the instance. +""" +from boto.pyami.installers.ubuntu.installer import Installer +import os +import boto +from boto.utils import ShellCommand +from ConfigParser import SafeConfigParser +import time + +ConfigSection = """ +[MySQL] +root_password = +data_dir = +""" + +class MySQL(Installer): + + def install(self): + self.run('apt-get update') + self.run('apt-get -y install mysql-server', notify=True, exit_on_error=True) + +# def set_root_password(self, password=None): +# if not password: +# password = boto.config.get('MySQL', 'root_password') +# if password: +# self.run('mysqladmin -u root password %s' % password) +# return password + + def change_data_dir(self, password=None): + data_dir = boto.config.get('MySQL', 'data_dir', '/mnt') + fresh_install = False; + is_mysql_running_command = ShellCommand('mysqladmin ping') # exit status 0 if mysql is running + is_mysql_running_command.run() + if is_mysql_running_command.getStatus() == 0: + # mysql is running. This is the state apt-get will leave it in. If it isn't running, + # that means mysql was already installed on the AMI and there's no need to stop it, + # saving 40 seconds on instance startup. + time.sleep(10) #trying to stop mysql immediately after installing it fails + # We need to wait until mysql creates the root account before we kill it + # or bad things will happen + i = 0 + while self.run("echo 'quit' | mysql -u root") != 0 and i<5: + time.sleep(5) + i = i + 1 + self.run('/etc/init.d/mysql stop') + self.run("pkill -9 mysql") + + mysql_path = os.path.join(data_dir, 'mysql') + if not os.path.exists(mysql_path): + self.run('mkdir %s' % mysql_path) + fresh_install = True; + self.run('chown -R mysql:mysql %s' % mysql_path) + fp = open('/etc/mysql/conf.d/use_mnt.cnf', 'w') + fp.write('# created by pyami\n') + fp.write('# use the %s volume for data\n' % data_dir) + fp.write('[mysqld]\n') + fp.write('datadir = %s\n' % mysql_path) + fp.write('log_bin = %s\n' % os.path.join(mysql_path, 'mysql-bin.log')) + fp.close() + if fresh_install: + self.run('cp -pr /var/lib/mysql/* %s/' % mysql_path) + self.start('mysql') + else: + #get the password ubuntu expects to use: + config_parser = SafeConfigParser() + config_parser.read('/etc/mysql/debian.cnf') + password = config_parser.get('client', 'password') + # start the mysql deamon, then mysql with the required grant statement piped into it: + self.start('mysql') + time.sleep(10) #time for mysql to start + grant_command = "echo \"GRANT ALL PRIVILEGES ON *.* TO 'debian-sys-maint'@'localhost' IDENTIFIED BY '%s' WITH GRANT OPTION;\" | mysql" % password + while self.run(grant_command) != 0: + time.sleep(5) + # leave mysqld running + + def main(self): + self.install() + # change_data_dir runs 'mysql -u root' which assumes there is no mysql password, i + # and changing that is too ugly to be worth it: + #self.set_root_password() + self.change_data_dir() + diff --git a/storefront/boto/pyami/installers/ubuntu/trac.py b/storefront/boto/pyami/installers/ubuntu/trac.py new file mode 100644 index 0000000..c97ddd2 --- /dev/null +++ b/storefront/boto/pyami/installers/ubuntu/trac.py @@ -0,0 +1,137 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.installers.ubuntu.installer import Installer +import boto +import os + +class Trac(Installer): + """ + Install Trac and DAV-SVN + Sets up a Vhost pointing to [Trac]->home + Using the config parameter [Trac]->hostname + Sets up a trac environment for every directory found under [Trac]->data_dir + + [Trac] + name = My Foo Server + hostname = trac.foo.com + home = /mnt/sites/trac + data_dir = /mnt/trac + svn_dir = /mnt/subversion + server_admin = root@foo.com + sdb_auth_domain = users + # Optional + SSLCertificateFile = /mnt/ssl/foo.crt + SSLCertificateKeyFile = /mnt/ssl/foo.key + SSLCertificateChainFile = /mnt/ssl/FooCA.crt + + """ + + def install(self): + self.run('apt-get -y install trac', notify=True, exit_on_error=True) + self.run('apt-get -y install libapache2-svn', notify=True, exit_on_error=True) + self.run("a2enmod ssl") + self.run("a2enmod python") + self.run("a2enmod dav_svn") + self.run("a2enmod rewrite") + + def setup_vhost(self): + domain = boto.config.get("Trac", "hostname").strip() + if domain: + cnf = open("/etc/apache2/sites-available/%s" % domain, "w") + cnf.write("NameVirtualHost *:80\n") + if boto.config.get("Trac", "SSLCertificateFile"): + cnf.write("NameVirtualHost *:443\n\n") + cnf.write("\n") + cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) + cnf.write("\tServerName %s\n" % domain) + cnf.write("\tRewriteEngine On\n") + cnf.write("\tRewriteRule ^(.*)$ https://%s$1\n" % domain) + cnf.write("\n\n") + + cnf.write("\n") + else: + cnf.write("\n") + + cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) + cnf.write("\tServerName %s\n" % domain) + cnf.write("\tDocumentRoot %s\n" % boto.config.get("Trac", "home").strip()) + + cnf.write("\t\n" % boto.config.get("Trac", "home").strip()) + cnf.write("\t\tOptions FollowSymLinks Indexes MultiViews\n") + cnf.write("\t\tAllowOverride All\n") + cnf.write("\t\tOrder allow,deny\n") + cnf.write("\t\tallow from all\n") + cnf.write("\t\n") + + cnf.write("\t\n") + cnf.write("\t\tAuthType Basic\n") + cnf.write("\t\tAuthName \"%s\"\n" % boto.config.get("Trac", "name")) + cnf.write("\t\tRequire valid-user\n") + cnf.write("\t\tAuthBasicAuthoritative off\n") + cnf.write("\t\tAuthUserFile /dev/null\n") + cnf.write("\t\tPythonAuthenHandler marajo.web.authen_handler\n") + cnf.write("\t\tPythonOption SDBDomain %s\n" % boto.config.get("Trac", "sdb_auth_domain")) + cnf.write("\t\n") + + data_dir = boto.config.get("Trac", "data_dir") + for env in os.listdir(data_dir): + if(env[0] != "."): + cnf.write("\t\n" % env) + cnf.write("\t\tSetHandler mod_python\n") + cnf.write("\t\tPythonInterpreter main_interpreter\n") + cnf.write("\t\tPythonHandler trac.web.modpython_frontend\n") + cnf.write("\t\tPythonOption TracEnv %s/%s\n" % (data_dir, env)) + cnf.write("\t\tPythonOption TracUriRoot /trac%s\n" % env) + cnf.write("\t\n") + + svn_dir = boto.config.get("Trac", "svn_dir") + for env in os.listdir(svn_dir): + if(env[0] != "."): + cnf.write("\t\n" % env) + cnf.write("\t\tDAV svn\n") + cnf.write("\t\tSVNPath %s/%s\n" % (svn_dir, env)) + cnf.write("\t\n") + + cnf.write("\tErrorLog /var/log/apache2/error.log\n") + cnf.write("\tLogLevel warn\n") + cnf.write("\tCustomLog /var/log/apache2/access.log combined\n") + cnf.write("\tServerSignature On\n") + SSLCertificateFile = boto.config.get("Trac", "SSLCertificateFile") + if SSLCertificateFile: + cnf.write("\tSSLEngine On\n") + cnf.write("\tSSLCertificateFile %s\n" % SSLCertificateFile) + + SSLCertificateKeyFile = boto.config.get("Trac", "SSLCertificateKeyFile") + if SSLCertificateKeyFile: + cnf.write("\tSSLCertificateKeyFile %s\n" % SSLCertificateKeyFile) + + SSLCertificateChainFile = boto.config.get("Trac", "SSLCertificateChainFile") + if SSLCertificateChainFile: + cnf.write("\tSSLCertificateChainFile %s\n" % SSLCertificateChainFile) + cnf.write("\n") + cnf.close() + self.run("a2ensite %s" % domain) + self.run("/etc/init.d/apache2 force-reload") + + def main(self): + self.install() + self.setup_vhost() diff --git a/storefront/boto/pyami/launch_ami.py b/storefront/boto/pyami/launch_ami.py new file mode 100755 index 0000000..c49c2a3 --- /dev/null +++ b/storefront/boto/pyami/launch_ami.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import getopt, sys, imp, time +import boto +from boto.utils import get_instance_userdata + +usage_string = """ +SYNOPSIS + launch_ami.py -a ami_id [-b script_bucket] [-s script_name] + [-m module] [-c class_name] [-r] + [-g group] [-k key_name] [-n num_instances] + [-w] [extra_data] + Where: + ami_id - the id of the AMI you wish to launch + module - The name of the Python module containing the class you + want to run when the instance is started. If you use this + option the Python module must already be stored on the + instance in a location that is on the Python path. + script_file - The name of a local Python module that you would like + to have copied to S3 and then run on the instance + when it is started. The specified module must be + import'able (i.e. in your local Python path). It + will then be copied to the specified bucket in S3 + (see the -b option). Once the new instance(s) + start up the script will be copied from S3 and then + run locally on the instance. + class_name - The name of the class to be instantiated within the + module or script file specified. + script_bucket - the name of the bucket in which the script will be + stored + group - the name of the security group the instance will run in + key_name - the name of the keypair to use when launching the AMI + num_instances - how many instances of the AMI to launch (default 1) + input_queue_name - Name of SQS to read input messages from + output_queue_name - Name of SQS to write output messages to + extra_data - additional name-value pairs that will be passed as + userdata to the newly launched instance. These should + be of the form "name=value" + The -r option reloads the Python module to S3 without launching + another instance. This can be useful during debugging to allow + you to test a new version of your script without shutting down + your instance and starting up another one. + The -w option tells the script to run synchronously, meaning to + wait until the instance is actually up and running. It then prints + the IP address and internal and external DNS names before exiting. +""" + +def usage(): + print usage_string + sys.exit() + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'a:b:c:g:hi:k:m:n:o:rs:w', + ['ami', 'bucket', 'class', 'group', 'help', + 'inputqueue', 'keypair', 'module', + 'numinstances', 'outputqueue', + 'reload', 'script_name', 'wait']) + except: + usage() + params = {'module_name' : None, + 'script_name' : None, + 'class_name' : None, + 'script_bucket' : None, + 'group' : 'default', + 'keypair' : None, + 'ami' : None, + 'num_instances' : 1, + 'input_queue_name' : None, + 'output_queue_name' : None} + reload = None + wait = None + for o, a in opts: + if o in ('-a', '--ami'): + params['ami'] = a + if o in ('-b', '--bucket'): + params['script_bucket'] = a + if o in ('-c', '--class'): + params['class_name'] = a + if o in ('-g', '--group'): + params['group'] = a + if o in ('-h', '--help'): + usage() + if o in ('-i', '--inputqueue'): + params['input_queue_name'] = a + if o in ('-k', '--keypair'): + params['keypair'] = a + if o in ('-m', '--module'): + params['module_name'] = a + if o in ('-n', '--num_instances'): + params['num_instances'] = int(a) + if o in ('-o', '--outputqueue'): + params['output_queue_name'] = a + if o in ('-r', '--reload'): + reload = True + if o in ('-s', '--script'): + params['script_name'] = a + if o in ('-w', '--wait'): + wait = True + + # check required fields + required = ['ami'] + for pname in required: + if not params.get(pname, None): + print '%s is required' % pname + usage() + if params['script_name']: + # first copy the desired module file to S3 bucket + if reload: + print 'Reloading module %s to S3' % params['script_name'] + else: + print 'Copying module %s to S3' % params['script_name'] + l = imp.find_module(params['script_name']) + c = boto.connect_s3() + bucket = c.get_bucket(params['script_bucket']) + key = bucket.new_key(params['script_name']+'.py') + key.set_contents_from_file(l[0]) + params['script_md5'] = key.md5 + # we have everything we need, now build userdata string + l = [] + for k, v in params.items(): + if v: + l.append('%s=%s' % (k, v)) + c = boto.connect_ec2() + l.append('aws_access_key_id=%s' % c.aws_access_key_id) + l.append('aws_secret_access_key=%s' % c.aws_secret_access_key) + for kv in args: + l.append(kv) + s = '|'.join(l) + if not reload: + rs = c.get_all_images([params['ami']]) + img = rs[0] + r = img.run(user_data=s, key_name=params['keypair'], + security_groups=[params['group']], + max_count=params.get('num_instances', 1)) + print 'AMI: %s - %s (Started)' % (params['ami'], img.location) + print 'Reservation %s contains the following instances:' % r.id + for i in r.instances: + print '\t%s' % i.id + if wait: + running = False + while not running: + time.sleep(30) + [i.update() for i in r.instances] + status = [i.state for i in r.instances] + print status + if status.count('running') == len(r.instances): + running = True + for i in r.instances: + print 'Instance: %s' % i.ami_launch_index + print 'Public DNS Name: %s' % i.public_dns_name + print 'Private DNS Name: %s' % i.private_dns_name + +if __name__ == "__main__": + main() + diff --git a/storefront/boto/pyami/scriptbase.py b/storefront/boto/pyami/scriptbase.py new file mode 100644 index 0000000..6fe92aa --- /dev/null +++ b/storefront/boto/pyami/scriptbase.py @@ -0,0 +1,44 @@ +import os, sys, time, traceback +import smtplib +from boto.utils import ShellCommand, get_ts +import boto +import boto.utils + +class ScriptBase: + + def __init__(self, config_file=None): + self.instance_id = boto.config.get('Instance', 'instance-id', 'default') + self.name = self.__class__.__name__ + self.ts = get_ts() + if config_file: + boto.config.read(config_file) + + def notify(self, subject, body=''): + boto.utils.notify(subject, body) + + def mkdir(self, path): + if not os.path.isdir(path): + try: + os.mkdir(path) + except: + boto.log.error('Error creating directory: %s' % path) + + def umount(self, path): + if os.path.ismount(path): + self.run('umount %s' % path) + + def run(self, command, notify=True, exit_on_error=False): + self.last_command = ShellCommand(command) + if self.last_command.status != 0: + boto.log.error('Error running command: "%s". Output: "%s"' % (command, self.last_command.output)) + if notify: + self.notify('Error encountered', \ + 'Error running the following command:\n\t%s\n\nCommand output:\n\t%s' % \ + (command, self.last_command.output)) + if exit_on_error: + sys.exit(-1) + return self.last_command.status + + def main(self): + pass + diff --git a/storefront/boto/pyami/startup.py b/storefront/boto/pyami/startup.py new file mode 100644 index 0000000..d6f1376 --- /dev/null +++ b/storefront/boto/pyami/startup.py @@ -0,0 +1,59 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import os, sys, traceback, StringIO +import boto +from boto.utils import find_class +from boto import config +from boto.pyami.scriptbase import ScriptBase +from boto.utils import find_class + +class Startup(ScriptBase): + + def run_scripts(self): + scripts = config.get('Pyami', 'scripts') + if scripts: + for script in scripts.split(','): + script = script.strip(" ") + try: + pos = script.rfind('.') + if pos > 0: + mod_name = script[0:pos] + cls_name = script[pos+1:] + cls = find_class(mod_name, cls_name) + boto.log.info('Running Script: %s' % script) + s = cls() + s.main() + else: + boto.log.warning('Trouble parsing script: %s' % script) + except Exception, e: + boto.log.exception('Problem Running Script: %s' % script) + + def main(self): + self.run_scripts() + self.notify('Startup Completed for %s' % config.get('Instance', 'instance-id')) + +if __name__ == "__main__": + if not config.has_section('loggers'): + boto.set_file_logger('startup', '/var/log/boto.log') + sys.path.append(config.get('Pyami', 'working_dir')) + su = Startup() + su.main() diff --git a/storefront/boto/rds/__init__.py b/storefront/boto/rds/__init__.py new file mode 100644 index 0000000..92b7199 --- /dev/null +++ b/storefront/boto/rds/__init__.py @@ -0,0 +1,813 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import xml.sax +import base64 +import time +import boto +import boto.utils +import urllib +from boto.connection import AWSQueryConnection +from boto import handler +from boto.resultset import ResultSet +from boto.rds.dbinstance import DBInstance +from boto.rds.dbsecuritygroup import DBSecurityGroup +from boto.rds.parametergroup import ParameterGroup +from boto.rds.dbsnapshot import DBSnapshot +from boto.rds.event import Event + +#boto.set_stream_logger('rds') + +class RDSConnection(AWSQueryConnection): + + DefaultHost = 'rds.amazonaws.com' + APIVersion = '2009-10-16' + SignatureVersion = '2' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=DefaultHost, debug=0, + https_connection_factory=None, path='/'): + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, + proxy_pass, self.DefaultHost, debug, + https_connection_factory, path) + + # DB Instance methods + + def get_all_dbinstances(self, instance_id=None, max_records=None, + marker=None): + """ + Retrieve all the DBInstances in your account. + + :type instance_id: str + :param instance_id: DB Instance identifier. If supplied, only information + this instance will be returned. Otherwise, info + about all DB Instances will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.rds.dbinstance.DBInstance` + """ + params = {} + if instance_id: + params['DBInstanceIdentifier'] = instance_id + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBInstances', params, [('DBInstance', DBInstance)]) + + def create_dbinstance(self, id, allocated_storage, instance_class, + master_username, master_password, port=3306, + engine='MySQL5.1', db_name=None, param_group=None, + security_groups=None, availability_zone=None, + preferred_maintenance_window=None, + backup_retention_period=None, + preferred_backup_window=None): + """ + Create a new DBInstance. + + :type id: str + :param id: Unique identifier for the new instance. + Must contain 1-63 alphanumeric characters. + First character must be a letter. + May not end with a hyphen or contain two consecutive hyphens + + :type allocated_storage: int + :param allocated_storage: Initially allocated storage size, in GBs. + Valid values are [5-1024] + + :type instance_class: str + :param instance_class: The compute and memory capacity of the DBInstance. + Valid values are: + db.m1.small | db.m1.large | db.m1.xlarge | + db.m2.2xlarge | db.m2.4xlarge + + :type engine: str +. :param engine: Name of database engine. Must be MySQL5.1 for now. + + :type master_username: str + :param master_username: Name of master user for the DBInstance. + Must be 1-15 alphanumeric characters, first must be + a letter. + + :type master_password: str + :param master_password: Password of master user for the DBInstance. + Must be 4-16 alphanumeric characters. + + :type port: int + :param port: Port number on which database accepts connections. + Valid values [1115-65535]. Defaults to 3306. + + :type db_name: str + :param db_name: Name of a database to create when the DBInstance + is created. Default is to create no databases. + + :type param_group: str + :param param_group: Name of DBParameterGroup to associate with + this DBInstance. If no groups are specified + no parameter groups will be used. + + :type security_groups: list of str or list of DBSecurityGroup objects + :param security_groups: List of names of DBSecurityGroup to authorize on + this DBInstance. + + :type availability_zone: str + :param availability_zone: Name of the availability zone to place + DBInstance into. + + :type preferred_maintenance_window: str + :param preferred_maintenance_window: The weekly time range (in UTC) during + which maintenance can occur. + Default is Sun:05:00-Sun:09:00 + + :type backup_retention_period: int + :param backup_retention_period: The number of days for which automated + backups are retained. Setting this to + zero disables automated backups. + + :type preferred_backup_window: str + :param preferred_backup_window: The daily time range during which + automated backups are created (if + enabled). Must be in h24:mi-hh24:mi + format (UTC). + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The new db instance. + """ + params = {'DBInstanceIdentifier' : id, + 'AllocatedStorage' : allocated_storage, + 'DBInstanceClass' : instance_class, + 'Engine' : engine, + 'MasterUsername' : master_username, + 'MasterUserPassword' : master_password} + if port: + params['Port'] = port + if db_name: + params['DBName'] = db_name + if param_group: + params['DBParameterGroup'] = param_group + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, DBSecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, 'DBSecurityGroups.member') + if availability_zone: + params['AvailabilityZone'] = availability_zone + if preferred_maintenance_window: + params['PreferredMaintenanceWindow'] = preferred_maintenance_window + if backup_retention_period: + params['BackupRetentionPeriod'] = backup_retention_period + if preferred_backup_window: + params['PreferredBackupWindow'] = preferred_backup_window + + return self.get_object('CreateDBInstance', params, DBInstance) + + def modify_dbinstance(self, id, param_group=None, security_groups=None, + preferred_maintenance_window=None, + master_password=None, allocated_storage=None, + backup_retention_period=None, + preferred_backup_window=None, + apply_immediately=False): + """ + Modify an existing DBInstance. + + :type id: str + :param id: Unique identifier for the new instance. + + :type security_groups: list of str or list of DBSecurityGroup objects + :param security_groups: List of names of DBSecurityGroup to authorize on + this DBInstance. + + :type preferred_maintenance_window: str + :param preferred_maintenance_window: The weekly time range (in UTC) during + which maintenance can occur. + Default is Sun:05:00-Sun:09:00 + + :type master_password: str + :param master_password: Password of master user for the DBInstance. + Must be 4-15 alphanumeric characters. + + :type allocated_storage: int + :param allocated_storage: The new allocated storage size, in GBs. + Valid values are [5-1024] + + :type instance_class: str + :param instance_class: The compute and memory capacity of the DBInstance. + Changes will be applied at next maintenance + window unless apply_immediately is True. + Valid values are: + db.m1.small | db.m1.large | db.m1.xlarge | + db.m2.2xlarge | db.m2.4xlarge + + :type apply_immediately: bool + :param apply_immediately: If true, the modifications will be applied + as soon as possible rather than waiting for + the next preferred maintenance window. + + :type backup_retention_period: int + :param backup_retention_period: The number of days for which automated + backups are retained. Setting this to + zero disables automated backups. + + :type preferred_backup_window: str + :param preferred_backup_window: The daily time range during which + automated backups are created (if + enabled). Must be in h24:mi-hh24:mi + format (UTC). + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The modified db instance. + """ + params = {'DBInstanceIdentifier' : id} + if param_group: + params['DBParameterGroupName'] = param_group + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, DBSecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, 'DBSecurityGroups.member') + if preferred_maintenance_window: + params['PreferredMaintenanceWindow'] = preferred_maintenance_window + if master_password: + params['MasterUserPassword'] = master_password + if allocated_storage: + params['AllocatedStorage'] = allocated_storage + if backup_retention_period: + params['BackupRetentionPeriod'] = backup_retention_period + if preferred_backup_window: + params['PreferredBackupWindow'] = preferred_backup_window + if apply_immediately: + params['ApplyImmediately'] = 'true' + + return self.get_object('ModifyDBInstance', params, DBInstance) + + def delete_dbinstance(self, id, skip_final_snapshot=False, + final_snapshot_id=''): + """ + Delete an existing DBInstance. + + :type id: str + :param id: Unique identifier for the new instance. + + :type skip_final_snapshot: bool + :param skip_final_snapshot: This parameter determines whether a final + db snapshot is created before the instance + is deleted. If True, no snapshot is created. + If False, a snapshot is created before + deleting the instance. + + :type final_snapshot_id: str + :param final_snapshot_id: If a final snapshot is requested, this + is the identifier used for that snapshot. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The deleted db instance. + """ + params = {'DBInstanceIdentifier' : id} + if skip_final_snapshot: + params['SkipFinalSnapshot'] = 'true' + else: + params['SkipFinalSnapshot'] = 'false' + params['FinalDBSnapshotIdentifier'] = final_snapshot_id + return self.get_object('DeleteDBInstance', params, DBInstance) + + # DBParameterGroup methods + + def get_all_dbparameter_groups(self, groupname=None, max_records=None, + marker=None): + """ + Get all parameter groups associated with your account in a region. + + :type groupname: str + :param groupname: The name of the DBParameter group to retrieve. + If not provided, all DBParameter groups will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.ec2.parametergroup.ParameterGroup` + """ + params = {} + if groupname: + params['DBParameterGroupName'] = groupname + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBParameterGroups', params, + [('DBParameterGroup', ParameterGroup)]) + + def get_all_dbparameters(self, groupname, source=None, + max_records=None, marker=None): + """ + Get all parameters associated with a ParameterGroup + + :type groupname: str + :param groupname: The name of the DBParameter group to retrieve. + + :type source: str + :param source: Specifies which parameters to return. + If not specified, all parameters will be returned. + Valid values are: user|system|engine-default + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: :class:`boto.ec2.parametergroup.ParameterGroup` + :return: The ParameterGroup + """ + params = {'DBParameterGroupName' : groupname} + if source: + params['Source'] = source + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + pg = self.get_object('DescribeDBParameters', params, ParameterGroup) + pg.name = groupname + return pg + + def create_parameter_group(self, name, engine='MySQL5.1', description=''): + """ + Create a new dbparameter group for your account. + + :type name: string + :param name: The name of the new dbparameter group + + :type engine: str + :param engine: Name of database engine. Must be MySQL5.1 for now. + + :type description: string + :param description: The description of the new security group + + :rtype: :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` + :return: The newly created DBSecurityGroup + """ + params = {'DBParameterGroupName': name, + 'Engine': engine, + 'Description' : description} + return self.get_object('CreateDBParameterGroup', params, ParameterGroup) + + def modify_parameter_group(self, name, parameters=None): + """ + Modify a parameter group for your account. + + :type name: string + :param name: The name of the new parameter group + + :type parameters: list of :class:`boto.rds.parametergroup.Parameter` + :param parameters: The new parameters + + :rtype: :class:`boto.rds.parametergroup.ParameterGroup` + :return: The newly created ParameterGroup + """ + params = {'DBParameterGroupName': name} + for i in range(0, len(parameters)): + parameter = parameters[i] + parameter.merge(params, i+1) + return self.get_list('ModifyDBParameterGroup', params, ParameterGroup) + + def reset_parameter_group(self, name, reset_all_params=False, parameters=None): + """ + Resets some or all of the parameters of a ParameterGroup to the + default value + + :type key_name: string + :param key_name: The name of the ParameterGroup to reset + + :type parameters: list of :class:`boto.rds.parametergroup.Parameter` + :param parameters: The parameters to reset. If not supplied, all parameters + will be reset. + """ + params = {'DBParameterGroupName':name} + if reset_all_params: + params['ResetAllParameters'] = 'true' + else: + params['ResetAllParameters'] = 'false' + for i in range(0, len(parameters)): + parameter = parameters[i] + parameter.merge(params, i+1) + return self.get_status('ResetDBParameterGroup', params) + + def delete_parameter_group(self, name): + """ + Delete a DBSecurityGroup from your account. + + :type key_name: string + :param key_name: The name of the DBSecurityGroup to delete + """ + params = {'DBParameterGroupName':name} + return self.get_status('DeleteDBParameterGroup', params) + + # DBSecurityGroup methods + + def get_all_dbsecurity_groups(self, groupname=None, max_records=None, + marker=None): + """ + Get all security groups associated with your account in a region. + + :type groupnames: list + :param groupnames: A list of the names of security groups to retrieve. + If not provided, all security groups will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` + """ + params = {} + if groupname: + params['DBSecurityGroupName'] = groupname + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBSecurityGroups', params, + [('DBSecurityGroup', DBSecurityGroup)]) + + def create_dbsecurity_group(self, name, description=None): + """ + Create a new security group for your account. + This will create the security group within the region you + are currently connected to. + + :type name: string + :param name: The name of the new security group + + :type description: string + :param description: The description of the new security group + + :rtype: :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` + :return: The newly created DBSecurityGroup + """ + params = {'DBSecurityGroupName':name} + if description: + params['DBSecurityGroupDescription'] = description + group = self.get_object('CreateDBSecurityGroup', params, DBSecurityGroup) + group.name = name + group.description = description + return group + + def delete_dbsecurity_group(self, name): + """ + Delete a DBSecurityGroup from your account. + + :type key_name: string + :param key_name: The name of the DBSecurityGroup to delete + """ + params = {'DBSecurityGroupName':name} + return self.get_status('DeleteDBSecurityGroup', params) + + def authorize_dbsecurity_group(self, group_name, cidr_ip=None, + ec2_security_group_name=None, + ec2_security_group_owner_id=None): + """ + Add a new rule to an existing security group. + You need to pass in either src_security_group_name and + src_security_group_owner_id OR a CIDR block but not both. + + :type group_name: string + :param group_name: The name of the security group you are adding + the rule to. + + :type ec2_security_group_name: string + :param ec2_security_group_name: The name of the EC2 security group you are + granting access to. + + :type ec2_security_group_owner_id: string + :param ec2_security_group_owner_id: The ID of the owner of the EC2 security + group you are granting access to. + + :type cidr_ip: string + :param cidr_ip: The CIDR block you are providing access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + params = {'DBSecurityGroupName':group_name} + if ec2_security_group_name: + params['EC2SecurityGroupName'] = ec2_security_group_name + if ec2_security_group_owner_id: + params['EC2SecurityGroupOwnerId'] = ec2_security_group_owner_id + if cidr_ip: + params['CIDRIP'] = urllib.quote(cidr_ip) + return self.get_object('AuthorizeDBSecurityGroupIngress', params, DBSecurityGroup) + + def revoke_security_group(self, group_name, ec2_security_group_name=None, + ec2_security_group_owner_id=None, cidr_ip=None): + """ + Remove an existing rule from an existing security group. + You need to pass in either ec2_security_group_name and + ec2_security_group_owner_id OR a CIDR block. + + :type group_name: string + :param group_name: The name of the security group you are removing + the rule from. + + :type ec2_security_group_name: string + :param ec2_security_group_name: The name of the EC2 security group you are + granting access to. + + :type ec2_security_group_owner_id: string + :param ec2_security_group_owner_id: The ID of the owner of the EC2 security + group you are granting access to. + + :type cidr_ip: string + :param cidr_ip: The CIDR block you are providing access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + params = {'DBSecurityGroupName':group_name} + if ec2_security_group_name: + params['EC2SecurityGroupName'] = ec2_security_group_name + if ec2_security_group_owner_id: + params['EC2SecurityGroupOwnerId'] = ec2_security_group_owner_id + if cidr_ip: + params['CIDRIP'] = cidr_ip + return self.get_object('RevokeDBSecurityGroupIngress', params, DBSecurityGroup) + + # DBSnapshot methods + + def get_all_dbsnapshots(self, snapshot_id=None, instance_id=None, + max_records=None, marker=None): + """ + Get information about DB Snapshots. + + :type snapshot_id: str + :param snapshot_id: The unique identifier of an RDS snapshot. + If not provided, all RDS snapshots will be returned. + + :type instance_id: str + :param instance_id: The identifier of a DBInstance. If provided, + only the DBSnapshots related to that instance will + be returned. + If not provided, all RDS snapshots will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.rds.dbsnapshot.DBSnapshot` + """ + params = {} + if snapshot_id: + params['DBSnapshotIdentifier'] = snapshot_id + if instance_id: + params['DBInstanceIdentifier'] = instance_id + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBSnapshots', params, + [('DBSnapshots', DBSnapshot)]) + + def create_dbsnapshot(self, snapshot_id, dbinstance_id): + """ + Create a new DB snapshot. + + :type snapshot_id: string + :param snapshot_id: The identifier for the DBSnapshot + + :type dbinstance_id: string + :param dbinstance_id: The source identifier for the RDS instance from + which the snapshot is created. + + :rtype: :class:`boto.rds.dbsnapshot.DBSnapshot` + :return: The newly created DBSnapshot + """ + params = {'DBSnapshotIdentifier' : snapshot_id, + 'DBInstanceIdentifier' : dbinstance_id} + return self.get_object('CreateDBSnapshot', params, DBSnapshot) + + def delete_dbsnapshot(self, identifier): + """ + Delete a DBSnapshot + + :type identifier: string + :param identifier: The identifier of the DBSnapshot to delete + """ + params = {'DBSnapshotIdentifier' : identifier} + return self.get_object('DeleteDBSnapshot', params, DBSnapshot) + + def restore_dbinstance_from_dbsnapshot(self, identifier, instance_id, + instance_class, port=None, + availability_zone=None): + + """ + Create a new DBInstance from a DB snapshot. + + :type identifier: string + :param identifier: The identifier for the DBSnapshot + + :type instance_id: string + :param instance_id: The source identifier for the RDS instance from + which the snapshot is created. + + :type instance_class: str + :param instance_class: The compute and memory capacity of the DBInstance. + Valid values are: + db.m1.small | db.m1.large | db.m1.xlarge | + db.m2.2xlarge | db.m2.4xlarge + + :type port: int + :param port: Port number on which database accepts connections. + Valid values [1115-65535]. Defaults to 3306. + + :type availability_zone: str + :param availability_zone: Name of the availability zone to place + DBInstance into. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The newly created DBInstance + """ + params = {'DBSnapshotIdentifier' : identifier, + 'DBInstanceIdentifier' : instance_id, + 'DBInstanceClass' : instance_class} + if port: + params['Port'] = port + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('RestoreDBInstanceFromDBSnapshot', + params, DBInstance) + + def restore_dbinstance_from_point_in_time(self, source_instance_id, + target_instance_id, + use_latest=False, + restore_time=None, + dbinstance_class=None, + port=None, + availability_zone=None): + + """ + Create a new DBInstance from a point in time. + + :type source_instance_id: string + :param source_instance_id: The identifier for the source DBInstance. + + :type target_instance_id: string + :param target_instance_id: The identifier of the new DBInstance. + + :type use_latest: bool + :param use_latest: If True, the latest snapshot availabile will + be used. + + :type restore_time: datetime + :param restore_time: The date and time to restore from. Only + used if use_latest is False. + + :type instance_class: str + :param instance_class: The compute and memory capacity of the DBInstance. + Valid values are: + db.m1.small | db.m1.large | db.m1.xlarge | + db.m2.2xlarge | db.m2.4xlarge + + :type port: int + :param port: Port number on which database accepts connections. + Valid values [1115-65535]. Defaults to 3306. + + :type availability_zone: str + :param availability_zone: Name of the availability zone to place + DBInstance into. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The newly created DBInstance + """ + params = {'SourceDBInstanceIdentifier' : source_instance_id, + 'TargetDBInstanceIdentifier' : target_instance_id} + if use_latest: + params['UseLatestRestorableTime'] = 'true' + elif restore_time: + params['RestoreTime'] = restore_time.isoformat() + if instance_class: + params['DBInstanceClass'] = instance_class + if port: + params['Port'] = port + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('RestoreDBInstanceToPointInTime', + params, DBInstance) + + # Events + + def get_all_events(self, source_identifier=None, source_type=None, + start_time=None, end_time=None, + max_records=None, marker=None): + """ + Get information about events related to your DBInstances, + DBSecurityGroups and DBParameterGroups. + + :type source_identifier: str + :param source_identifier: If supplied, the events returned will be + limited to those that apply to the identified + source. The value of this parameter depends + on the value of source_type. If neither + parameter is specified, all events in the time + span will be returned. + + :type source_type: str + :param source_type: Specifies how the source_identifier should + be interpreted. Valid values are: + b-instance | db-security-group | + db-parameter-group | db-snapshot + + :type start_time: datetime + :param start_time: The beginning of the time interval for events. + If not supplied, all available events will + be returned. + + :type end_time: datetime + :param end_time: The ending of the time interval for events. + If not supplied, all available events will + be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of class:`boto.rds.event.Event` + """ + params = {} + if source_identifier and source_type: + params['SourceIdentifier'] = source_identifier + params['SourceType'] = source_type + if start_time: + params['StartTime'] = start_time.isoformat() + if end_time: + params['EndTime'] = end_time.isoformat() + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeEvents', params, [('Event', Event)]) + + diff --git a/storefront/boto/rds/dbinstance.py b/storefront/boto/rds/dbinstance.py new file mode 100644 index 0000000..23e1c98 --- /dev/null +++ b/storefront/boto/rds/dbinstance.py @@ -0,0 +1,136 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.rds.dbsecuritygroup import DBSecurityGroup +from boto.rds.parametergroup import ParameterGroup + +class DBInstance(object): + """ + Represents a RDS DBInstance + """ + + def __init__(self, connection=None, id=None): + self.connection = connection + self.id = id + self.create_time = None + self.engine = None + self.status = None + self.allocated_storage = None + self.endpoint = None + self.instance_class = None + self.master_username = None + self.parameter_group = None + self.security_group = None + self.availability_zone = None + self.backup_retention_period = None + self.preferred_backup_window = None + self.preferred_maintenance_window = None + self.latest_restorable_time = None + self._in_endpoint = False + self._port = None + self._address = None + + def __repr__(self): + return 'DBInstance:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'Endpoint': + self._in_endpoint = True + elif name == 'DBParameterGroup': + self.parameter_group = ParameterGroup(self.connection) + return self.parameter_group + elif name == 'DBSecurityGroup': + self.security_group = DBSecurityGroup(self.connection) + return self.security_group + return None + + def endElement(self, name, value, connection): + if name == 'DBInstanceIdentifier': + self.id = value + elif name == 'DBInstanceStatus': + self.status = value + elif name == 'InstanceCreateTime': + self.create_time = value + elif name == 'Engine': + self.engine = value + elif name == 'DBInstanceStatus': + self.status = value + elif name == 'AllocatedStorage': + self.allocated_storage = int(value) + elif name == 'DBInstanceClass': + self.instance_class = value + elif name == 'MasterUsername': + self.master_username = value + elif name == 'Port': + if self._in_endpoint: + self._port = int(value) + elif name == 'Address': + if self._in_endpoint: + self._address = value + elif name == 'Endpoint': + self.endpoint = (self._address, self._port) + self._in_endpoint = False + elif name == 'AvailabilityZone': + self.availability_zone = value + elif name == 'BackupRetentionPeriod': + self.backup_retention_period = value + elif name == 'LatestRestorableTime': + self.latest_restorable_time = value + elif name == 'PreferredMaintenanceWindow': + self.preferred_maintenance_window = value + elif name == 'PreferredBackupWindow': + self.preferred_backup_window = value + else: + setattr(self, name, value) + + def snapshot(self, snapshot_id): + """ + Create a new DB snapshot of this DBInstance. + + :type identifier: string + :param identifier: The identifier for the DBSnapshot + + :rtype: :class:`boto.rds.dbsnapshot.DBSnapshot` + :return: The newly created DBSnapshot + """ + return self.connection.create_dbsnapshot(snapshot_id, self.id) + + def stop(self, skip_final_snapshot, final_snapshot_id): + """ + Delete this DBInstance. + + :type skip_final_snapshot: bool + :param skip_final_snapshot: This parameter determines whether a final + db snapshot is created before the instance + is deleted. If True, no snapshot is created. + If False, a snapshot is created before + deleting the instance. + + :type final_snapshot_id: str + :param final_snapshot_id: If a final snapshot is requested, this + is the identifier used for that snapshot. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The deleted db instance. + """ + return self.connection.delete_dbinstance(self.id, + skip_final_snapshot, + final_snapshot_id) diff --git a/storefront/boto/rds/dbsecuritygroup.py b/storefront/boto/rds/dbsecuritygroup.py new file mode 100644 index 0000000..9ec6cc0 --- /dev/null +++ b/storefront/boto/rds/dbsecuritygroup.py @@ -0,0 +1,159 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an DBSecurityGroup +""" +from boto.ec2.securitygroup import SecurityGroup + +class DBSecurityGroup(object): + + def __init__(self, connection=None, owner_id=None, + name=None, description=None): + self.connection = connection + self.owner_id = owner_id + self.name = name + self.description = description + self.ec2_groups = [] + self.ip_ranges = [] + + def __repr__(self): + return 'DBSecurityGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'IPRange': + cidr = IPRange(self) + self.ip_ranges.append(cidr) + return cidr + elif name == 'EC2SecurityGroup': + ec2_grp = EC2SecurityGroup(self) + self.ec2_groups.append(ec2_grp) + return ec2_grp + else: + return None + + def endElement(self, name, value, connection): + if name == 'OwnerId': + self.owner_id = value + elif name == 'DBSecurityGroupName': + self.name = value + elif name == 'DBSecurityGroupDescription': + self.description = value + elif name == 'IPRanges': + pass + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_dbsecurity_group(self.name) + + def authorize(self, cidr_ip=None, ec2_group=None): + """ + Add a new rule to this DBSecurity group. + You need to pass in either a CIDR block to authorize or + and EC2 SecurityGroup. + + @type cidr_ip: string + @param cidr_ip: A valid CIDR IP range to authorize + + @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>`b + + @rtype: bool + @return: True if successful. + """ + if isinstance(ec2_group, SecurityGroup): + group_name = ec2_group.name + group_owner_id = ec2_group.owner_id + else: + group_name = None + group_owner_id = None + return self.connection.authorize_dbsecurity_group(self.name, + cidr_ip, + group_name, + group_owner_id) + + def revoke(self, cidr_ip=None, ec2_group=None): + """ + Revoke access to a CIDR range or EC2 SecurityGroup + You need to pass in either a CIDR block to authorize or + and EC2 SecurityGroup. + + @type cidr_ip: string + @param cidr_ip: A valid CIDR IP range to authorize + + @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>`b + + @rtype: bool + @return: True if successful. + """ + if isinstance(ec2_group, SecurityGroup): + group_name = ec2_group.name + group_owner_id = ec2_group.owner_id + else: + group_name = None + group_owner_id = None + return self.connection.revoke_dbsecurity_group(self.name, + cidr_ip, + group_name, + group_owner_id) + +class IPRange(object): + + def __init__(self, parent=None): + self.parent = parent + self.cidr_ip = None + self.status = None + + def __repr__(self): + return 'IPRange:%s' % self.cidr_ip + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'CIDRIP': + self.cidr_ip = value + elif name == 'Status': + self.status = value + else: + setattr(self, name, value) + +class EC2SecurityGroup(object): + + def __init__(self, parent=None): + self.parent = parent + self.name = None + self.owner_id = None + + def __repr__(self): + return 'EC2SecurityGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'EC2SecurityGroupName': + self.name = value + elif name == 'EC2SecurityGroupOwnerId': + self.owner_id = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/rds/dbsnapshot.py b/storefront/boto/rds/dbsnapshot.py new file mode 100644 index 0000000..78d0230 --- /dev/null +++ b/storefront/boto/rds/dbsnapshot.py @@ -0,0 +1,74 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class DBSnapshot(object): + """ + Represents a RDS DB Snapshot + """ + + def __init__(self, connection=None, id=None): + self.connection = connection + self.id = id + self.engine = None + self.snapshot_create_time = None + self.instance_create_time = None + self.port = None + self.status = None + self.availability_zone = None + self.master_username = None + self.allocated_storage = None + self.instance_id = None + self.availability_zone = None + + def __repr__(self): + return 'DBSnapshot:%s' % self.id + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'Engine': + self.engine = value + elif name == 'InstanceCreateTime': + self.instance_create_time = value + elif name == 'SnapshotCreateTime': + self.snapshot_create_time = value + elif name == 'DBInstanceIdentifier': + self.instance_id = value + elif name == 'DBSnapshotIdentifier': + self.id = value + elif name == 'Port': + self.port = int(value) + elif name == 'Status': + self.status = value + elif name == 'AvailabilityZone': + self.availability_zone = value + elif name == 'MasterUsername': + self.master_username = value + elif name == 'AllocatedStorage': + self.allocated_storage = int(value) + elif name == 'SnapshotTime': + self.time = value + else: + setattr(self, name, value) + + + diff --git a/storefront/boto/rds/event.py b/storefront/boto/rds/event.py new file mode 100644 index 0000000..a91f8f0 --- /dev/null +++ b/storefront/boto/rds/event.py @@ -0,0 +1,49 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Event(object): + + def __init__(self, connection=None): + self.connection = connection + self.message = None + self.source_identifier = None + self.source_type = None + self.engine = None + self.date = None + + def __repr__(self): + return '"%s"' % self.message + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'SourceIdentifier': + self.source_identifier = value + elif name == 'SourceType': + self.source_type = value + elif name == 'Message': + self.message = value + elif name == 'Date': + self.date = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/rds/parametergroup.py b/storefront/boto/rds/parametergroup.py new file mode 100644 index 0000000..081e263 --- /dev/null +++ b/storefront/boto/rds/parametergroup.py @@ -0,0 +1,201 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class ParameterGroup(dict): + + def __init__(self, connection=None): + dict.__init__(self) + self.connection = connection + self.name = None + self.description = None + self.engine = None + self._current_param = None + + def __repr__(self): + return 'ParameterGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'Parameter': + if self._current_param: + self[self._current_param.name] = self._current_param + self._current_param = Parameter(self) + return self._current_param + + def endElement(self, name, value, connection): + if name == 'DBParameterGroupName': + self.name = value + elif name == 'Description': + self.description = value + elif name == 'Engine': + self.engine = value + else: + setattr(self, name, value) + + def modifiable(self): + mod = [] + for key in self: + p = self[key] + if p.is_modifiable: + mod.append(p) + return mod + + def get_params(self): + pg = self.connection.get_all_dbparameters(self.name) + self.update(pg) + + def add_param(self, name, value, apply_method): + param = Parameter() + param.name = name + param.value = value + param.apply_method = apply_method + self.params.append(param) + +class Parameter(object): + """ + Represents a RDS Parameter + """ + + ValidTypes = {'integer' : int, + 'string' : str, + 'boolean' : bool} + ValidSources = ['user', 'system', 'engine-default'] + ValidApplyTypes = ['static', 'dynamic'] + ValidApplyMethods = ['immediate', 'pending-reboot'] + + def __init__(self, group=None, name=None): + self.group = group + self.name = name + self._value = None + self.type = str + self.source = None + self.is_modifiable = True + self.description = None + self.apply_method = None + self.allowed_values = None + + def __repr__(self): + return 'Parameter:%s' % self.name + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'ParameterName': + self.name = value + elif name == 'ParameterValue': + self._value = value + elif name == 'DataType': + if value in self.ValidTypes: + self.type = value + elif name == 'Source': + if value in self.ValidSources: + self.source = value + elif name == 'IsModifiable': + if value.lower() == 'true': + self.is_modifiable = True + else: + self.is_modifiable = False + elif name == 'Description': + self.description = value + elif name == 'ApplyType': + if value in self.ValidApplyTypes: + self.apply_type = value + elif name == 'AllowedValues': + self.allowed_values = value + else: + setattr(self, name, value) + + def merge(self, d, i): + prefix = 'Parameters.member.%d.' % i + if self.name: + d[prefix+'ParameterName'] = self.name + if self._value: + d[prefix+'ParameterValue'] = self._value + if self.apply_type: + d[prefix+'ApplyMethod'] = self.apply_method + + def _set_string_value(self, value): + if not isinstance(value, str) or isinstance(value, unicode): + raise ValueError, 'value must be of type str' + if self.allowed_values: + choices = self.allowed_values.split(',') + if value not in choices: + raise ValueError, 'value must be in %s' % self.allowed_values + set._value = value + + def _set_integer_value(self, value): + if isinstance(value, str) or isinstance(value, unicode): + value = int(value) + if isinstance(value, int) or isinstance(value, long): + if self.allowed_values: + min, max = self.allowed_values.split('-') + if value < int(min) or value > int(max): + raise ValueError, 'range is %s' % self.allowed_values + self._value = value + else: + raise ValueError, 'value must be integer' + + def _set_boolean_value(self, value): + if isinstance(value, bool): + self._value = value + elif isinstance(value, str) or isinstance(value, unicode): + if value.lower() == 'true': + self._value = True + else: + self._value = False + else: + raise ValueError, 'value must be boolean' + + def set_value(self, value): + if self.type == 'string': + self._set_string_value(value) + elif self.type == 'integer': + self._set_integer_value(value) + elif self.type == 'boolean': + self._set_boolean_value(value) + else: + raise TypeError, 'unknown type (%s)' % self.type + + def get_value(self): + if self._value == None: + return self._value + if self.type == 'string': + return self._value + elif self.type == 'integer': + if not isinstance(self._value, int) and not isinstance(self._value, long): + self._set_integer_value(self._value) + return self._value + elif self.type == 'boolean': + if not isinstance(self._value, bool): + self._set_boolean_value(self._value) + return self._value + else: + raise TypeError, 'unknown type (%s)' % self.type + + value = property(get_value, set_value, 'The value of the parameter') + + def apply(self, immediate=False): + if immediate: + self.apply_method = 'immediate' + else: + self.apply_method = 'pending-reboot' + self.group.connection.modify_parameter_group(self.group.name, [self]) + diff --git a/storefront/boto/resultset.py b/storefront/boto/resultset.py new file mode 100644 index 0000000..aab1b68 --- /dev/null +++ b/storefront/boto/resultset.py @@ -0,0 +1,130 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class ResultSet(list): + """ + The ResultSet is used to pass results back from the Amazon services + to the client. It has an ugly but workable mechanism for parsing + the XML results from AWS. Because I don't really want any dependencies + on external libraries, I'm using the standard SAX parser that comes + with Python. The good news is that it's quite fast and efficient but + it makes some things rather difficult. + + You can pass in, as the marker_elem parameter, a list of tuples. + Each tuple contains a string as the first element which represents + the XML element that the resultset needs to be on the lookout for + and a Python class as the second element of the tuple. Each time the + specified element is found in the XML, a new instance of the class + will be created and popped onto the stack. + + """ + + def __init__(self, marker_elem=None): + list.__init__(self) + if isinstance(marker_elem, list): + self.markers = marker_elem + else: + self.markers = [] + self.marker = None + self.is_truncated = False + self.next_token = None + self.status = True + + def startElement(self, name, attrs, connection): + for t in self.markers: + if name == t[0]: + obj = t[1](connection) + self.append(obj) + return obj + return None + + def to_boolean(self, value, true_value='true'): + if value == true_value: + return True + else: + return False + + def endElement(self, name, value, connection): + if name == 'IsTruncated': + self.is_truncated = self.to_boolean(value) + elif name == 'Marker': + self.marker = value + elif name == 'Prefix': + self.prefix = value + elif name == 'return': + self.status = self.to_boolean(value) + elif name == 'StatusCode': + self.status = self.to_boolean(value, 'Success') + elif name == 'ItemName': + self.append(value) + elif name == 'NextToken': + self.next_token = value + elif name == 'BoxUsage': + try: + connection.box_usage += float(value) + except: + pass + elif name == 'IsValid': + self.status = self.to_boolean(value, 'True') + else: + setattr(self, name, value) + +class BooleanResult(object): + + def __init__(self, marker_elem=None): + self.status = True + self.request_id = None + self.box_usage = None + + def __repr__(self): + if self.status: + return 'True' + else: + return 'False' + + def __nonzero__(self): + return self.status + + def startElement(self, name, attrs, connection): + return None + + def to_boolean(self, value, true_value='true'): + if value == true_value: + return True + else: + return False + + def endElement(self, name, value, connection): + if name == 'return': + self.status = self.to_boolean(value) + elif name == 'StatusCode': + self.status = self.to_boolean(value, 'Success') + elif name == 'IsValid': + self.status = self.to_boolean(value, 'True') + elif name == 'RequestId': + self.request_id = value + elif name == 'requestId': + self.request_id = value + elif name == 'BoxUsage': + self.request_id = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/s3/__init__.py b/storefront/boto/s3/__init__.py new file mode 100644 index 0000000..be2de1d --- /dev/null +++ b/storefront/boto/s3/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto + +boto.check_extensions(__name__, __path__) + +from connection import S3Connection as Connection +from key import Key +from bucket import Bucket + +__all__ = ['Connection', 'Key', 'Bucket'] diff --git a/storefront/boto/s3/acl.py b/storefront/boto/s3/acl.py new file mode 100644 index 0000000..702551e --- /dev/null +++ b/storefront/boto/s3/acl.py @@ -0,0 +1,161 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.s3.user import User +import StringIO + +CannedACLStrings = ['private', 'public-read', + 'public-read-write', 'authenticated-read'] + +class Policy: + + def __init__(self, parent=None): + self.parent = parent + self.acl = None + + def __repr__(self): + grants = [] + for g in self.acl.grants: + if g.id == self.owner.id: + grants.append("%s (owner) = %s" % (g.display_name, g.permission)) + else: + if g.type == 'CanonicalUser': + u = g.display_name + elif g.type == 'Group': + u = g.uri + else: + u = g.email + grants.append("%s = %s" % (u, g.permission)) + return "" % ", ".join(grants) + + def startElement(self, name, attrs, connection): + if name == 'Owner': + self.owner = User(self) + return self.owner + elif name == 'AccessControlList': + self.acl = ACL(self) + return self.acl + else: + return None + + def endElement(self, name, value, connection): + if name == 'Owner': + pass + elif name == 'AccessControlList': + pass + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + s += self.owner.to_xml() + s += self.acl.to_xml() + s += '' + return s + +class ACL: + + def __init__(self, policy=None): + self.policy = policy + self.grants = [] + + def add_grant(self, grant): + self.grants.append(grant) + + def add_email_grant(self, permission, email_address): + grant = Grant(permission=permission, type='AmazonCustomerByEmail', + email_address=email_address) + self.grants.append(grant) + + def add_user_grant(self, permission, user_id): + grant = Grant(permission=permission, type='CanonicalUser', id=user_id) + self.grants.append(grant) + + def startElement(self, name, attrs, connection): + if name == 'Grant': + self.grants.append(Grant(self)) + return self.grants[-1] + else: + return None + + def endElement(self, name, value, connection): + if name == 'Grant': + pass + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + for grant in self.grants: + s += grant.to_xml() + s += '' + return s + +class Grant: + + NameSpace = 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' + + def __init__(self, permission=None, type=None, id=None, + display_name=None, uri=None, email_address=None): + self.permission = permission + self.id = id + self.display_name = display_name + self.uri = uri + self.email_address = email_address + self.type = type + + def startElement(self, name, attrs, connection): + if name == 'Grantee': + self.type = attrs['xsi:type'] + return None + + def endElement(self, name, value, connection): + if name == 'ID': + self.id = value + elif name == 'DisplayName': + self.display_name = value + elif name == 'URI': + self.uri = value + elif name == 'EmailAddress': + self.email_address = value + elif name == 'Grantee': + pass + elif name == 'Permission': + self.permission = value + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + s += '' % (self.NameSpace, self.type) + if self.type == 'CanonicalUser': + s += '%s' % self.id + s += '%s' % self.display_name + elif self.type == 'Group': + s += '%s' % self.uri + else: + s += '%s' % self.email_address + s += '' + s += '%s' % self.permission + s += '' + return s + + diff --git a/storefront/boto/s3/bucket.py b/storefront/boto/s3/bucket.py new file mode 100644 index 0000000..297f0a2 --- /dev/null +++ b/storefront/boto/s3/bucket.py @@ -0,0 +1,495 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto import handler +from boto.resultset import ResultSet +from boto.s3.acl import Policy, CannedACLStrings, ACL, Grant +from boto.s3.user import User +from boto.s3.key import Key +from boto.s3.prefix import Prefix +from boto.exception import S3ResponseError, S3PermissionsError, S3CopyError +from boto.s3.bucketlistresultset import BucketListResultSet +import boto.utils +import xml.sax +import urllib + +S3Permissions = ['READ', 'WRITE', 'READ_ACP', 'WRITE_ACP', 'FULL_CONTROL'] + +class Bucket: + + BucketLoggingBody = """ + + + %s + %s + + """ + + EmptyBucketLoggingBody = """ + + """ + + LoggingGroup = 'http://acs.amazonaws.com/groups/s3/LogDelivery' + + BucketPaymentBody = """ + + %s + """ + + def __init__(self, connection=None, name=None, key_class=Key): + self.name = name + self.connection = connection + self.key_class = key_class + + def __repr__(self): + return '' % self.name + + def __iter__(self): + return iter(BucketListResultSet(self)) + + def __contains__(self, key_name): + return not (self.get_key(key_name) is None) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Name': + self.name = value + elif name == 'CreationDate': + self.creation_date = value + else: + setattr(self, name, value) + + def set_key_class(self, key_class): + """ + Set the Key class associated with this bucket. By default, this + would be the boto.s3.key.Key class but if you want to subclass that + for some reason this allows you to associate your new class with a + bucket so that when you call bucket.new_key() or when you get a listing + of keys in the bucket you will get an instances of your key class + rather than the default. + + :type key_class: class + :param key_class: A subclass of Key that can be more specific + """ + self.key_class = key_class + + def lookup(self, key_name, headers=None): + """ + Deprecated: Please use get_key method. + + :type key_name: string + :param key_name: The name of the key to retrieve + + :rtype: :class:`boto.s3.key.Key` + :returns: A Key object from this bucket. + """ + return self.get_key(key_name, headers=headers) + + def get_key(self, key_name, headers=None): + """ + Check to see if a particular key exists within the bucket. This + method uses a HEAD request to check for the existance of the key. + Returns: An instance of a Key object or None + + :type key_name: string + :param key_name: The name of the key to retrieve + + :rtype: :class:`boto.s3.key.Key` + :returns: A Key object from this bucket. + """ + response = self.connection.make_request('HEAD', self.name, key_name, headers=headers) + if response.status == 200: + body = response.read() + k = self.key_class(self) + k.metadata = boto.utils.get_aws_metadata(response.msg) + k.etag = response.getheader('etag') + k.content_type = response.getheader('content-type') + k.content_encoding = response.getheader('content-encoding') + k.last_modified = response.getheader('last-modified') + k.size = int(response.getheader('content-length')) + k.name = key_name + return k + else: + if response.status == 404: + body = response.read() + return None + else: + raise S3ResponseError(response.status, response.reason, '') + + def list(self, prefix='', delimiter='', marker='', headers=None): + """ + List key objects within a bucket. This returns an instance of an + BucketListResultSet that automatically handles all of the result + paging, etc. from S3. You just need to keep iterating until + there are no more results. + Called with no arguments, this will return an iterator object across + all keys within the bucket. + + :type prefix: string + :param prefix: allows you to limit the listing to a particular + prefix. For example, if you call the method with prefix='/foo/' + then the iterator will only cycle through the keys that begin with + the string '/foo/'. + + :type delimiter: string + :param delimiter: can be used in conjunction with the prefix + to allow you to organize and browse your keys hierarchically. See: + http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ + for more details. + + :type marker: string + :param marker: The "marker" of where you are in the result set + + :rtype: :class:`boto.s3.bucketlistresultset.BucketListResultSet` + :return: an instance of a BucketListResultSet that handles paging, etc + """ + return BucketListResultSet(self, prefix, delimiter, marker, headers) + + def get_all_keys(self, headers=None, **params): + """ + A lower-level method for listing contents of a bucket. This closely models the actual S3 + API and requires you to manually handle the paging of results. For a higher-level method + that handles the details of paging for you, you can use the list method. + + :type maxkeys: int + :param maxkeys: The maximum number of keys to retrieve + + :type prefix: string + :param prefix: The prefix of the keys you want to retrieve + + :type marker: string + :param marker: The "marker" of where you are in the result set + + :type delimiter: string + :param delimiter: "If this optional, Unicode string parameter is included with your request, then keys that contain the same string between the prefix and the first occurrence of the delimiter will be rolled up into a single result element in the CommonPrefixes collection. These rolled-up keys are not returned elsewhere in the response." + + :rtype: ResultSet + :return: The result from S3 listing the keys requested + + """ + l = [] + for k,v in params.items(): + if k == 'maxkeys': + k = 'max-keys' + if isinstance(v, unicode): + v = v.encode('utf-8') + if v is not None: + l.append('%s=%s' % (urllib.quote(k), urllib.quote(str(v)))) + if len(l): + s = '&'.join(l) + else: + s = None + response = self.connection.make_request('GET', self.name, + headers=headers, query_args=s) + body = response.read() + boto.log.debug(body) + if response.status == 200: + rs = ResultSet([('Contents', self.key_class), + ('CommonPrefixes', Prefix)]) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise S3ResponseError(response.status, response.reason, body) + + def new_key(self, key_name=None): + """ + Creates a new key + + :type key_name: string + :param key_name: The name of the key to create + + :rtype: :class:`boto.s3.key.Key` or subclass + :returns: An instance of the newly created key object + """ + return self.key_class(self, key_name) + + def generate_url(self, expires_in, method='GET', headers=None, force_http=False): + return self.connection.generate_url(expires_in, method, self.name, headers=headers, + force_http=force_http) + + def delete_key(self, key_name, headers=None): + """ + Deletes a key from the bucket. + + :type key_name: string + :param key_name: The key name to delete + """ + response = self.connection.make_request('DELETE', self.name, key_name, headers=headers) + body = response.read() + if response.status != 204: + raise S3ResponseError(response.status, response.reason, body) + + def copy_key(self, new_key_name, src_bucket_name, src_key_name, metadata=None): + """ + Create a new key in the bucket by copying another existing key. + + :type new_key_name: string + :param new_key_name: The name of the new key + + :type src_bucket_name: string + :param src_bucket_name: The name of the source bucket + + :type src_key_name: string + :param src_key_name: The name of the source key + + :type metadata: dict + :param metadata: Metadata to be associated with new key. + If metadata is supplied, it will replace the + metadata of the source key being copied. + If no metadata is supplied, the source key's + metadata will be copied to the new key. + + :rtype: :class:`boto.s3.key.Key` or subclass + :returns: An instance of the newly created key object + """ + src = '%s/%s' % (src_bucket_name, urllib.quote(src_key_name)) + if metadata: + headers = {'x-amz-copy-source' : src, + 'x-amz-metadata-directive' : 'REPLACE'} + headers = boto.utils.merge_meta(headers, metadata) + else: + headers = {'x-amz-copy-source' : src, + 'x-amz-metadata-directive' : 'COPY'} + response = self.connection.make_request('PUT', self.name, new_key_name, + headers=headers) + body = response.read() + if response.status == 200: + key = self.new_key(new_key_name) + h = handler.XmlHandler(key, self) + xml.sax.parseString(body, h) + if hasattr(key, 'Error'): + raise S3CopyError(key.Code, key.Message, body) + return key + else: + raise S3ResponseError(response.status, response.reason, body) + + def set_canned_acl(self, acl_str, key_name='', headers=None): + assert acl_str in CannedACLStrings + + if headers: + headers['x-amz-acl'] = acl_str + else: + headers={'x-amz-acl': acl_str} + + response = self.connection.make_request('PUT', self.name, key_name, + headers=headers, query_args='acl') + body = response.read() + if response.status != 200: + raise S3ResponseError(response.status, response.reason, body) + + def get_xml_acl(self, key_name='', headers=None): + response = self.connection.make_request('GET', self.name, key_name, + query_args='acl', headers=headers) + body = response.read() + if response.status != 200: + raise S3ResponseError(response.status, response.reason, body) + return body + + def set_xml_acl(self, acl_str, key_name='', headers=None): + response = self.connection.make_request('PUT', self.name, key_name, + data=acl_str, query_args='acl', headers=headers) + body = response.read() + if response.status != 200: + raise S3ResponseError(response.status, response.reason, body) + + def set_acl(self, acl_or_str, key_name='', headers=None): + if isinstance(acl_or_str, Policy): + self.set_xml_acl(acl_or_str.to_xml(), key_name, headers=headers) + else: + self.set_canned_acl(acl_or_str, key_name, headers=headers) + + def get_acl(self, key_name='', headers=None): + response = self.connection.make_request('GET', self.name, key_name, + query_args='acl', headers=headers) + body = response.read() + if response.status == 200: + policy = Policy(self) + h = handler.XmlHandler(policy, self) + xml.sax.parseString(body, h) + return policy + else: + raise S3ResponseError(response.status, response.reason, body) + + def make_public(self, recursive=False, headers=None): + self.set_canned_acl('public-read', headers=headers) + if recursive: + for key in self: + self.set_canned_acl('public-read', key.name, headers=headers) + + def add_email_grant(self, permission, email_address, recursive=False, headers=None): + """ + Convenience method that provides a quick way to add an email grant to a bucket. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + + :param permission: The permission being granted. Should be one of: (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html for more details on permissions. + :type permission: string + + :param email_address: The email address associated with the AWS account your are granting + the permission to. + :type email_address: string + + :param recursive: A boolean value to controls whether the command will apply the + grant to all keys within the bucket or not. The default value is False. + By passing a True value, the call will iterate through all keys in the + bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take a long time! + :type recursive: boolean + """ + if permission not in S3Permissions: + raise S3PermissionsError('Unknown Permission: %s' % permission) + policy = self.get_acl(headers=headers) + policy.acl.add_email_grant(permission, email_address) + self.set_acl(policy, headers=headers) + if recursive: + for key in self: + key.add_email_grant(permission, email_address, headers=headers) + + def add_user_grant(self, permission, user_id, recursive=False, headers=None): + """ + Convenience method that provides a quick way to add a canonical user grant to a bucket. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + + :type permission: string + :param permission: The permission being granted. Should be one of: + READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html + for more details on permissions. + + :type user_id: string + :param user_id: The canonical user id associated with the AWS account your are granting + the permission to. + + :type recursive: bool + :param recursive: A boolean value that controls whether the command will apply the + grant to all keys within the bucket or not. The default value is False. + By passing a True value, the call will iterate through all keys in the + bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take a long time! + """ + if permission not in S3Permissions: + raise S3PermissionsError('Unknown Permission: %s' % permission) + policy = self.get_acl(headers=headers) + policy.acl.add_user_grant(permission, user_id) + self.set_acl(policy, headers=headers) + if recursive: + for key in self: + key.add_user_grant(permission, user_id, headers=headers) + + def list_grants(self, headers=None): + policy = self.get_acl(headers=headers) + return policy.acl.grants + + def get_location(self): + """ + Returns the LocationConstraint for the bucket. + + :rtype: str + :return: The LocationConstraint for the bucket or the empty string if + no constraint was specified when bucket was created. + """ + response = self.connection.make_request('GET', self.name, + query_args='location') + body = response.read() + if response.status == 200: + rs = ResultSet(self) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.LocationConstraint + else: + raise S3ResponseError(response.status, response.reason, body) + + def enable_logging(self, target_bucket, target_prefix='', headers=None): + if isinstance(target_bucket, Bucket): + target_bucket = target_bucket.name + body = self.BucketLoggingBody % (target_bucket, target_prefix) + response = self.connection.make_request('PUT', self.name, data=body, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise S3ResponseError(response.status, response.reason, body) + + def disable_logging(self, headers=None): + body = self.EmptyBucketLoggingBody + response = self.connection.make_request('PUT', self.name, data=body, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise S3ResponseError(response.status, response.reason, body) + + def get_logging_status(self, headers=None): + response = self.connection.make_request('GET', self.name, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return body + else: + raise S3ResponseError(response.status, response.reason, body) + + def set_as_logging_target(self, headers=None): + policy = self.get_acl(headers=headers) + g1 = Grant(permission='WRITE', type='Group', uri=self.LoggingGroup) + g2 = Grant(permission='READ_ACP', type='Group', uri=self.LoggingGroup) + policy.acl.add_grant(g1) + policy.acl.add_grant(g2) + self.set_acl(policy, headers=headers) + + def disable_logging(self, headers=None): + body = self.EmptyBucketLoggingBody + response = self.connection.make_request('PUT', self.name, data=body, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise S3ResponseError(response.status, response.reason, body) + + def get_request_payment(self, headers=None): + response = self.connection.make_request('GET', self.name, + query_args='requestPayment', headers=headers) + body = response.read() + if response.status == 200: + return body + else: + raise S3ResponseError(response.status, response.reason, body) + + def set_request_payment(self, payer='BucketOwner', headers=None): + body = self.BucketPaymentBody % payer + response = self.connection.make_request('PUT', self.name, data=body, + query_args='requestPayment', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise S3ResponseError(response.status, response.reason, body) + + def delete(self, headers=None): + return self.connection.delete_bucket(self.name, headers=headers) diff --git a/storefront/boto/s3/bucketlistresultset.py b/storefront/boto/s3/bucketlistresultset.py new file mode 100644 index 0000000..66ed4ee --- /dev/null +++ b/storefront/boto/s3/bucketlistresultset.py @@ -0,0 +1,57 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +def bucket_lister(bucket, prefix='', delimiter='', marker='', headers=None): + """ + A generator function for listing keys in a bucket. + """ + more_results = True + k = None + while more_results: + rs = bucket.get_all_keys(prefix=prefix, marker=marker, + delimiter=delimiter, headers=headers) + for k in rs: + yield k + if k: + marker = k.name + more_results= rs.is_truncated + +class BucketListResultSet: + """ + A resultset for listing keys within a bucket. Uses the bucket_lister + generator function and implements the iterator interface. This + transparently handles the results paging from S3 so even if you have + many thousands of keys within the bucket you can iterate over all + keys in a reasonably efficient manner. + """ + + def __init__(self, bucket=None, prefix='', delimiter='', marker='', headers=None): + self.bucket = bucket + self.prefix = prefix + self.delimiter = delimiter + self.marker = marker + self.headers = headers + + def __iter__(self): + return bucket_lister(self.bucket, prefix=self.prefix, + delimiter=self.delimiter, marker=self.marker, headers=self.headers) + + diff --git a/storefront/boto/s3/connection.py b/storefront/boto/s3/connection.py new file mode 100644 index 0000000..e366f7e --- /dev/null +++ b/storefront/boto/s3/connection.py @@ -0,0 +1,343 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax +import urllib, base64 +import time +import boto.utils +import types +from boto.connection import AWSAuthConnection +from boto import handler +from boto.s3.bucket import Bucket +from boto.s3.key import Key +from boto.resultset import ResultSet +from boto.exception import S3ResponseError, S3CreateError, BotoClientError + +def assert_case_insensitive(f): + def wrapper(*args, **kwargs): + if len(args) == 3 and not (args[2].islower() or args[2].isalnum()): + raise BotoClientError("Bucket names cannot contain upper-case " \ + "characters when using either the sub-domain or virtual " \ + "hosting calling format.") + return f(*args, **kwargs) + return wrapper + +class _CallingFormat: + def build_url_base(self, protocol, server, bucket, key=''): + url_base = '%s://' % protocol + url_base += self.build_host(server, bucket) + url_base += self.build_path_base(bucket, key) + return url_base + + def build_host(self, server, bucket): + if bucket == '': + return server + else: + return self.get_bucket_server(server, bucket) + + def build_auth_path(self, bucket, key=''): + path = '' + if bucket != '': + path = '/' + bucket + return path + '/%s' % urllib.quote(key) + + def build_path_base(self, bucket, key=''): + return '/%s' % urllib.quote(key) + +class SubdomainCallingFormat(_CallingFormat): + @assert_case_insensitive + def get_bucket_server(self, server, bucket): + return '%s.%s' % (bucket, server) + +class VHostCallingFormat(_CallingFormat): + @assert_case_insensitive + def get_bucket_server(self, server, bucket): + return bucket + +class OrdinaryCallingFormat(_CallingFormat): + def get_bucket_server(self, server, bucket): + return server + + def build_path_base(self, bucket, key=''): + path_base = '/' + if bucket: + path_base += "%s/" % bucket + return path_base + urllib.quote(key) + +class Location: + DEFAULT = '' + EU = 'EU' + +class S3Connection(AWSAuthConnection): + + DefaultHost = 's3.amazonaws.com' + QueryString = 'Signature=%s&Expires=%d&AWSAccessKeyId=%s' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, + host=DefaultHost, debug=0, https_connection_factory=None, + calling_format=SubdomainCallingFormat(), path='/'): + self.calling_format = calling_format + AWSAuthConnection.__init__(self, host, + aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + debug=debug, https_connection_factory=https_connection_factory, + path=path) + + def __iter__(self): + return self.get_all_buckets() + + def __contains__(self, bucket_name): + return not (self.lookup(bucket_name) is None) + + def build_post_policy(self, expiration_time, conditions): + """ + Taken from the AWS book Python examples and modified for use with boto + """ + if type(expiration_time) != time.struct_time: + raise 'Policy document must include a valid expiration Time object' + + # Convert conditions object mappings to condition statements + + return '{"expiration": "%s",\n"conditions": [%s]}' % \ + (time.strftime(boto.utils.ISO8601, expiration_time), ",".join(conditions)) + + + def build_post_form_args(self, bucket_name, key, expires_in = 6000, + acl = None, success_action_redirect = None, max_content_length = None, + http_method = "http", fields=None, conditions=None): + """ + Taken from the AWS book Python examples and modified for use with boto + This only returns the arguments required for the post form, not the actual form + This does not return the file input field which also needs to be added + + :param bucket_name: Bucket to submit to + :type bucket_name: string + + :param key: Key name, optionally add ${filename} to the end to attach the submitted filename + :type key: string + + :param expires_in: Time (in seconds) before this expires, defaults to 6000 + :type expires_in: integer + + :param acl: ACL rule to use, if any + :type acl: :class:`boto.s3.acl.ACL` + + :param success_action_redirect: URL to redirect to on success + :type success_action_redirect: string + + :param max_content_length: Maximum size for this file + :type max_content_length: integer + + :type http_method: string + :param http_method: HTTP Method to use, "http" or "https" + + + :rtype: dict + :return: A dictionary containing field names/values as well as a url to POST to + + .. code-block:: python + + { + "action": action_url_to_post_to, + "fields": [ + { + "name": field_name, + "value": field_value + }, + { + "name": field_name2, + "value": field_value2 + } + ] + } + + """ + if fields == None: + fields = [] + if conditions == None: + conditions = [] + expiration = time.gmtime(int(time.time() + expires_in)) + + # Generate policy document + conditions.append('{"bucket": "%s"}' % bucket_name) + if key.endswith("${filename}"): + conditions.append('["starts-with", "$key", "%s"]' % key[:-len("${filename}")]) + else: + conditions.append('{"key": "%s"}' % key) + if acl: + conditions.append('{"acl": "%s"}' % acl) + fields.append({ "name": "acl", "value": acl}) + if success_action_redirect: + conditions.append('{"success_action_redirect": "%s"}' % success_action_redirect) + fields.append({ "name": "success_action_redirect", "value": success_action_redirect}) + if max_content_length: + conditions.append('["content-length-range", 0, %i]' % max_content_length) + fields.append({"name":'content-length-range', "value": "0,%i" % max_content_length}) + + policy = self.build_post_policy(expiration, conditions) + + # Add the base64-encoded policy document as the 'policy' field + policy_b64 = base64.b64encode(policy) + fields.append({"name": "policy", "value": policy_b64}) + + # Add the AWS access key as the 'AWSAccessKeyId' field + fields.append({"name": "AWSAccessKeyId", "value": self.aws_access_key_id}) + + # Add signature for encoded policy document as the 'AWSAccessKeyId' field + hmac_copy = self.hmac.copy() + hmac_copy.update(policy_b64) + signature = base64.encodestring(hmac_copy.digest()).strip() + fields.append({"name": "signature", "value": signature}) + fields.append({"name": "key", "value": key}) + + # HTTPS protocol will be used if the secure HTTP option is enabled. + url = '%s://%s.s3.amazonaws.com/' % (http_method, bucket_name) + + return {"action": url, "fields": fields} + + + def generate_url(self, expires_in, method, bucket='', key='', + headers=None, query_auth=True, force_http=False): + if not headers: + headers = {} + expires = int(time.time() + expires_in) + auth_path = self.calling_format.build_auth_path(bucket, key) + canonical_str = boto.utils.canonical_string(method, auth_path, + headers, expires) + hmac_copy = self.hmac.copy() + hmac_copy.update(canonical_str) + b64_hmac = base64.encodestring(hmac_copy.digest()).strip() + encoded_canonical = urllib.quote_plus(b64_hmac) + path = self.calling_format.build_path_base(bucket, key) + if query_auth: + query_part = '?' + self.QueryString % (encoded_canonical, expires, + self.aws_access_key_id) + if 'x-amz-security-token' in headers: + query_part += '&x-amz-security-token=%s' % urllib.quote(headers['x-amz-security-token']); + else: + query_part = '' + if force_http: + protocol = 'http' + port = 80 + else: + protocol = self.protocol + port = self.port + return self.calling_format.build_url_base(protocol, self.server_name(port), + bucket, key) + query_part + + def get_all_buckets(self, headers=None): + response = self.make_request('GET') + body = response.read() + if response.status > 300: + raise S3ResponseError(response.status, response.reason, body, headers=headers) + rs = ResultSet([('Bucket', Bucket)]) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def get_canonical_user_id(self, headers=None): + """ + Convenience method that returns the "CanonicalUserID" of the user who's credentials + are associated with the connection. The only way to get this value is to do a GET + request on the service which returns all buckets associated with the account. As part + of that response, the canonical userid is returned. This method simply does all of + that and then returns just the user id. + + :rtype: string + :return: A string containing the canonical user id. + """ + rs = self.get_all_buckets(headers=headers) + return rs.ID + + def get_bucket(self, bucket_name, validate=True, headers=None): + bucket = Bucket(self, bucket_name) + if validate: + rs = bucket.get_all_keys(headers, maxkeys=0) + return bucket + + def lookup(self, bucket_name, validate=True, headers=None): + try: + bucket = self.get_bucket(bucket_name, validate, headers=headers) + except: + bucket = None + return bucket + + def create_bucket(self, bucket_name, headers=None, location=Location.DEFAULT, policy=None): + """ + Creates a new located bucket. By default it's in the USA. You can pass + Location.EU to create an European bucket. + + :type bucket_name: string + :param bucket_name: The name of the new bucket + + :type headers: dict + :param headers: Additional headers to pass along with the request to AWS. + + :type location: :class:`boto.s3.connection.Location` + :param location: The location of the new bucket + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + """ + if policy: + if headers: + headers['x-amz-acl'] = policy + else: + headers = {'x-amz-acl' : policy} + if location == Location.DEFAULT: + data = '' + else: + data = '' + \ + location + '' + response = self.make_request('PUT', bucket_name, headers=headers, + data=data) + body = response.read() + if response.status == 409: + raise S3CreateError(response.status, response.reason, body) + if response.status == 200: + return Bucket(self, bucket_name) + else: + raise S3ResponseError(response.status, response.reason, body) + + def delete_bucket(self, bucket, headers=None): + response = self.make_request('DELETE', bucket, headers=headers) + body = response.read() + if response.status != 204: + raise S3ResponseError(response.status, response.reason, body) + + def make_request(self, method, bucket='', key='', headers=None, data='', + query_args=None, sender=None): + if isinstance(bucket, Bucket): + bucket = bucket.name + if isinstance(key, Key): + key = key.name + path = self.calling_format.build_path_base(bucket, key) + auth_path = self.calling_format.build_auth_path(bucket, key) + host = self.calling_format.build_host(self.server_name(), bucket) + if query_args: + path += '?' + query_args + auth_path += '?' + query_args + return AWSAuthConnection.make_request(self, method, path, headers, + data, host, auth_path, sender) + diff --git a/storefront/boto/s3/key.py b/storefront/boto/s3/key.py new file mode 100644 index 0000000..ada4352 --- /dev/null +++ b/storefront/boto/s3/key.py @@ -0,0 +1,764 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import mimetypes +import os +import rfc822 +import StringIO +import base64 +import boto.utils +from boto.exception import S3ResponseError, S3DataError, BotoClientError +from boto.s3.user import User +from boto import UserAgent, config +try: + from hashlib import md5 +except ImportError: + from md5 import md5 + +class Key(object): + + DefaultContentType = 'application/octet-stream' + + BufferSize = 8192 + + def __init__(self, bucket=None, name=None): + self.bucket = bucket + self.name = name + self.metadata = {} + self.content_type = self.DefaultContentType + self.content_encoding = None + self.filename = None + self.etag = None + self.last_modified = None + self.owner = None + self.storage_class = None + self.md5 = None + self.base64md5 = None + self.path = None + self.resp = None + self.mode = None + self.size = None + + def __repr__(self): + if self.bucket: + return '' % (self.bucket.name, self.name) + else: + return '' % self.name + + def __getattr__(self, name): + if name == 'key': + return self.name + else: + raise AttributeError + + def __setattr__(self, name, value): + if name == 'key': + self.__dict__['name'] = value + else: + self.__dict__[name] = value + + def __iter__(self): + return self + + def open_read(self, headers=None, query_args=None): + """ + Open this key for reading + + :type headers: dict + :param headers: Headers to pass in the web request + + :type query_args: string + :param query_args: Arguments to pass in the query string (ie, 'torrent') + """ + if self.resp == None: + self.mode = 'r' + + self.resp = self.bucket.connection.make_request('GET', self.bucket.name, self.name, headers, query_args=query_args) + if self.resp.status < 199 or self.resp.status > 299: + raise S3ResponseError(self.resp.status, self.resp.reason) + response_headers = self.resp.msg + self.metadata = boto.utils.get_aws_metadata(response_headers) + for name,value in response_headers.items(): + if name.lower() == 'content-length': + self.size = int(value) + elif name.lower() == 'etag': + self.etag = value + elif name.lower() == 'content-type': + self.content_type = value + elif name.lower() == 'content-encoding': + self.content_encoding = value + elif name.lower() == 'last-modified': + self.last_modified = value + + def open_write(self, headers=None): + """ + Open this key for writing. + Not yet implemented + + :type headers: dict + :param headers: Headers to pass in the write request + """ + raise BotoClientError('Not Implemented') + + def open(self, mode='r', headers=None, query_args=None): + if mode == 'r': + self.mode = 'r' + self.open_read(headers=headers, query_args=query_args) + elif mode == 'w': + self.mode = 'w' + self.open_write(headers=headers) + else: + raise BotoClientError('Invalid mode: %s' % mode) + + closed = False + def close(self): + if self.resp: + self.resp.read() + self.resp = None + self.mode = None + self.closed = True + + def next(self): + """ + By providing a next method, the key object supports use as an iterator. + For example, you can now say: + + for bytes in key: + write bytes to a file or whatever + + All of the HTTP connection stuff is handled for you. + """ + self.open_read() + data = self.resp.read(self.BufferSize) + if not data: + self.close() + raise StopIteration + return data + + def read(self, size=0): + if size == 0: + size = self.BufferSize + self.open_read() + data = self.resp.read(size) + if not data: + self.close() + return data + + def copy(self, dst_bucket, dst_key, metadata=None): + """ + Copy this Key to another bucket. + + :type dst_bucket: string + :param dst_bucket: The name of the destination bucket + + :type dst_key: string + :param dst_key: The name of the destinatino key + + :type metadata: dict + :param metadata: Metadata to be associated with new key. + If metadata is supplied, it will replace the + metadata of the source key being copied. + If no metadata is supplied, the source key's + metadata will be copied to the new key. + + :rtype: :class:`boto.s3.key.Key` or subclass + :returns: An instance of the newly created key object + """ + dst_bucket = self.bucket.connection.lookup(dst_bucket) + return dst_bucket.copy_key(dst_key, self.bucket.name, self.name, metadata) + + def startElement(self, name, attrs, connection): + if name == 'Owner': + self.owner = User(self) + return self.owner + else: + return None + + def endElement(self, name, value, connection): + if name == 'Key': + self.name = value.encode('utf-8') + elif name == 'ETag': + self.etag = value + elif name == 'LastModified': + self.last_modified = value + elif name == 'Size': + self.size = int(value) + elif name == 'StorageClass': + self.storage_class = value + elif name == 'Owner': + pass + else: + setattr(self, name, value) + + def exists(self): + """ + Returns True if the key exists + + :rtype: bool + :return: Whether the key exists on S3 + """ + return bool(self.bucket.lookup(self.name)) + + def delete(self): + """ + Delete this key from S3 + """ + return self.bucket.delete_key(self.name) + + def get_metadata(self, name): + return self.metadata.get(name) + + def set_metadata(self, name, value): + self.metadata[name] = value + + def update_metadata(self, d): + self.metadata.update(d) + + # convenience methods for setting/getting ACL + def set_acl(self, acl_str, headers=None): + if self.bucket != None: + self.bucket.set_acl(acl_str, self.name, headers=headers) + + def get_acl(self, headers=None): + if self.bucket != None: + return self.bucket.get_acl(self.name, headers=headers) + + def get_xml_acl(self, headers=None): + if self.bucket != None: + return self.bucket.get_xml_acl(self.name, headers=headers) + + def set_xml_acl(self, acl_str, headers=None): + if self.bucket != None: + return self.bucket.set_xml_acl(acl_str, self.name, headers=headers) + + def set_canned_acl(self, acl_str, headers=None): + return self.bucket.set_canned_acl(acl_str, self.name, headers) + + def make_public(self, headers=None): + return self.bucket.set_canned_acl('public-read', self.name, headers) + + def generate_url(self, expires_in, method='GET', headers=None, + query_auth=True, force_http=False): + """ + Generate a URL to access this key. + + :type expires_in: int + :param expires_in: How long the url is valid for, in seconds + + :type method: string + :param method: The method to use for retrieving the file (default is GET) + + :type headers: dict + :param headers: Any headers to pass along in the request + + :type query_auth: bool + :param query_auth: + + :rtype: string + :return: The URL to access the key + """ + return self.bucket.connection.generate_url(expires_in, method, + self.bucket.name, self.name, + headers, query_auth, force_http) + + def send_file(self, fp, headers=None, cb=None, num_cb=10): + """ + Upload a file to a key into a bucket on S3. + + :type fp: file + :param fp: The file pointer to upload + + :type headers: dict + :param headers: The headers to pass along with the PUT request + + :type cb: function + :param cb: a callback function that will be called to report + progress on the upload. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted to S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + """ + def sender(http_conn, method, path, data, headers): + http_conn.putrequest(method, path) + for key in headers: + http_conn.putheader(key, headers[key]) + http_conn.endheaders() + fp.seek(0) + save_debug = self.bucket.connection.debug + self.bucket.connection.debug = 0 + if cb: + if num_cb > 2: + cb_count = self.size / self.BufferSize / (num_cb-2) + else: + cb_count = 0 + i = total_bytes = 0 + cb(total_bytes, self.size) + l = fp.read(self.BufferSize) + while len(l) > 0: + http_conn.send(l) + if cb: + total_bytes += len(l) + i += 1 + if i == cb_count: + cb(total_bytes, self.size) + i = 0 + l = fp.read(self.BufferSize) + if cb: + cb(total_bytes, self.size) + response = http_conn.getresponse() + body = response.read() + fp.seek(0) + self.bucket.connection.debug = save_debug + if response.status == 500 or response.status == 503 or \ + response.getheader('location'): + # we'll try again + return response + elif response.status >= 200 and response.status <= 299: + self.etag = response.getheader('etag') + if self.etag != '"%s"' % self.md5: + raise S3DataError('ETag from S3 did not match computed MD5') + return response + else: + raise S3ResponseError(response.status, response.reason, body) + + if not headers: + headers = {} + else: + headers = headers.copy() + headers['User-Agent'] = UserAgent + headers['Content-MD5'] = self.base64md5 + if headers.has_key('Content-Type'): + self.content_type = headers['Content-Type'] + elif self.path: + self.content_type = mimetypes.guess_type(self.path)[0] + if self.content_type == None: + self.content_type = self.DefaultContentType + headers['Content-Type'] = self.content_type + else: + headers['Content-Type'] = self.content_type + headers['Content-Length'] = str(self.size) + headers['Expect'] = '100-Continue' + headers = boto.utils.merge_meta(headers, self.metadata) + return self.bucket.connection.make_request('PUT', self.bucket.name, + self.name, headers, sender=sender) + + def compute_md5(self, fp): + """ + :type fp: file + :param fp: File pointer to the file to MD5 hash. The file pointer will be + reset to the beginning of the file before the method returns. + + :rtype: tuple + :return: A tuple containing the hex digest version of the MD5 hash + as the first element and the base64 encoded version of the + plain digest as the second element. + """ + m = md5() + fp.seek(0) + s = fp.read(self.BufferSize) + while s: + m.update(s) + s = fp.read(self.BufferSize) + hex_md5 = m.hexdigest() + base64md5 = base64.encodestring(m.digest()) + if base64md5[-1] == '\n': + base64md5 = base64md5[0:-1] + self.size = fp.tell() + fp.seek(0) + return (hex_md5, base64md5) + + def set_contents_from_file(self, fp, headers=None, replace=True, cb=None, num_cb=10, + policy=None, md5=None): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the contents of the file pointed to by 'fp' as the + contents. + + :type fp: file + :param fp: the file whose contents to upload + + :type headers: dict + :param headers: additional HTTP headers that will be sent with the PUT request. + + :type replace: bool + :param replace: If this parameter is False, the method + will first check to see if an object exists in the + bucket with the same key. If it does, it won't + overwrite it. The default value is True which will + overwrite the object. + + :type cb: function + :param cb: a callback function that will be called to report + progress on the upload. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted to S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum of the + file as the first element and the Base64-encoded version of the plain + checksum as the second element. This is the same format returned by + the compute_md5 method. + :param md5: If you need to compute the MD5 for any reason prior to upload, + it's silly to have to do it twice so this param, if present, will be + used as the MD5 values of the file. Otherwise, the checksum will be computed. + """ + if policy: + if headers: + headers['x-amz-acl'] = policy + else: + headers = {'x-amz-acl' : policy} + if hasattr(fp, 'name'): + self.path = fp.name + if self.bucket != None: + if not md5: + md5 = self.compute_md5(fp) + self.md5 = md5[0] + self.base64md5 = md5[1] + if self.name == None: + self.name = self.md5 + if not replace: + k = self.bucket.lookup(self.name) + if k: + return + self.send_file(fp, headers, cb, num_cb) + + def set_contents_from_filename(self, filename, headers=None, replace=True, cb=None, num_cb=10, + policy=None, md5=None): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the contents of the file named by 'filename'. + See set_contents_from_file method for details about the + parameters. + + :type filename: string + :param filename: The name of the file that you want to put onto S3 + + :type headers: dict + :param headers: Additional headers to pass along with the request to AWS. + + :type replace: bool + :param replace: If True, replaces the contents of the file if it already exists. + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum of the + file as the first element and the Base64-encoded version of the plain + checksum as the second element. This is the same format returned by + the compute_md5 method. + :param md5: If you need to compute the MD5 for any reason prior to upload, + it's silly to have to do it twice so this param, if present, will be + used as the MD5 values of the file. Otherwise, the checksum will be computed. + """ + fp = open(filename, 'rb') + self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy) + fp.close() + + def set_contents_from_string(self, s, headers=None, replace=True, cb=None, num_cb=10, + policy=None, md5=None): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the string 's' as the contents. + See set_contents_from_file method for details about the + parameters. + + :type headers: dict + :param headers: Additional headers to pass along with the request to AWS. + + :type replace: bool + :param replace: If True, replaces the contents of the file if it already exists. + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum of the + file as the first element and the Base64-encoded version of the plain + checksum as the second element. This is the same format returned by + the compute_md5 method. + :param md5: If you need to compute the MD5 for any reason prior to upload, + it's silly to have to do it twice so this param, if present, will be + used as the MD5 values of the file. Otherwise, the checksum will be computed. + """ + fp = StringIO.StringIO(s) + self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy) + fp.close() + + def get_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieves a file from an S3 Key + + :type fp: file + :param fp: File pointer to put the data into + + :type headers: string + :param: headers to send when retrieving the files + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type torrent: bool + :param torrent: Flag for whether to get a torrent for the file + """ + if cb: + if num_cb > 2: + cb_count = self.size / self.BufferSize / (num_cb-2) + else: + cb_count = 0 + i = total_bytes = 0 + cb(total_bytes, self.size) + save_debug = self.bucket.connection.debug + if self.bucket.connection.debug == 1: + self.bucket.connection.debug = 0 + + if torrent: torrent = "torrent" + self.open('r', headers, query_args=torrent) + for bytes in self: + fp.write(bytes) + if cb: + total_bytes += len(bytes) + i += 1 + if i == cb_count: + cb(total_bytes, self.size) + i = 0 + if cb: + cb(total_bytes, self.size) + self.close() + self.bucket.connection.debug = save_debug + + def get_torrent_file(self, fp, headers=None, cb=None, num_cb=10): + """ + Get a torrent file (see to get_file) + + :type fp: file + :param fp: The file pointer of where to put the torrent + + :type headers: dict + :param headers: Headers to be passed + + :type cb: function + :param cb: Callback function to call on retrieved data + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + """ + return self.get_file(fp, headers, cb, num_cb, torrent=True) + + def get_contents_to_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Write the contents of the object to the file pointed + to by 'fp'. + + :type fp: File -like object + :param fp: + + :type headers: dict + :param headers: additional HTTP headers that will be sent with the GET request. + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type torrent: bool + :param torrent: If True, returns the contents of a torrent file as a string. + + """ + if self.bucket != None: + self.get_file(fp, headers, cb, num_cb, torrent=torrent) + + def get_contents_to_filename(self, filename, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Store contents of the object to a file named by 'filename'. + See get_contents_to_file method for details about the + parameters. + + :type filename: string + :param filename: The filename of where to put the file contents + + :type headers: dict + :param headers: Any additional headers to send in the request + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type torrent: bool + :param torrent: If True, returns the contents of a torrent file as a string. + + """ + fp = open(filename, 'wb') + self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent) + fp.close() + # if last_modified date was sent from s3, try to set file's timestamp + if self.last_modified != None: + try: + modified_tuple = rfc822.parsedate_tz(self.last_modified) + modified_stamp = int(rfc822.mktime_tz(modified_tuple)) + os.utime(fp.name, (modified_stamp, modified_stamp)) + except Exception: pass + + def get_contents_as_string(self, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Return the contents of the object as a string. + See get_contents_to_file method for details about the + parameters. + + :type headers: dict + :param headers: Any additional headers to send in the request + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type torrent: bool + :param torrent: If True, returns the contents of a torrent file as a string. + + :rtype: string + :returns: The contents of the file as a string + """ + fp = StringIO.StringIO() + self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent) + return fp.getvalue() + + def add_email_grant(self, permission, email_address): + """ + Convenience method that provides a quick way to add an email grant to a key. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + + :type permission: string + :param permission: The permission being granted. Should be one of: + READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html + for more details on permissions. + + :type email_address: string + :param email_address: The email address associated with the AWS account your are granting + the permission to. + """ + policy = self.get_acl() + policy.acl.add_email_grant(permission, email_address) + self.set_acl(policy) + + def add_user_grant(self, permission, user_id): + """ + Convenience method that provides a quick way to add a canonical user grant to a key. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + + :type permission: string + :param permission: The permission being granted. Should be one of: + READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html + for more details on permissions. + + :type user_id: string + :param user_id: The canonical user id associated with the AWS account your are granting + the permission to. + """ + policy = self.get_acl() + policy.acl.add_user_grant(permission, user_id) + self.set_acl(policy) diff --git a/storefront/boto/s3/prefix.py b/storefront/boto/s3/prefix.py new file mode 100644 index 0000000..fc0f26a --- /dev/null +++ b/storefront/boto/s3/prefix.py @@ -0,0 +1,35 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Prefix: + def __init__(self, bucket=None, name=None): + self.bucket = bucket + self.name = name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Prefix': + self.name = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/s3/user.py b/storefront/boto/s3/user.py new file mode 100644 index 0000000..f45f038 --- /dev/null +++ b/storefront/boto/s3/user.py @@ -0,0 +1,49 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class User: + def __init__(self, parent=None, id='', display_name=''): + if parent: + parent.owner = self + self.type = None + self.id = id + self.display_name = display_name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'DisplayName': + self.display_name = value + elif name == 'ID': + self.id = value + else: + setattr(self, name, value) + + def to_xml(self, element_name='Owner'): + if self.type: + s = '<%s xsi:type="%s">' % (element_name, self.type) + else: + s = '<%s>' % element_name + s += '%s' % self.id + s += '%s' % self.display_name + s += '' % element_name + return s diff --git a/storefront/boto/sdb/__init__.py b/storefront/boto/sdb/__init__.py new file mode 100644 index 0000000..42af6a9 --- /dev/null +++ b/storefront/boto/sdb/__init__.py @@ -0,0 +1,41 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto +from regioninfo import SDBRegionInfo + +def regions(): + """ + Get all available regions for the SDB service. + + :rtype: list + :return: A list of :class:`boto.sdb.regioninfo.RegionInfo` + """ + return [SDBRegionInfo(name='us-east-1', endpoint='sdb.amazonaws.com'), + SDBRegionInfo(name='eu-west-1', endpoint='sdb.eu-west-1.amazonaws.com'), + SDBRegionInfo(name='us-west-1', endpoint='sdb.us-west-1.amazonaws.com')] + +def connect_to_region(region_name): + for region in regions(): + if region.name == region_name: + return region.connect() + return None diff --git a/storefront/boto/sdb/connection.py b/storefront/boto/sdb/connection.py new file mode 100644 index 0000000..28e130a --- /dev/null +++ b/storefront/boto/sdb/connection.py @@ -0,0 +1,471 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import urllib +import xml.sax +import threading +import boto +from boto import handler +from boto.connection import AWSQueryConnection +from boto.sdb.domain import Domain, DomainMetaData +from boto.sdb.item import Item +from boto.sdb.regioninfo import SDBRegionInfo +from boto.exception import SDBResponseError +from boto.resultset import ResultSet +import warnings + +class ItemThread(threading.Thread): + + def __init__(self, name, domain_name, item_names): + threading.Thread.__init__(self, name=name) + print 'starting %s with %d items' % (name, len(item_names)) + self.domain_name = domain_name + self.conn = SDBConnection() + self.item_names = item_names + self.items = [] + + def run(self): + for item_name in self.item_names: + item = self.conn.get_attributes(self.domain_name, item_name) + self.items.append(item) + +class SDBConnection(AWSQueryConnection): + + DefaultRegionName = 'us-east-1' + DefaultRegionEndpoint = 'sdb.amazonaws.com' + APIVersion = '2007-11-07' + SignatureVersion = '2' + ResponseError = SDBResponseError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/', converter=None): + if not region: + region = SDBRegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + self.region.endpoint, debug, https_connection_factory, path) + self.box_usage = 0.0 + self.converter = converter + self.item_cls = Item + + def set_item_cls(self, cls): + self.item_cls = cls + + def build_name_value_list(self, params, attributes, replace=False): + keys = attributes.keys() + keys.sort() + i = 1 + for key in keys: + value = attributes[key] + if isinstance(value, list): + for v in value: + params['Attribute.%d.Name'%i] = key + if self.converter: + v = self.converter.encode(v) + params['Attribute.%d.Value'%i] = v + if replace: + params['Attribute.%d.Replace'%i] = 'true' + i += 1 + else: + params['Attribute.%d.Name'%i] = key + if self.converter: + value = self.converter.encode(value) + params['Attribute.%d.Value'%i] = value + if replace: + params['Attribute.%d.Replace'%i] = 'true' + i += 1 + + def build_batch_list(self, params, items, replace=False): + item_names = items.keys() + i = 0 + for item_name in item_names: + j = 0 + item = items[item_name] + attr_names = item.keys() + params['Item.%d.ItemName' % i] = item_name + for attr_name in attr_names: + value = item[attr_name] + if isinstance(value, list): + for v in value: + if self.converter: + v = self.converter.encode(v) + params['Item.%d.Attribute.%d.Name' % (i,j)] = attr_name + params['Item.%d.Attribute.%d.Value' % (i,j)] = v + if replace: + params['Item.%d.Attribute.%d.Replace' % (i,j)] = 'true' + j += 1 + else: + params['Item.%d.Attribute.%d.Name' % (i,j)] = attr_name + if self.converter: + value = self.converter.encode(value) + params['Item.%d.Attribute.%d.Value' % (i,j)] = value + if replace: + params['Item.%d.Attribute.%d.Replace' % (i,j)] = 'true' + j += 1 + i += 1 + + def build_name_list(self, params, attribute_names): + i = 1 + attribute_names.sort() + for name in attribute_names: + params['Attribute.%d.Name'%i] = name + i += 1 + + def get_usage(self): + """ + Returns the BoxUsage accumulated on this SDBConnection object. + + :rtype: float + :return: The accumulated BoxUsage of all requests made on the connection. + """ + return self.box_usage + + def print_usage(self): + """ + Print the BoxUsage and approximate costs of all requests made on this connection. + """ + print 'Total Usage: %f compute seconds' % self.box_usage + cost = self.box_usage * 0.14 + print 'Approximate Cost: $%f' % cost + + def get_domain(self, domain_name, validate=True): + domain = Domain(self, domain_name) + if validate: + self.select(domain, """select * from `%s` limit 1""" % domain_name) + return domain + + def lookup(self, domain_name, validate=True): + """ + Lookup an existing SimpleDB domain + + :type domain_name: string + :param domain_name: The name of the new domain + + :rtype: :class:`boto.sdb.domain.Domain` object or None + :return: The Domain object or None if the domain does not exist. + """ + try: + domain = self.get_domain(domain_name, validate) + except: + domain = None + return domain + + def get_all_domains(self, max_domains=None, next_token=None): + params = {} + if max_domains: + params['MaxNumberOfDomains'] = max_domains + if next_token: + params['NextToken'] = next_token + return self.get_list('ListDomains', params, [('DomainName', Domain)]) + + def create_domain(self, domain_name): + """ + Create a SimpleDB domain. + + :type domain_name: string + :param domain_name: The name of the new domain + + :rtype: :class:`boto.sdb.domain.Domain` object + :return: The newly created domain + """ + params = {'DomainName':domain_name} + d = self.get_object('CreateDomain', params, Domain) + d.name = domain_name + return d + + def get_domain_and_name(self, domain_or_name): + if (isinstance(domain_or_name, Domain)): + return (domain_or_name, domain_or_name.name) + else: + return (self.get_domain(domain_or_name), domain_or_name) + + def delete_domain(self, domain_or_name): + """ + Delete a SimpleDB domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :rtype: bool + :return: True if successful + + B{Note:} This will delete the domain and all items within the domain. + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name} + return self.get_status('DeleteDomain', params) + + def domain_metadata(self, domain_or_name): + """ + Get the Metadata for a SimpleDB domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :rtype: :class:`boto.sdb.domain.DomainMetaData` object + :return: The newly created domain metadata object + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name} + d = self.get_object('DomainMetadata', params, DomainMetaData) + d.domain = domain + return d + + def put_attributes(self, domain_or_name, item_name, attributes, replace=True): + """ + Store attributes for a given item in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type item_name: string + :param item_name: The name of the item whose attributes are being stored. + + :type attribute_names: dict or dict-like object + :param attribute_names: The name/value pairs to store as attributes + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name, + 'ItemName' : item_name} + self.build_name_value_list(params, attributes, replace) + return self.get_status('PutAttributes', params) + + def batch_put_attributes(self, domain_or_name, items, replace=True): + """ + Store attributes for multiple items in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type items: dict or dict-like object + :param items: A dictionary-like object. The keys of the dictionary are + the item names and the values are themselves dictionaries + of attribute names/values, exactly the same as the + attribute_names parameter of the scalar put_attributes + call. + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name} + self.build_batch_list(params, items, replace) + return self.get_status('BatchPutAttributes', params, verb='POST') + + def get_attributes(self, domain_or_name, item_name, attribute_names=None, item=None): + """ + Retrieve attributes for a given item in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type item_name: string + :param item_name: The name of the item whose attributes are being retrieved. + + :type attribute_names: string or list of strings + :param attribute_names: An attribute name or list of attribute names. This + parameter is optional. If not supplied, all attributes + will be retrieved for the item. + + :rtype: :class:`boto.sdb.item.Item` + :return: An Item mapping type containing the requested attribute name/values + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name, + 'ItemName' : item_name} + if attribute_names: + if not isinstance(attribute_names, list): + attribute_names = [attribute_names] + self.build_list_params(params, attribute_names, 'AttributeName') + response = self.make_request('GetAttributes', params) + body = response.read() + if response.status == 200: + if item == None: + item = self.item_cls(domain, item_name) + h = handler.XmlHandler(item, self) + xml.sax.parseString(body, h) + return item + else: + raise SDBResponseError(response.status, response.reason, body) + + def delete_attributes(self, domain_or_name, item_name, attr_names=None): + """ + Delete attributes from a given item in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type item_name: string + :param item_name: The name of the item whose attributes are being deleted. + + :type attributes: dict, list or :class:`boto.sdb.item.Item` + :param attributes: Either a list containing attribute names which will cause + all values associated with that attribute name to be deleted or + a dict or Item containing the attribute names and keys and list + of values to delete as the value. If no value is supplied, + all attribute name/values for the item will be deleted. + + :rtype: bool + :return: True if successful + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name, + 'ItemName' : item_name} + if attr_names: + if isinstance(attr_names, list): + self.build_name_list(params, attr_names) + elif isinstance(attr_names, dict) or isinstance(attr_names, self.item_cls): + self.build_name_value_list(params, attr_names) + return self.get_status('DeleteAttributes', params) + + def query(self, domain_or_name, query='', max_items=None, next_token=None): + """ + Returns a list of item names within domain_name that match the query. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type query: string + :param query: The SimpleDB query to be performed. + + :type max_items: int + :param max_items: The maximum number of items to return. If not + supplied, the default is None which returns all + items matching the query. + + :rtype: ResultSet + :return: An iterator containing the results. + """ + warnings.warn('Query interface is deprecated', DeprecationWarning) + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name, + 'QueryExpression' : query} + if max_items: + params['MaxNumberOfItems'] = max_items + if next_token: + params['NextToken'] = next_token + return self.get_object('Query', params, ResultSet) + + def query_with_attributes(self, domain_or_name, query='', attr_names=None, + max_items=None, next_token=None): + """ + Returns a set of Attributes for item names within domain_name that match the query. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type query: string + :param query: The SimpleDB query to be performed. + + :type attr_names: list + :param attr_names: The name of the attributes to be returned. + If no attributes are specified, all attributes + will be returned. + + :type max_items: int + :param max_items: The maximum number of items to return. If not + supplied, the default is None which returns all + items matching the query. + + :rtype: ResultSet + :return: An iterator containing the results. + """ + warnings.warn('Query interface is deprecated', DeprecationWarning) + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name, + 'QueryExpression' : query} + if max_items: + params['MaxNumberOfItems'] = max_items + if next_token: + params['NextToken'] = next_token + if attr_names: + self.build_list_params(params, attr_names, 'AttributeName') + return self.get_list('QueryWithAttributes', params, [('Item', self.item_cls)], parent=domain) + + def select(self, domain_or_name, query='', next_token=None): + """ + Returns a set of Attributes for item names within domain_name that match the query. + The query must be expressed in using the SELECT style syntax rather than the + original SimpleDB query language. + Even though the select request does not require a domain object, a domain + object must be passed into this method so the Item objects returned can + point to the appropriate domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type query: string + :param query: The SimpleDB query to be performed. + + :rtype: ResultSet + :return: An iterator containing the results. + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'SelectExpression' : query} + if next_token: + params['NextToken'] = next_token + return self.get_list('Select', params, [('Item', self.item_cls)], parent=domain) + + def threaded_query(self, domain_or_name, query='', max_items=None, next_token=None, num_threads=6): + """ + Returns a list of fully populated items that match the query provided. + + The name/value pairs for all of the matching item names are retrieved in a number of separate + threads (specified by num_threads) to achieve maximum throughput. + The ResultSet that is returned has an attribute called next_token that can be used + to retrieve additional results for the same query. + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + if max_items and num_threads > max_items: + num_threads = max_items + rs = self.query(domain_or_name, query, max_items, next_token) + threads = [] + n = len(rs) / num_threads + for i in range(0, num_threads): + if i+1 == num_threads: + thread = ItemThread('Thread-%d' % i, domain_name, rs[n*i:]) + else: + thread = ItemThread('Thread-%d' % i, domain_name, rs[n*i:n*(i+1)]) + threads.append(thread) + thread.start() + del rs[0:] + for thread in threads: + thread.join() + for item in thread.items: + rs.append(item) + return rs + diff --git a/storefront/boto/sdb/db/__init__.py b/storefront/boto/sdb/db/__init__.py new file mode 100644 index 0000000..86044ed --- /dev/null +++ b/storefront/boto/sdb/db/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + diff --git a/storefront/boto/sdb/db/blob.py b/storefront/boto/sdb/db/blob.py new file mode 100644 index 0000000..d92eb65 --- /dev/null +++ b/storefront/boto/sdb/db/blob.py @@ -0,0 +1,64 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +class Blob(object): + """Blob object""" + def __init__(self, value=None, file=None, id=None): + self._file = file + self.id = id + self.value = value + + @property + def file(self): + from StringIO import StringIO + if self._file: + f = self._file + else: + f = StringIO(self.value) + return f + + def __str__(self): + if hasattr(self.file, "get_contents_as_string"): + return str(self.file.get_contents_as_string()) + else: + return str(self.file.getvalue()) + + def read(self): + return self.file.read() + + def readline(self): + return self.file.readline() + + def next(self): + return sefl.file.next() + + def __iter__(self): + return iter(self.file) + + @property + def size(self): + if self._file: + return self._file.size + elif self.value: + return len(self.value) + else: + return 0 diff --git a/storefront/boto/sdb/db/key.py b/storefront/boto/sdb/db/key.py new file mode 100644 index 0000000..42a9d8d --- /dev/null +++ b/storefront/boto/sdb/db/key.py @@ -0,0 +1,59 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Key(object): + + @classmethod + def from_path(cls, *args, **kwds): + raise NotImplementedError, "Paths are not currently supported" + + def __init__(self, encoded=None, obj=None): + self.name = None + if obj: + self.id = obj.id + self.kind = obj.kind() + else: + self.id = None + self.kind = None + + def app(self): + raise NotImplementedError, "Applications are not currently supported" + + def kind(self): + return self.kind + + def id(self): + return self.id + + def name(self): + raise NotImplementedError, "Key Names are not currently supported" + + def id_or_name(self): + return self.id + + def has_id_or_name(self): + return self.id != None + + def parent(self): + raise NotImplementedError, "Key parents are not currently supported" + + def __str__(self): + return self.id_or_name() diff --git a/storefront/boto/sdb/db/manager/__init__.py b/storefront/boto/sdb/db/manager/__init__.py new file mode 100644 index 0000000..1d75549 --- /dev/null +++ b/storefront/boto/sdb/db/manager/__init__.py @@ -0,0 +1,86 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import boto + +def get_manager(cls): + """ + Returns the appropriate Manager class for a given Model class. It does this by + looking in the boto config for a section like this:: + + [DB] + db_type = SimpleDB + db_user = + db_passwd = + db_name = my_domain + [DB_TestBasic] + db_type = SimpleDB + db_user = + db_passwd = + db_name = basic_domain + db_port = 1111 + + The values in the DB section are "generic values" that will be used if nothing more + specific is found. You can also create a section for a specific Model class that + gives the db info for that class. In the example above, TestBasic is a Model subclass. + """ + db_user = boto.config.get('DB', 'db_user', None) + db_passwd = boto.config.get('DB', 'db_passwd', None) + db_type = boto.config.get('DB', 'db_type', 'SimpleDB') + db_name = boto.config.get('DB', 'db_name', None) + db_table = boto.config.get('DB', 'db_table', None) + db_host = boto.config.get('DB', 'db_host', "sdb.amazonaws.com") + db_port = boto.config.getint('DB', 'db_port', 443) + enable_ssl = boto.config.getbool('DB', 'enable_ssl', True) + sql_dir = boto.config.get('DB', 'sql_dir', None) + debug = boto.config.getint('DB', 'debug', 0) + # first see if there is a fully qualified section name in the Boto config file + module_name = cls.__module__.replace('.', '_') + db_section = 'DB_' + module_name + '_' + cls.__name__ + if not boto.config.has_section(db_section): + db_section = 'DB_' + cls.__name__ + if boto.config.has_section(db_section): + db_user = boto.config.get(db_section, 'db_user', db_user) + db_passwd = boto.config.get(db_section, 'db_passwd', db_passwd) + db_type = boto.config.get(db_section, 'db_type', db_type) + db_name = boto.config.get(db_section, 'db_name', db_name) + db_table = boto.config.get(db_section, 'db_table', db_table) + db_host = boto.config.get(db_section, 'db_host', db_host) + db_port = boto.config.getint(db_section, 'db_port', db_port) + enable_ssl = boto.config.getint(db_section, 'enable_ssl', enable_ssl) + debug = boto.config.getint(db_section, 'debug', debug) + if db_type == 'SimpleDB': + from sdbmanager import SDBManager + return SDBManager(cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl) + elif db_type == 'PostgreSQL': + from pgmanager import PGManager + if db_table: + return PGManager(cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl) + else: + return None + elif db_type == 'XML': + from xmlmanager import XMLManager + return XMLManager(cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl) + else: + raise ValueError, 'Unknown db_type: %s' % db_type + diff --git a/storefront/boto/sdb/db/manager/pgmanager.py b/storefront/boto/sdb/db/manager/pgmanager.py new file mode 100644 index 0000000..4c7e3ad --- /dev/null +++ b/storefront/boto/sdb/db/manager/pgmanager.py @@ -0,0 +1,387 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +from boto.sdb.db.key import Key +from boto.sdb.db.model import Model +import psycopg2 +import psycopg2.extensions +import uuid, sys, os, string +from boto.exception import * + +psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) + +class PGConverter: + + def __init__(self, manager): + self.manager = manager + self.type_map = {Key : (self.encode_reference, self.decode_reference), + Model : (self.encode_reference, self.decode_reference)} + + def encode(self, type, value): + if type in self.type_map: + encode = self.type_map[type][0] + return encode(value) + return value + + def decode(self, type, value): + if type in self.type_map: + decode = self.type_map[type][1] + return decode(value) + return value + + def encode_prop(self, prop, value): + if isinstance(value, list): + if hasattr(prop, 'item_type'): + s = "{" + new_value = [] + for v in value: + item_type = getattr(prop, 'item_type') + if Model in item_type.mro(): + item_type = Model + new_value.append('%s' % self.encode(item_type, v)) + s += ','.join(new_value) + s += "}" + return s + else: + return value + return self.encode(prop.data_type, value) + + def decode_prop(self, prop, value): + if prop.data_type == list: + if value != None: + if not isinstance(value, list): + value = [value] + if hasattr(prop, 'item_type'): + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + if item_type != self.manager.cls: + return item_type._manager.decode_value(prop, value) + else: + item_type = Model + return [self.decode(item_type, v) for v in value] + return value + elif hasattr(prop, 'reference_class'): + ref_class = getattr(prop, 'reference_class') + if ref_class != self.manager.cls: + return ref_class._manager.decode_value(prop, value) + else: + return self.decode(prop.data_type, value) + elif hasattr(prop, 'calculated_type'): + calc_type = getattr(prop, 'calculated_type') + return self.decode(calc_type, value) + else: + return self.decode(prop.data_type, value) + + def encode_reference(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return value.id + + def decode_reference(self, value): + if not value: + return None + try: + return self.manager.get_object_from_id(value) + except: + raise ValueError, 'Unable to convert %s to Object' % value + +class PGManager(object): + + def __init__(self, cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl): + self.cls = cls + self.db_name = db_name + self.db_user = db_user + self.db_passwd = db_passwd + self.db_host = db_host + self.db_port = db_port + self.db_table = db_table + self.sql_dir = sql_dir + self.in_transaction = False + self.converter = PGConverter(self) + self._connect() + + def _build_connect_string(self): + cs = 'dbname=%s user=%s password=%s host=%s port=%d' + return cs % (self.db_name, self.db_user, self.db_passwd, + self.db_host, self.db_port) + + def _connect(self): + self.connection = psycopg2.connect(self._build_connect_string()) + self.connection.set_client_encoding('UTF8') + self.cursor = self.connection.cursor() + + def _object_lister(self, cursor): + try: + for row in cursor: + yield self._object_from_row(row, cursor.description) + except StopIteration: + cursor.close() + raise StopIteration + + def _dict_from_row(self, row, description): + d = {} + for i in range(0, len(row)): + d[description[i][0]] = row[i] + return d + + def _object_from_row(self, row, description=None): + if not description: + description = self.cursor.description + d = self._dict_from_row(row, description) + obj = self.cls(d['id']) + obj._manager = self + obj._auto_update = False + for prop in obj.properties(hidden=False): + if prop.data_type != Key: + v = self.decode_value(prop, d[prop.name]) + v = prop.make_value_from_datastore(v) + if hasattr(prop, 'calculated_type'): + prop._set_direct(obj, v) + elif not prop.empty(v): + setattr(obj, prop.name, v) + else: + setattr(obj, prop.name, prop.default_value()) + return obj + + def _build_insert_qs(self, obj, calculated): + fields = [] + values = [] + templs = [] + id_calculated = [p for p in calculated if p.name == 'id'] + for prop in obj.properties(hidden=False): + if prop not in calculated: + value = prop.get_value_for_datastore(obj) + if value != prop.default_value() or prop.required: + value = self.encode_value(prop, value) + values.append(value) + fields.append('"%s"' % prop.name) + templs.append('%s') + qs = 'INSERT INTO "%s" (' % self.db_table + if len(id_calculated) == 0: + qs += '"id",' + qs += ','.join(fields) + qs += ") VALUES (" + if len(id_calculated) == 0: + qs += "'%s'," % obj.id + qs += ','.join(templs) + qs += ')' + if calculated: + qs += ' RETURNING ' + calc_values = ['"%s"' % p.name for p in calculated] + qs += ','.join(calc_values) + qs += ';' + return qs, values + + def _build_update_qs(self, obj, calculated): + fields = [] + values = [] + for prop in obj.properties(hidden=False): + if prop not in calculated: + value = prop.get_value_for_datastore(obj) + if value != prop.default_value() or prop.required: + value = self.encode_value(prop, value) + values.append(value) + field = '"%s"=' % prop.name + field += '%s' + fields.append(field) + qs = 'UPDATE "%s" SET ' % self.db_table + qs += ','.join(fields) + qs += """ WHERE "id" = '%s'""" % obj.id + if calculated: + qs += ' RETURNING ' + calc_values = ['"%s"' % p.name for p in calculated] + qs += ','.join(calc_values) + qs += ';' + return qs, values + + def _get_sql(self, mapping=None): + print '_get_sql' + sql = None + if self.sql_dir: + path = os.path.join(self.sql_dir, self.cls.__name__ + '.sql') + print path + if os.path.isfile(path): + fp = open(path) + sql = fp.read() + fp.close() + t = string.Template(sql) + sql = t.safe_substitute(mapping) + return sql + + def start_transaction(self): + print 'start_transaction' + self.in_transaction = True + + def end_transaction(self): + print 'end_transaction' + self.in_transaction = False + self.commit() + + def commit(self): + if not self.in_transaction: + print '!!commit on %s' % self.db_table + try: + self.connection.commit() + + except psycopg2.ProgrammingError, err: + self.connection.rollback() + raise err + + def rollback(self): + print '!!rollback on %s' % self.db_table + self.connection.rollback() + + def delete_table(self): + self.cursor.execute('DROP TABLE "%s";' % self.db_table) + self.commit() + + def create_table(self, mapping=None): + self.cursor.execute(self._get_sql(mapping)) + self.commit() + + def encode_value(self, prop, value): + return self.converter.encode_prop(prop, value) + + def decode_value(self, prop, value): + return self.converter.decode_prop(prop, value) + + def execute_sql(self, query): + self.cursor.execute(query, None) + self.commit() + + def query_sql(self, query, vars=None): + self.cursor.execute(query, vars) + return self.cursor.fetchall() + + def lookup(self, cls, name, value): + values = [] + qs = 'SELECT * FROM "%s" WHERE ' % self.db_table + found = False + for property in cls.properties(hidden=False): + if property.name == name: + found = True + value = self.encode_value(property, value) + values.append(value) + qs += "%s=" % name + qs += "%s" + if not found: + raise SDBPersistenceError('%s is not a valid field' % key) + qs += ';' + print qs + self.cursor.execute(qs, values) + if self.cursor.rowcount == 1: + row = self.cursor.fetchone() + return self._object_from_row(row, self.cursor.description) + elif self.cursor.rowcount == 0: + raise KeyError, 'Object not found' + else: + raise LookupError, 'Multiple Objects Found' + + def query(self, cls, filters, limit=None, order_by=None): + parts = [] + qs = 'SELECT * FROM "%s"' % self.db_table + if filters: + qs += ' WHERE ' + properties = cls.properties(hidden=False) + for filter, value in filters: + name, op = filter.strip().split() + found = False + for property in properties: + if property.name == name: + found = True + value = self.encode_value(property, value) + parts.append(""""%s"%s'%s'""" % (name, op, value)) + if not found: + raise SDBPersistenceError('%s is not a valid field' % key) + qs += ','.join(parts) + qs += ';' + print qs + cursor = self.connection.cursor() + cursor.execute(qs) + return self._object_lister(cursor) + + def get_property(self, prop, obj, name): + qs = """SELECT "%s" FROM "%s" WHERE id='%s';""" % (name, self.db_table, obj.id) + print qs + self.cursor.execute(qs, None) + if self.cursor.rowcount == 1: + rs = self.cursor.fetchone() + for prop in obj.properties(hidden=False): + if prop.name == name: + v = self.decode_value(prop, rs[0]) + return v + raise AttributeError, '%s not found' % name + + def set_property(self, prop, obj, name, value): + pass + value = self.encode_value(prop, value) + qs = 'UPDATE "%s" SET ' % self.db_table + qs += "%s='%s'" % (name, self.encode_value(prop, value)) + qs += " WHERE id='%s'" % obj.id + qs += ';' + print qs + self.cursor.execute(qs) + self.commit() + + def get_object(self, cls, id): + qs = """SELECT * FROM "%s" WHERE id='%s';""" % (self.db_table, id) + self.cursor.execute(qs, None) + if self.cursor.rowcount == 1: + row = self.cursor.fetchone() + return self._object_from_row(row, self.cursor.description) + else: + raise SDBPersistenceError('%s object with id=%s does not exist' % (cls.__name__, id)) + + def get_object_from_id(self, id): + return self.get_object(self.cls, id) + + def _find_calculated_props(self, obj): + return [p for p in obj.properties() if hasattr(p, 'calculated_type')] + + def save_object(self, obj): + obj._auto_update = False + calculated = self._find_calculated_props(obj) + if not obj.id: + obj.id = str(uuid.uuid4()) + qs, values = self._build_insert_qs(obj, calculated) + else: + qs, values = self._build_update_qs(obj, calculated) + print qs + self.cursor.execute(qs, values) + if calculated: + calc_values = self.cursor.fetchone() + print calculated + print calc_values + for i in range(0, len(calculated)): + prop = calculated[i] + prop._set_direct(obj, calc_values[i]) + self.commit() + + def delete_object(self, obj): + qs = """DELETE FROM "%s" WHERE id='%s';""" % (self.db_table, obj.id) + print qs + self.cursor.execute(qs) + self.commit() + + diff --git a/storefront/boto/sdb/db/manager/sdbmanager.py b/storefront/boto/sdb/db/manager/sdbmanager.py new file mode 100644 index 0000000..2bb2440 --- /dev/null +++ b/storefront/boto/sdb/db/manager/sdbmanager.py @@ -0,0 +1,518 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import boto +import re +from boto.utils import find_class +import uuid +from boto.sdb.db.key import Key +from boto.sdb.db.model import Model +from boto.sdb.db.blob import Blob +from boto.sdb.db.property import ListProperty, MapProperty +from datetime import datetime +from boto.exception import SDBPersistenceError +from tempfile import TemporaryFile + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +class SDBConverter: + """ + Responsible for converting base Python types to format compatible with underlying + database. For SimpleDB, that means everything needs to be converted to a string + when stored in SimpleDB and from a string when retrieved. + + To convert a value, pass it to the encode or decode method. The encode method + will take a Python native value and convert to DB format. The decode method will + take a DB format value and convert it to Python native format. To find the appropriate + method to call, the generic encode/decode methods will look for the type-specific + method by searching for a method called "encode_" or "decode_". + """ + def __init__(self, manager): + self.manager = manager + self.type_map = { bool : (self.encode_bool, self.decode_bool), + int : (self.encode_int, self.decode_int), + long : (self.encode_long, self.decode_long), + float : (self.encode_float, self.decode_float), + Model : (self.encode_reference, self.decode_reference), + Key : (self.encode_reference, self.decode_reference), + datetime : (self.encode_datetime, self.decode_datetime), + Blob: (self.encode_blob, self.decode_blob), + } + + def encode(self, item_type, value): + if item_type in self.type_map: + encode = self.type_map[item_type][0] + return encode(value) + return value + + def decode(self, item_type, value): + if item_type in self.type_map: + decode = self.type_map[item_type][1] + return decode(value) + return value + + def encode_list(self, prop, value): + if not isinstance(value, list): + value = [value] + new_value = [] + for v in value: + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + item_type = Model + new_value.append(self.encode(item_type, v)) + return new_value + + def encode_map(self, prop, value): + if not isinstance(value, dict): + raise ValueError, 'Expected a dict value, got %s' % type(value) + new_value = [] + for key in value: + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + item_type = Model + encoded_value = self.encode(item_type, value[key]) + new_value.append('%s:%s' % (key, encoded_value)) + return new_value + + def encode_prop(self, prop, value): + if isinstance(prop, ListProperty): + return self.encode_list(prop, value) + elif isinstance(prop, MapProperty): + return self.encode_map(prop, value) + else: + return self.encode(prop.data_type, value) + + def decode_list(self, prop, value): + if not isinstance(value, list): + value = [value] + if hasattr(prop, 'item_type'): + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + return [item_type(id=v) for v in value] + return [self.decode(item_type, v) for v in value] + else: + return value + + def decode_map(self, prop, value): + if not isinstance(value, list): + value = [value] + ret_value = {} + item_type = getattr(prop, "item_type") + for keyval in value: + key, val = keyval.split(':', 1) + if Model in item_type.mro(): + val = item_type(id=val) + else: + val = self.decode(item_type, val) + ret_value[key] = val + return ret_value + + def decode_prop(self, prop, value): + if isinstance(prop, ListProperty): + return self.decode_list(prop, value) + elif isinstance(prop, MapProperty): + return self.decode_map(prop, value) + else: + return self.decode(prop.data_type, value) + + def encode_int(self, value): + value = int(value) + value += 2147483648 + return '%010d' % value + + def decode_int(self, value): + value = int(value) + value -= 2147483648 + return int(value) + + def encode_long(self, value): + value = long(value) + value += 9223372036854775808 + return '%020d' % value + + def decode_long(self, value): + value = long(value) + value -= 9223372036854775808 + return value + + def encode_bool(self, value): + if value == True: + return 'true' + else: + return 'false' + + def decode_bool(self, value): + if value.lower() == 'true': + return True + else: + return False + + def encode_float(self, value): + """ + See http://tools.ietf.org/html/draft-wood-ldapext-float-00. + """ + s = '%e' % value + l = s.split('e') + mantissa = l[0].ljust(18, '0') + exponent = l[1] + if value == 0.0: + case = '3' + exponent = '000' + elif mantissa[0] != '-' and exponent[0] == '+': + case = '5' + exponent = exponent[1:].rjust(3, '0') + elif mantissa[0] != '-' and exponent[0] == '-': + case = '4' + exponent = 999 + int(exponent) + exponent = '%03d' % exponent + elif mantissa[0] == '-' and exponent[0] == '-': + case = '2' + mantissa = '%f' % (10 + float(mantissa)) + mantissa = mantissa.ljust(18, '0') + exponent = exponent[1:].rjust(3, '0') + else: + case = '1' + mantissa = '%f' % (10 + float(mantissa)) + mantissa = mantissa.ljust(18, '0') + exponent = 999 - int(exponent) + exponent = '%03d' % exponent + return '%s %s %s' % (case, exponent, mantissa) + + def decode_float(self, value): + case = value[0] + exponent = value[2:5] + mantissa = value[6:] + if case == '3': + return 0.0 + elif case == '5': + pass + elif case == '4': + exponent = '%03d' % (int(exponent) - 999) + elif case == '2': + mantissa = '%f' % (float(mantissa) - 10) + exponent = '-' + exponent + else: + mantissa = '%f' % (float(mantissa) - 10) + exponent = '%03d' % abs((int(exponent) - 999)) + return float(mantissa + 'e' + exponent) + + def encode_datetime(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + return value.strftime(ISO8601) + + def decode_datetime(self, value): + try: + return datetime.strptime(value, ISO8601) + except: + return None + + def encode_reference(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return value.id + + def decode_reference(self, value): + if not value: + return None + return value + + def encode_blob(self, value): + if not value: + return None + + if not value.id: + bucket = self.manager.get_blob_bucket() + key = bucket.new_key(str(uuid.uuid4())) + value.id = "s3://%s/%s" % (key.bucket.name, key.name) + else: + match = re.match("^s3:\/\/([^\/]*)\/(.*)$", value.id) + if match: + s3 = self.manager.get_s3_connection() + bucket = s3.get_bucket(match.group(1), validate=False) + key = bucket.get_key(match.group(2)) + else: + raise SDBPersistenceError("Invalid Blob ID: %s" % value.id) + + if value.value != None: + key.set_contents_from_string(value.value) + return value.id + + + def decode_blob(self, value): + if not value: + return None + match = re.match("^s3:\/\/([^\/]*)\/(.*)$", value) + if match: + s3 = self.manager.get_s3_connection() + bucket = s3.get_bucket(match.group(1), validate=False) + key = bucket.get_key(match.group(2)) + else: + return None + if key: + return Blob(file=key, id="s3://%s/%s" % (key.bucket.name, key.name)) + else: + return None + +class SDBManager(object): + + def __init__(self, cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, ddl_dir, enable_ssl): + self.cls = cls + self.db_name = db_name + self.db_user = db_user + self.db_passwd = db_passwd + self.db_host = db_host + self.db_port = db_port + self.db_table = db_table + self.ddl_dir = ddl_dir + self.enable_ssl = enable_ssl + self.s3 = None + self.bucket = None + self.converter = SDBConverter(self) + self._connect() + + def _connect(self): + self.sdb = boto.connect_sdb(aws_access_key_id=self.db_user, + aws_secret_access_key=self.db_passwd, + is_secure=self.enable_ssl) + # This assumes that the domain has already been created + # It's much more efficient to do it this way rather than + # having this make a roundtrip each time to validate. + # The downside is that if the domain doesn't exist, it breaks + self.domain = self.sdb.lookup(self.db_name, validate=False) + if not self.domain: + self.domain = self.sdb.create_domain(self.db_name) + + def _object_lister(self, cls, query_lister): + for item in query_lister: + obj = self.get_object(cls, item.name, item) + if obj: + yield obj + + def encode_value(self, prop, value): + if value == None: + return None + if not prop: + return str(value) + return self.converter.encode_prop(prop, value) + + def decode_value(self, prop, value): + return self.converter.decode_prop(prop, value) + + def get_s3_connection(self): + if not self.s3: + self.s3 = boto.connect_s3(self.db_user, self.db_passwd) + return self.s3 + + def get_blob_bucket(self, bucket_name=None): + s3 = self.get_s3_connection() + bucket_name = "%s-%s" % (s3.aws_access_key_id, self.domain.name) + bucket_name = bucket_name.lower() + try: + self.bucket = s3.get_bucket(bucket_name) + except: + self.bucket = s3.create_bucket(bucket_name) + return self.bucket + + def load_object(self, obj): + if not obj._loaded: + a = self.domain.get_attributes(obj.id) + if a.has_key('__type__'): + for prop in obj.properties(hidden=False): + if a.has_key(prop.name): + value = self.decode_value(prop, a[prop.name]) + value = prop.make_value_from_datastore(value) + setattr(obj, prop.name, value) + obj._loaded = True + + def get_object(self, cls, id, a=None): + obj = None + if not a: + a = self.domain.get_attributes(id) + if a.has_key('__type__'): + if not cls or a['__type__'] != cls.__name__: + cls = find_class(a['__module__'], a['__type__']) + if cls: + params = {} + for prop in cls.properties(hidden=False): + if a.has_key(prop.name): + value = self.decode_value(prop, a[prop.name]) + value = prop.make_value_from_datastore(value) + params[prop.name] = value + obj = cls(id, **params) + obj._loaded = True + else: + s = '(%s) class %s.%s not found' % (id, a['__module__'], a['__type__']) + boto.log.info('sdbmanager: %s' % s) + return obj + + def get_object_from_id(self, id): + return self.get_object(None, id) + + def query(self, query): + query_str = "select * from `%s` %s" % (self.domain.name, self._build_filter_part(query.model_class, query.filters, query.sort_by)) + if query.limit: + query_str += " limit %s" % query.limit + rs = self.domain.select(query_str, max_items=query.limit, next_token = query.next_token) + query.rs = rs + return self._object_lister(query.model_class, rs) + + def count(self, cls, filters): + """ + Get the number of results that would + be returned in this query + """ + query = "select count(*) from `%s` %s" % (self.domain.name, self._build_filter_part(cls, filters)) + count = int(self.domain.select(query).next()["Count"]) + return count + + def _build_filter_part(self, cls, filters, order_by=None): + """ + Build the filter part + """ + import types + query_parts = [] + order_by_filtered = False + if order_by: + if order_by[0] == "-": + order_by_method = "desc"; + order_by = order_by[1:] + else: + order_by_method = "asc"; + + for filter in filters: + (name, op) = filter[0].strip().split(" ", 1) + value = filter[1] + property = cls.find_property(name) + if name == order_by: + order_by_filtered = True + if types.TypeType(value) == types.ListType: + filter_parts = [] + for val in value: + val = self.encode_value(property, val) + if isinstance(val, list): + for v in val: + filter_parts.append("`%s` %s '%s'" % (name, op, v.replace("'", "''"))) + else: + filter_parts.append("`%s` %s '%s'" % (name, op, val.replace("'", "''"))) + query_parts.append("(%s)" % (" or ".join(filter_parts))) + else: + if op == 'is' and value == None: + query_parts.append("`%s` is null" % name) + elif op == 'is not' and value == None: + query_parts.append("`%s` is not null" % name) + else: + val = self.encode_value(property, value) + if isinstance(val, list): + for v in val: + query_parts.append("`%s` %s '%s'" % (name, op, v.replace("'", "''"))) + else: + query_parts.append("`%s` %s '%s'" % (name, op, val.replace("'", "''"))) + + type_query = "(`__type__` = '%s'" % cls.__name__ + for subclass in cls.__sub_classes__: + type_query += " or `__type__` = '%s'" % subclass.__name__ + type_query +=")" + query_parts.append(type_query) + + order_by_query = "" + if order_by: + if not order_by_filtered: + query_parts.append("`%s` like '%%'" % order_by) + order_by_query = " order by `%s` %s" % (order_by, order_by_method) + + if len(query_parts) > 0: + return "where %s %s" % (" and ".join(query_parts), order_by_query) + else: + return "" + + + def query_gql(self, query_string, *args, **kwds): + raise NotImplementedError, "GQL queries not supported in SimpleDB" + + def save_object(self, obj): + if not obj.id: + obj.id = str(uuid.uuid4()) + + attrs = {'__type__' : obj.__class__.__name__, + '__module__' : obj.__class__.__module__, + '__lineage__' : obj.get_lineage()} + for property in obj.properties(hidden=False): + value = property.get_value_for_datastore(obj) + if value is not None: + value = self.encode_value(property, value) + attrs[property.name] = value + if property.unique: + try: + args = {property.name: value} + obj2 = obj.find(**args).next() + if obj2.id != obj.id: + raise SDBPersistenceError("Error: %s must be unique!" % property.name) + except(StopIteration): + pass + self.domain.put_attributes(obj.id, attrs, replace=True) + + def delete_object(self, obj): + self.domain.delete_attributes(obj.id) + + def set_property(self, prop, obj, name, value): + value = prop.get_value_for_datastore(obj) + value = self.encode_value(prop, value) + if prop.unique: + try: + args = {prop.name: value} + obj2 = obj.find(**args).next() + if obj2.id != obj.id: + raise SDBPersistenceError("Error: %s must be unique!" % prop.name) + except(StopIteration): + pass + self.domain.put_attributes(obj.id, {name : value}, replace=True) + + def get_property(self, prop, obj, name): + a = self.domain.get_attributes(obj.id) + + # try to get the attribute value from SDB + if name in a: + value = self.decode_value(prop, a[name]) + value = prop.make_value_from_datastore(value) + setattr(obj, prop.name, value) + return value + raise AttributeError, '%s not found' % name + + def set_key_value(self, obj, name, value): + self.domain.put_attributes(obj.id, {name : value}, replace=True) + + def delete_key_value(self, obj, name): + self.domain.delete_attributes(obj.id, name) + + def get_key_value(self, obj, name): + a = self.domain.get_attributes(obj.id, name) + if a.has_key(name): + return a[name] + else: + return None + + def get_raw_item(self, obj): + return self.domain.get_item(obj.id) + diff --git a/storefront/boto/sdb/db/manager/xmlmanager.py b/storefront/boto/sdb/db/manager/xmlmanager.py new file mode 100644 index 0000000..b12f5df --- /dev/null +++ b/storefront/boto/sdb/db/manager/xmlmanager.py @@ -0,0 +1,519 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import boto +from boto.utils import find_class, Password +import uuid +from boto.sdb.db.key import Key +from boto.sdb.db.model import Model +from datetime import datetime +from boto.exception import SDBPersistenceError +from xml.dom.minidom import getDOMImplementation, parse, parseString, Node + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +class XMLConverter: + """ + Responsible for converting base Python types to format compatible with underlying + database. For SimpleDB, that means everything needs to be converted to a string + when stored in SimpleDB and from a string when retrieved. + + To convert a value, pass it to the encode or decode method. The encode method + will take a Python native value and convert to DB format. The decode method will + take a DB format value and convert it to Python native format. To find the appropriate + method to call, the generic encode/decode methods will look for the type-specific + method by searching for a method called "encode_" or "decode_". + """ + def __init__(self, manager): + self.manager = manager + self.type_map = { bool : (self.encode_bool, self.decode_bool), + int : (self.encode_int, self.decode_int), + long : (self.encode_long, self.decode_long), + Model : (self.encode_reference, self.decode_reference), + Key : (self.encode_reference, self.decode_reference), + Password : (self.encode_password, self.decode_password), + datetime : (self.encode_datetime, self.decode_datetime)} + + def get_text_value(self, parent_node): + value = '' + for node in parent_node.childNodes: + if node.nodeType == node.TEXT_NODE: + value += node.data + return value + + def encode(self, item_type, value): + if item_type in self.type_map: + encode = self.type_map[item_type][0] + return encode(value) + return value + + def decode(self, item_type, value): + if item_type in self.type_map: + decode = self.type_map[item_type][1] + return decode(value) + else: + value = self.get_text_value(value) + return value + + def encode_prop(self, prop, value): + if isinstance(value, list): + if hasattr(prop, 'item_type'): + new_value = [] + for v in value: + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + item_type = Model + new_value.append(self.encode(item_type, v)) + return new_value + else: + return value + else: + return self.encode(prop.data_type, value) + + def decode_prop(self, prop, value): + if prop.data_type == list: + if hasattr(prop, 'item_type'): + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + item_type = Model + values = [] + for item_node in value.getElementsByTagName('item'): + value = self.decode(item_type, item_node) + values.append(value) + return values + else: + return self.get_text_value(value) + else: + return self.decode(prop.data_type, value) + + def encode_int(self, value): + value = int(value) + return '%d' % value + + def decode_int(self, value): + value = self.get_text_value(value) + if value: + value = int(value) + else: + value = None + return value + + def encode_long(self, value): + value = long(value) + return '%d' % value + + def decode_long(self, value): + value = self.get_text_value(value) + return long(value) + + def encode_bool(self, value): + if value == True: + return 'true' + else: + return 'false' + + def decode_bool(self, value): + value = self.get_text_value(value) + if value.lower() == 'true': + return True + else: + return False + + def encode_datetime(self, value): + return value.strftime(ISO8601) + + def decode_datetime(self, value): + value = self.get_text_value(value) + try: + return datetime.strptime(value, ISO8601) + except: + return None + + def encode_reference(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + val_node = self.manager.doc.createElement("object") + val_node.setAttribute('id', value.id) + val_node.setAttribute('class', '%s.%s' % (value.__class__.__module__, value.__class__.__name__)) + return val_node + + def decode_reference(self, value): + if not value: + return None + try: + value = value.childNodes[0] + class_name = value.getAttribute("class") + id = value.getAttribute("id") + cls = find_class(class_name) + return cls.get_by_ids(id) + except: + return None + + def encode_password(self, value): + if value and len(value) > 0: + return str(value) + else: + return None + + def decode_password(self, value): + value = self.get_text_value(value) + return Password(value) + + +class XMLManager(object): + + def __init__(self, cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, ddl_dir, enable_ssl): + self.cls = cls + if not db_name: + db_name = cls.__name__.lower() + self.db_name = db_name + self.db_user = db_user + self.db_passwd = db_passwd + self.db_host = db_host + self.db_port = db_port + self.db_table = db_table + self.ddl_dir = ddl_dir + self.s3 = None + self.converter = XMLConverter(self) + self.impl = getDOMImplementation() + self.doc = self.impl.createDocument(None, 'objects', None) + + self.connection = None + self.enable_ssl = enable_ssl + self.auth_header = None + if self.db_user: + import base64 + base64string = base64.encodestring('%s:%s' % (self.db_user, self.db_passwd))[:-1] + authheader = "Basic %s" % base64string + self.auth_header = authheader + + def _connect(self): + if self.db_host: + if self.enable_ssl: + from httplib import HTTPSConnection as Connection + else: + from httplib import HTTPConnection as Connection + + self.connection = Connection(self.db_host, self.db_port) + + def _make_request(self, method, url, post_data=None, body=None): + """ + Make a request on this connection + """ + if not self.connection: + self._connect() + try: + self.connection.close() + except: + pass + self.connection.connect() + headers = {} + if self.auth_header: + headers["Authorization"] = self.auth_header + self.connection.request(method, url, body, headers) + resp = self.connection.getresponse() + return resp + + def new_doc(self): + return self.impl.createDocument(None, 'objects', None) + + def _object_lister(self, cls, doc): + for obj_node in doc.getElementsByTagName('object'): + if not cls: + class_name = obj_node.getAttribute('class') + cls = find_class(class_name) + id = obj_node.getAttribute('id') + obj = cls(id) + for prop_node in obj_node.getElementsByTagName('property'): + prop_name = prop_node.getAttribute('name') + prop = obj.find_property(prop_name) + if prop: + if hasattr(prop, 'item_type'): + value = self.get_list(prop_node, prop.item_type) + else: + value = self.decode_value(prop, prop_node) + value = prop.make_value_from_datastore(value) + setattr(obj, prop.name, value) + yield obj + + def reset(self): + self._connect() + + def get_doc(self): + return self.doc + + def encode_value(self, prop, value): + return self.converter.encode_prop(prop, value) + + def decode_value(self, prop, value): + return self.converter.decode_prop(prop, value) + + def get_s3_connection(self): + if not self.s3: + self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key) + return self.s3 + + def get_list(self, prop_node, item_type): + values = [] + try: + items_node = prop_node.getElementsByTagName('items')[0] + except: + return [] + for item_node in items_node.getElementsByTagName('item'): + value = self.converter.decode(item_type, item_node) + values.append(value) + return values + + def get_object_from_doc(self, cls, id, doc): + obj_node = doc.getElementsByTagName('object')[0] + if not cls: + class_name = obj_node.getAttribute('class') + cls = find_class(class_name) + if not id: + id = obj_node.getAttribute('id') + obj = cls(id) + for prop_node in obj_node.getElementsByTagName('property'): + prop_name = prop_node.getAttribute('name') + prop = obj.find_property(prop_name) + value = self.decode_value(prop, prop_node) + value = prop.make_value_from_datastore(value) + if value != None: + try: + setattr(obj, prop.name, value) + except: + pass + return obj + + def get_props_from_doc(self, cls, id, doc): + """ + Pull out the properties from this document + Returns the class, the properties in a hash, and the id if provided as a tuple + :return: (cls, props, id) + """ + obj_node = doc.getElementsByTagName('object')[0] + if not cls: + class_name = obj_node.getAttribute('class') + cls = find_class(class_name) + if not id: + id = obj_node.getAttribute('id') + props = {} + for prop_node in obj_node.getElementsByTagName('property'): + prop_name = prop_node.getAttribute('name') + prop = cls.find_property(prop_name) + value = self.decode_value(prop, prop_node) + value = prop.make_value_from_datastore(value) + if value != None: + props[prop.name] = value + return (cls, props, id) + + + def get_object(self, cls, id): + if not self.connection: + self._connect() + + if not self.connection: + raise NotImplementedError("Can't query without a database connection") + url = "/%s/%s" % (self.db_name, id) + resp = self._make_request('GET', url) + if resp.status == 200: + doc = parse(resp) + else: + raise Exception("Error: %s" % resp.status) + return self.get_object_from_doc(cls, id, doc) + + def query(self, cls, filters, limit=None, order_by=None): + if not self.connection: + self._connect() + + if not self.connection: + raise NotImplementedError("Can't query without a database connection") + + from urllib import urlencode + + query = str(self._build_query(cls, filters, limit, order_by)) + if query: + url = "/%s?%s" % (self.db_name, urlencode({"query": query})) + else: + url = "/%s" % self.db_name + resp = self._make_request('GET', url) + if resp.status == 200: + doc = parse(resp) + else: + raise Exception("Error: %s" % resp.status) + return self._object_lister(cls, doc) + + def _build_query(self, cls, filters, limit, order_by): + import types + if len(filters) > 4: + raise Exception('Too many filters, max is 4') + parts = [] + properties = cls.properties(hidden=False) + for filter, value in filters: + name, op = filter.strip().split() + found = False + for property in properties: + if property.name == name: + found = True + if types.TypeType(value) == types.ListType: + filter_parts = [] + for val in value: + val = self.encode_value(property, val) + filter_parts.append("'%s' %s '%s'" % (name, op, val)) + parts.append("[%s]" % " OR ".join(filter_parts)) + else: + value = self.encode_value(property, value) + parts.append("['%s' %s '%s']" % (name, op, value)) + if not found: + raise Exception('%s is not a valid field' % name) + if order_by: + if order_by.startswith("-"): + key = order_by[1:] + type = "desc" + else: + key = order_by + type = "asc" + parts.append("['%s' starts-with ''] sort '%s' %s" % (key, key, type)) + return ' intersection '.join(parts) + + def query_gql(self, query_string, *args, **kwds): + raise NotImplementedError, "GQL queries not supported in XML" + + def save_list(self, doc, items, prop_node): + items_node = doc.createElement('items') + prop_node.appendChild(items_node) + for item in items: + item_node = doc.createElement('item') + items_node.appendChild(item_node) + if isinstance(item, Node): + item_node.appendChild(item) + else: + text_node = doc.createTextNode(item) + item_node.appendChild(text_node) + + def save_object(self, obj): + """ + Marshal the object and do a PUT + """ + doc = self.marshal_object(obj) + if obj.id: + url = "/%s/%s" % (self.db_name, obj.id) + else: + url = "/%s" % (self.db_name) + resp = self._make_request("PUT", url, body=doc.toxml()) + new_obj = self.get_object_from_doc(obj.__class__, None, parse(resp)) + obj.id = new_obj.id + for prop in obj.properties(): + try: + propname = prop.name + except AttributeError: + propname = None + if propname: + value = getattr(new_obj, prop.name) + if value: + setattr(obj, prop.name, value) + return obj + + + def marshal_object(self, obj, doc=None): + if not doc: + doc = self.new_doc() + if not doc: + doc = self.doc + obj_node = doc.createElement('object') + + if obj.id: + obj_node.setAttribute('id', obj.id) + + obj_node.setAttribute('class', '%s.%s' % (obj.__class__.__module__, + obj.__class__.__name__)) + root = doc.documentElement + root.appendChild(obj_node) + for property in obj.properties(hidden=False): + prop_node = doc.createElement('property') + prop_node.setAttribute('name', property.name) + prop_node.setAttribute('type', property.type_name) + value = property.get_value_for_datastore(obj) + if value is not None: + value = self.encode_value(property, value) + if isinstance(value, list): + self.save_list(doc, value, prop_node) + elif isinstance(value, Node): + prop_node.appendChild(value) + else: + text_node = doc.createTextNode(str(value)) + prop_node.appendChild(text_node) + obj_node.appendChild(prop_node) + + return doc + + def unmarshal_object(self, fp, cls=None, id=None): + if isinstance(fp, str) or isinstance(fp, unicode): + doc = parseString(fp) + else: + doc = parse(fp) + return self.get_object_from_doc(cls, id, doc) + + def unmarshal_props(self, fp, cls=None, id=None): + """ + Same as unmarshalling an object, except it returns + from "get_props_from_doc" + """ + if isinstance(fp, str) or isinstance(fp, unicode): + doc = parseString(fp) + else: + doc = parse(fp) + return self.get_props_from_doc(cls, id, doc) + + def delete_object(self, obj): + url = "/%s/%s" % (self.db_name, obj.id) + return self._make_request("DELETE", url) + + def set_key_value(self, obj, name, value): + self.domain.put_attributes(obj.id, {name : value}, replace=True) + + def delete_key_value(self, obj, name): + self.domain.delete_attributes(obj.id, name) + + def get_key_value(self, obj, name): + a = self.domain.get_attributes(obj.id, name) + if a.has_key(name): + return a[name] + else: + return None + + def get_raw_item(self, obj): + return self.domain.get_item(obj.id) + + def set_property(self, prop, obj, name, value): + pass + + def get_property(self, prop, obj, name): + pass + + def load_object(self, obj): + if not obj._loaded: + obj = obj.get_by_id(obj.id) + obj._loaded = True + return obj + diff --git a/storefront/boto/sdb/db/model.py b/storefront/boto/sdb/db/model.py new file mode 100644 index 0000000..dc142e8 --- /dev/null +++ b/storefront/boto/sdb/db/model.py @@ -0,0 +1,232 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.sdb.db.manager import get_manager +from boto.sdb.db.property import * +from boto.sdb.db.key import Key +from boto.sdb.db.query import Query +import boto + +class ModelMeta(type): + "Metaclass for all Models" + + def __init__(cls, name, bases, dict): + super(ModelMeta, cls).__init__(name, bases, dict) + # Make sure this is a subclass of Model - mainly copied from django ModelBase (thanks!) + cls.__sub_classes__ = [] + try: + if filter(lambda b: issubclass(b, Model), bases): + for base in bases: + base.__sub_classes__.append(cls) + cls._manager = get_manager(cls) + # look for all of the Properties and set their names + for key in dict.keys(): + if isinstance(dict[key], Property): + property = dict[key] + property.__property_config__(cls, key) + prop_names = [] + props = cls.properties() + for prop in props: + if not prop.__class__.__name__.startswith('_'): + prop_names.append(prop.name) + setattr(cls, '_prop_names', prop_names) + except NameError: + # 'Model' isn't defined yet, meaning we're looking at our own + # Model class, defined below. + pass + +class Model(object): + __metaclass__ = ModelMeta + + @classmethod + def get_lineage(cls): + l = [c.__name__ for c in cls.mro()] + l.reverse() + return '.'.join(l) + + @classmethod + def kind(cls): + return cls.__name__ + + @classmethod + def _get_by_id(cls, id, manager=None): + if not manager: + manager = cls._manager + return manager.get_object(cls, id) + + @classmethod + def get_by_id(cls, ids=None, parent=None): + if isinstance(ids, list): + objs = [cls._get_by_id(id) for id in ids] + return objs + else: + return cls._get_by_id(ids) + + get_by_ids = get_by_id + + @classmethod + def get_by_key_name(cls, key_names, parent=None): + raise NotImplementedError, "Key Names are not currently supported" + + @classmethod + def find(cls, limit=None, next_token=None, **params): + q = Query(cls, limit=limit, next_token=next_token) + for key, value in params.items(): + q.filter('%s =' % key, value) + return q + + @classmethod + def lookup(cls, name, value): + return cls._manager.lookup(cls, name, value) + + @classmethod + def all(cls, limit=None, next_token=None): + return cls.find(limit=limit, next_token=next_token) + + @classmethod + def get_or_insert(key_name, **kw): + raise NotImplementedError, "get_or_insert not currently supported" + + @classmethod + def properties(cls, hidden=True): + properties = [] + while cls: + for key in cls.__dict__.keys(): + prop = cls.__dict__[key] + if isinstance(prop, Property): + if hidden or not prop.__class__.__name__.startswith('_'): + properties.append(prop) + if len(cls.__bases__) > 0: + cls = cls.__bases__[0] + else: + cls = None + return properties + + @classmethod + def find_property(cls, prop_name): + property = None + while cls: + for key in cls.__dict__.keys(): + prop = cls.__dict__[key] + if isinstance(prop, Property): + if not prop.__class__.__name__.startswith('_') and prop_name == prop.name: + property = prop + if len(cls.__bases__) > 0: + cls = cls.__bases__[0] + else: + cls = None + return property + + @classmethod + def get_xmlmanager(cls): + if not hasattr(cls, '_xmlmanager'): + from boto.sdb.db.manager.xmlmanager import XMLManager + cls._xmlmanager = XMLManager(cls, None, None, None, + None, None, None, None, False) + return cls._xmlmanager + + @classmethod + def from_xml(cls, fp): + xmlmanager = cls.get_xmlmanager() + return xmlmanager.unmarshal_object(fp) + + def __init__(self, id=None, **kw): + self._loaded = False + # first initialize all properties to their default values + for prop in self.properties(hidden=False): + setattr(self, prop.name, prop.default_value()) + if kw.has_key('manager'): + self._manager = kw['manager'] + self.id = id + for key in kw: + if key != 'manager': + # We don't want any errors populating up when loading an object, + # so if it fails we just revert to it's default value + try: + setattr(self, key, kw[key]) + except Exception, e: + boto.log.exception(e) + + def __repr__(self): + return '%s<%s>' % (self.__class__.__name__, self.id) + + def __str__(self): + return str(self.id) + + def __eq__(self, other): + return other and isinstance(other, Model) and self.id == other.id + + def _get_raw_item(self): + return self._manager.get_raw_item(self) + + def load(self): + if self.id and not self._loaded: + self._manager.load_object(self) + + def put(self): + self._manager.save_object(self) + + save = put + + def delete(self): + self._manager.delete_object(self) + + def key(self): + return Key(obj=self) + + def set_manager(self, manager): + self._manager = manager + + def to_dict(self): + props = {} + for prop in self.properties(hidden=False): + props[prop.name] = getattr(self, prop.name) + obj = {'properties' : props, + 'id' : self.id} + return {self.__class__.__name__ : obj} + + def to_xml(self, doc=None): + xmlmanager = self.get_xmlmanager() + doc = xmlmanager.marshal_object(self, doc) + return doc + +class Expando(Model): + + def __setattr__(self, name, value): + if name in self._prop_names: + object.__setattr__(self, name, value) + elif name.startswith('_'): + object.__setattr__(self, name, value) + elif name == 'id': + object.__setattr__(self, name, value) + else: + self._manager.set_key_value(self, name, value) + object.__setattr__(self, name, value) + + def __getattr__(self, name): + if not name.startswith('_'): + value = self._manager.get_key_value(self, name) + if value: + object.__setattr__(self, name, value) + return value + raise AttributeError + + diff --git a/storefront/boto/sdb/db/property.py b/storefront/boto/sdb/db/property.py new file mode 100644 index 0000000..61d424a --- /dev/null +++ b/storefront/boto/sdb/db/property.py @@ -0,0 +1,556 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import datetime +from key import Key +from boto.utils import Password +from boto.sdb.db.query import Query +from tempfile import TemporaryFile + +import re +import boto +import boto.s3.key +from boto.sdb.db.blob import Blob + +class Property(object): + + data_type = str + type_name = '' + name = '' + verbose_name = '' + + def __init__(self, verbose_name=None, name=None, default=None, required=False, + validator=None, choices=None, unique=False): + self.verbose_name = verbose_name + self.name = name + self.default = default + self.required = required + self.validator = validator + self.choices = choices + self.slot_name = '_' + self.unique = unique + + def __get__(self, obj, objtype): + if obj: + obj.load() + return getattr(obj, self.slot_name) + else: + return None + + def __set__(self, obj, value): + self.validate(value) + + # Fire off any on_set functions + try: + if obj._loaded and hasattr(obj, "on_set_%s" % self.name): + fnc = getattr(obj, "on_set_%s" % self.name) + value = fnc(value) + except Exception, e: + boto.log.exception("Exception running on_set_%s" % self.name) + + setattr(obj, self.slot_name, value) + + def __property_config__(self, model_class, property_name): + self.model_class = model_class + self.name = property_name + self.slot_name = '_' + self.name + + def default_validator(self, value): + if value == self.default_value(): + return + if not isinstance(value, self.data_type): + raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + + def default_value(self): + return self.default + + def validate(self, value): + if self.required and value==None: + raise ValueError, '%s is a required property' % self.name + if self.choices and value and not value in self.choices: + raise ValueError, '%s not a valid choice for %s.%s' % (value, self.model_class.__name__, self.name) + if self.validator: + self.validator(value) + else: + self.default_validator(value) + return value + + def empty(self, value): + return not value + + def get_value_for_datastore(self, model_instance): + return getattr(model_instance, self.name) + + def make_value_from_datastore(self, value): + return value + + def get_choices(self): + if callable(self.choices): + return self.choices() + return self.choices + +def validate_string(value): + if isinstance(value, str) or isinstance(value, unicode): + if len(value) > 1024: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting String, got %s' % type(value) + +class StringProperty(Property): + + type_name = 'String' + + def __init__(self, verbose_name=None, name=None, default='', required=False, + validator=validate_string, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + +class TextProperty(Property): + + type_name = 'Text' + + def __init__(self, verbose_name=None, name=None, default='', required=False, + validator=None, choices=None, unique=False, max_length=None): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.max_length = max_length + + def validate(self, value): + if not isinstance(value, str) and not isinstance(value, unicode): + raise TypeError, 'Expecting Text, got %s' % type(value) + if self.max_length and len(value) > self.max_length: + raise ValueError, 'Length of value greater than maxlength %s' % self.max_length + +class PasswordProperty(StringProperty): + """ + Hashed property who's original value can not be + retrieved, but still can be compaired. + """ + data_type = Password + type_name = 'Password' + + def __init__(self, verbose_name=None, name=None, default='', required=False, + validator=None, choices=None, unique=False): + StringProperty.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def make_value_from_datastore(self, value): + p = Password(value) + return p + + def get_value_for_datastore(self, model_instance): + value = StringProperty.get_value_for_datastore(self, model_instance) + if value and len(value): + return str(value) + else: + return None + + def __set__(self, obj, value): + if not isinstance(value, Password): + p = Password() + p.set(value) + value = p + Property.__set__(self, obj, value) + + def __get__(self, obj, objtype): + return Password(StringProperty.__get__(self, obj, objtype)) + + def validate(self, value): + value = Property.validate(self, value) + if isinstance(value, Password): + if len(value) > 1024: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting Password, got %s' % type(value) + +class BlobProperty(Property): + data_type = Blob + type_name = "blob" + + def __set__(self, obj, value): + if value != self.default_value(): + if not isinstance(value, Blob): + oldb = self.__get__(obj, type(obj)) + id = None + if oldb: + id = oldb.id + b = Blob(value=value, id=id) + value = b + Property.__set__(self, obj, value) + +class S3KeyProperty(Property): + + data_type = boto.s3.key.Key + type_name = 'S3Key' + validate_regex = "^s3:\/\/([^\/]*)\/(.*)$" + + def __init__(self, verbose_name=None, name=None, default=None, + required=False, validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, + validator, choices, unique) + + def validate(self, value): + if value == self.default_value() or value == str(self.default_value()): + return self.default_value() + if isinstance(value, self.data_type): + return + match = re.match(self.validate_regex, value) + if match: + return + raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + + def __get__(self, obj, objtype): + value = Property.__get__(self, obj, objtype) + if value: + if isinstance(value, self.data_type): + return value + match = re.match(self.validate_regex, value) + if match: + s3 = obj._manager.get_s3_connection() + bucket = s3.get_bucket(match.group(1), validate=False) + k = bucket.get_key(match.group(2)) + if not k: + k = bucket.new_key(match.group(2)) + k.set_contents_from_string("") + return k + else: + return value + + def get_value_for_datastore(self, model_instance): + value = Property.get_value_for_datastore(self, model_instance) + if value: + return "s3://%s/%s" % (value.bucket.name, value.name) + else: + return None + +class IntegerProperty(Property): + + data_type = int + type_name = 'Integer' + + def __init__(self, verbose_name=None, name=None, default=0, required=False, + validator=None, choices=None, unique=False, max=2147483647, min=-2147483648): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.max = max + self.min = min + + def validate(self, value): + value = int(value) + value = Property.validate(self, value) + if value > self.max: + raise ValueError, 'Maximum value is %d' % self.max + if value < self.min: + raise ValueError, 'Minimum value is %d' % self.min + return value + + def empty(self, value): + return value is None + +class LongProperty(Property): + + data_type = long + type_name = 'Long' + + def __init__(self, verbose_name=None, name=None, default=0, required=False, + validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def validate(self, value): + value = long(value) + value = Property.validate(self, value) + min = -9223372036854775808 + max = 9223372036854775807 + if value > max: + raise ValueError, 'Maximum value is %d' % max + if value < min: + raise ValueError, 'Minimum value is %d' % min + return value + + def empty(self, value): + return value is None + +class BooleanProperty(Property): + + data_type = bool + type_name = 'Boolean' + + def __init__(self, verbose_name=None, name=None, default=False, required=False, + validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def empty(self, value): + return value is None + +class FloatProperty(Property): + + data_type = float + type_name = 'Float' + + def __init__(self, verbose_name=None, name=None, default=0.0, required=False, + validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def validate(self, value): + value = float(value) + value = Property.validate(self, value) + return value + + def empty(self, value): + return value is None + +class DateTimeProperty(Property): + + data_type = datetime.datetime + type_name = 'DateTime' + + def __init__(self, verbose_name=None, auto_now=False, auto_now_add=False, name=None, + default=None, required=False, validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.auto_now = auto_now + self.auto_now_add = auto_now_add + + def default_value(self): + if self.auto_now or self.auto_now_add: + return self.now() + return Property.default_value(self) + + def validate(self, value): + if value == None: + return + if not isinstance(value, self.data_type): + raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + + def get_value_for_datastore(self, model_instance): + if self.auto_now: + setattr(model_instance, self.name, self.now()) + return Property.get_value_for_datastore(self, model_instance) + + def now(self): + return datetime.datetime.utcnow() + +class ReferenceProperty(Property): + + data_type = Key + type_name = 'Reference' + + def __init__(self, reference_class=None, collection_name=None, + verbose_name=None, name=None, default=None, required=False, validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.reference_class = reference_class + self.collection_name = collection_name + + def __get__(self, obj, objtype): + if obj: + value = getattr(obj, self.slot_name) + if value == self.default_value(): + return value + # If the value is still the UUID for the referenced object, we need to create + # the object now that is the attribute has actually been accessed. This lazy + # instantiation saves unnecessary roundtrips to SimpleDB + if isinstance(value, str) or isinstance(value, unicode): + # This is some minor handling to allow us to use the base "Model" class + # as our reference class. If we do so, we're going to assume we're using + # our own class's manager to fetch objects + if hasattr(self.reference_class, "_manager"): + manager = self.reference_class._manager + else: + manager = obj._manager + value = manager.get_object(self.reference_class, value) + setattr(obj, self.name, value) + return value + + def __property_config__(self, model_class, property_name): + Property.__property_config__(self, model_class, property_name) + if self.collection_name is None: + self.collection_name = '%s_%s_set' % (model_class.__name__.lower(), self.name) + if hasattr(self.reference_class, self.collection_name): + raise ValueError, 'duplicate property: %s' % self.collection_name + setattr(self.reference_class, self.collection_name, + _ReverseReferenceProperty(model_class, property_name, self.collection_name)) + + def check_uuid(self, value): + # This does a bit of hand waving to "type check" the string + t = value.split('-') + if len(t) != 5: + raise ValueError + + def check_instance(self, value): + try: + obj_lineage = value.get_lineage() + cls_lineage = self.reference_class.get_lineage() + if obj_lineage.startswith(cls_lineage): + return + raise TypeError, '%s not instance of %s' % (obj_lineage, cls_lineage) + except: + raise ValueError, '%s is not a Model' % value + + def validate(self, value): + if self.required and value==None: + raise ValueError, '%s is a required property' % self.name + if value == self.default_value(): + return + if not isinstance(value, str) and not isinstance(value, unicode): + self.check_instance(value) + +class _ReverseReferenceProperty(Property): + data_type = Query + type_name = 'query' + + def __init__(self, model, prop, name): + self.__model = model + self.__property = prop + self.name = name + self.item_type = model + + def __get__(self, model_instance, model_class): + """Fetches collection of model instances of this collection property.""" + if model_instance is not None: + query = Query(self.__model) + return query.filter(self.__property + ' =', model_instance) + else: + return self + + def __set__(self, model_instance, value): + """Not possible to set a new collection.""" + raise ValueError, 'Virtual property is read-only' + + +class CalculatedProperty(Property): + + def __init__(self, verbose_name=None, name=None, default=None, + required=False, validator=None, choices=None, + calculated_type=int, unique=False, use_method=False): + Property.__init__(self, verbose_name, name, default, required, + validator, choices, unique) + self.calculated_type = calculated_type + self.use_method = use_method + + def __get__(self, obj, objtype): + value = self.default_value() + if obj: + try: + value = getattr(obj, self.slot_name) + if self.use_method: + value = value() + except AttributeError: + pass + return value + + def __set__(self, obj, value): + """Not possible to set a new AutoID.""" + pass + + def _set_direct(self, obj, value): + if not self.use_method: + setattr(obj, self.slot_name, value) + + def get_value_for_datastore(self, model_instance): + if self.calculated_type in [str, int, bool]: + value = self.__get__(model_instance, model_instance.__class__) + return value + else: + return None + +class ListProperty(Property): + + data_type = list + type_name = 'List' + + def __init__(self, item_type, verbose_name=None, name=None, default=None, **kwds): + if default is None: + default = [] + self.item_type = item_type + Property.__init__(self, verbose_name, name, default=default, required=True, **kwds) + + def validate(self, value): + if value is not None: + if not isinstance(value, list): + value = [value] + + if self.item_type in (int, long): + item_type = (int, long) + elif self.item_type in (str, unicode): + item_type = (str, unicode) + else: + item_type = self.item_type + + for item in value: + if not isinstance(item, item_type): + if item_type == (int, long): + raise ValueError, 'Items in the %s list must all be integers.' % self.name + else: + raise ValueError('Items in the %s list must all be %s instances' % + (self.name, self.item_type.__name__)) + return value + + def empty(self, value): + return value is None + + def default_value(self): + return list(super(ListProperty, self).default_value()) + + def __set__(self, obj, value): + """Override the set method to allow them to set the property to an instance of the item_type instead of requiring a list to be passed in""" + if self.item_type in (int, long): + item_type = (int, long) + elif self.item_type in (str, unicode): + item_type = (str, unicode) + else: + item_type = self.item_type + if isinstance(value, item_type): + value = [value] + return super(ListProperty, self).__set__(obj,value) + + +class MapProperty(Property): + + data_type = dict + type_name = 'Map' + + def __init__(self, item_type=str, verbose_name=None, name=None, default=None, **kwds): + if default is None: + default = {} + self.item_type = item_type + Property.__init__(self, verbose_name, name, default=default, required=True, **kwds) + + def validate(self, value): + if value is not None: + if not isinstance(value, dict): + raise ValueError, 'Value must of type dict' + + if self.item_type in (int, long): + item_type = (int, long) + elif self.item_type in (str, unicode): + item_type = (str, unicode) + else: + item_type = self.item_type + + for key in value: + if not isinstance(value[key], item_type): + if item_type == (int, long): + raise ValueError, 'Values in the %s Map must all be integers.' % self.name + else: + raise ValueError('Values in the %s Map must all be %s instances' % + (self.name, self.item_type.__name__)) + return value + + def empty(self, value): + return value is None + + def default_value(self): + return {} diff --git a/storefront/boto/sdb/db/query.py b/storefront/boto/sdb/db/query.py new file mode 100644 index 0000000..034d9d3 --- /dev/null +++ b/storefront/boto/sdb/db/query.py @@ -0,0 +1,76 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Query(object): + __local_iter__ = None + def __init__(self, model_class, limit=None, next_token=None, manager=None): + self.model_class = model_class + self.limit = limit + if manager: + self.manager = manager + else: + self.manager = self.model_class._manager + self.filters = [] + self.sort_by = None + self.rs = None + self.next_token = next_token + + def __iter__(self): + return iter(self.manager.query(self)) + + def next(self): + if self.__local_iter__ == None: + self.__local_iter__ = self.__iter__() + return self.__local_iter__.next() + + def filter(self, property_operator, value): + self.filters.append((property_operator, value)) + return self + + def fetch(self, limit, offset=0): + raise NotImplementedError, "fetch mode is not currently supported" + + def count(self): + return self.manager.count(self.model_class, self.filters) + + def order(self, key): + self.sort_by = key + return self + + def to_xml(self, doc=None): + if not doc: + xmlmanager = self.model_class.get_xmlmanager() + doc = xmlmanager.new_doc() + for obj in self: + obj.to_xml(doc) + return doc + + def get_next_token(self): + if self.rs: + return self.rs.next_token + if self._next_token: + return self._next_token + return None + + def set_next_token(self, token): + self._next_token = token + + next_token = property(get_next_token, set_next_token) diff --git a/storefront/boto/sdb/db/test_db.py b/storefront/boto/sdb/db/test_db.py new file mode 100644 index 0000000..b790b9e --- /dev/null +++ b/storefront/boto/sdb/db/test_db.py @@ -0,0 +1,224 @@ +from boto.sdb.db.model import Model +from boto.sdb.db.property import * +from boto.sdb.db.manager import get_manager +from datetime import datetime +import time +from boto.exception import SDBPersistenceError + +_objects = {} + +# +# This will eventually be moved to the boto.tests module and become a real unit test +# but for now it will live here. It shows examples of each of the Property types in +# use and tests the basic operations. +# +class TestBasic(Model): + + name = StringProperty() + size = IntegerProperty() + foo = BooleanProperty() + date = DateTimeProperty() + +class TestFloat(Model): + + name = StringProperty() + value = FloatProperty() + +class TestRequired(Model): + + req = StringProperty(required=True, default='foo') + +class TestReference(Model): + + ref = ReferenceProperty(reference_class=TestBasic, collection_name='refs') + +class TestSubClass(TestBasic): + + answer = IntegerProperty() + +class TestPassword(Model): + password = PasswordProperty() + +class TestList(Model): + + name = StringProperty() + nums = ListProperty(int) + +class TestMap(Model): + + name = StringProperty() + map = MapProperty() + +class TestListReference(Model): + + name = StringProperty() + basics = ListProperty(TestBasic) + +class TestAutoNow(Model): + + create_date = DateTimeProperty(auto_now_add=True) + modified_date = DateTimeProperty(auto_now=True) + +class TestUnique(Model): + name = StringProperty(unique=True) + +def test_basic(): + global _objects + t = TestBasic() + t.name = 'simple' + t.size = -42 + t.foo = True + t.date = datetime.now() + print 'saving object' + t.put() + _objects['test_basic_t'] = t + time.sleep(5) + print 'now try retrieving it' + tt = TestBasic.get_by_id(t.id) + _objects['test_basic_tt'] = tt + assert tt.id == t.id + l = TestBasic.get_by_id([t.id]) + assert len(l) == 1 + assert l[0].id == t.id + assert t.size == tt.size + assert t.foo == tt.foo + assert t.name == tt.name + #assert t.date == tt.date + return t + +def test_float(): + global _objects + t = TestFloat() + t.name = 'float object' + t.value = 98.6 + print 'saving object' + t.save() + _objects['test_float_t'] = t + time.sleep(5) + print 'now try retrieving it' + tt = TestFloat.get_by_id(t.id) + _objects['test_float_tt'] = tt + assert tt.id == t.id + assert tt.name == t.name + assert tt.value == t.value + return t + +def test_required(): + global _objects + t = TestRequired() + _objects['test_required_t'] = t + t.put() + return t + +def test_reference(t=None): + global _objects + if not t: + t = test_basic() + tt = TestReference() + tt.ref = t + tt.put() + time.sleep(10) + tt = TestReference.get_by_id(tt.id) + _objects['test_reference_tt'] = tt + assert tt.ref.id == t.id + for o in t.refs: + print o + +def test_subclass(): + global _objects + t = TestSubClass() + _objects['test_subclass_t'] = t + t.name = 'a subclass' + t.size = -489 + t.save() + +def test_password(): + global _objects + t = TestPassword() + _objects['test_password_t'] = t + t.password = "foo" + t.save() + time.sleep(5) + # Make sure it stored ok + tt = TestPassword.get_by_id(t.id) + _objects['test_password_tt'] = tt + #Testing password equality + assert tt.password == "foo" + #Testing password not stored as string + assert str(tt.password) != "foo" + +def test_list(): + global _objects + t = TestList() + _objects['test_list_t'] = t + t.name = 'a list of ints' + t.nums = [1,2,3,4,5] + t.put() + tt = TestList.get_by_id(t.id) + _objects['test_list_tt'] = tt + assert tt.name == t.name + for n in tt.nums: + assert isinstance(n, int) + +def test_list_reference(): + global _objects + t = TestBasic() + t.put() + _objects['test_list_ref_t'] = t + tt = TestListReference() + tt.name = "foo" + tt.basics = [t] + tt.put() + time.sleep(5) + _objects['test_list_ref_tt'] = tt + ttt = TestListReference.get_by_id(tt.id) + assert ttt.basics[0].id == t.id + +def test_unique(): + global _objects + t = TestUnique() + name = 'foo' + str(int(time.time())) + t.name = name + t.put() + _objects['test_unique_t'] = t + time.sleep(10) + tt = TestUnique() + _objects['test_unique_tt'] = tt + tt.name = name + try: + tt.put() + assert False + except(SDBPersistenceError): + pass + +def test_datetime(): + global _objects + t = TestAutoNow() + t.put() + _objects['test_datetime_t'] = t + time.sleep(5) + tt = TestAutoNow.get_by_id(t.id) + assert tt.create_date.timetuple() == t.create_date.timetuple() + +def test(): + print 'test_basic' + t1 = test_basic() + print 'test_required' + test_required() + print 'test_reference' + test_reference(t1) + print 'test_subclass' + test_subclass() + print 'test_password' + test_password() + print 'test_list' + test_list() + print 'test_list_reference' + test_list_reference() + print "test_datetime" + test_datetime() + print 'test_unique' + test_unique() + +if __name__ == "__main__": + test() diff --git a/storefront/boto/sdb/domain.py b/storefront/boto/sdb/domain.py new file mode 100644 index 0000000..3c0def6 --- /dev/null +++ b/storefront/boto/sdb/domain.py @@ -0,0 +1,330 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SDB Domain +""" +from boto.sdb.queryresultset import QueryResultSet, SelectResultSet + +class Domain: + + def __init__(self, connection=None, name=None): + self.connection = connection + self.name = name + self._metadata = None + + def __repr__(self): + return 'Domain:%s' % self.name + + def __iter__(self): + return iter(self.select("SELECT * FROM `%s`" % self.name)) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'DomainName': + self.name = value + else: + setattr(self, name, value) + + def get_metadata(self): + if not self._metadata: + self._metadata = self.connection.domain_metadata(self) + return self._metadata + + def put_attributes(self, item_name, attributes, replace=True): + """ + Store attributes for a given item. + + :type item_name: string + :param item_name: The name of the item whose attributes are being stored. + + :type attribute_names: dict or dict-like object + :param attribute_names: The name/value pairs to store as attributes + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + return self.connection.put_attributes(self, item_name, attributes, replace) + + def batch_put_attributes(self, items, replace=True): + """ + Store attributes for multiple items. + + :type items: dict or dict-like object + :param items: A dictionary-like object. The keys of the dictionary are + the item names and the values are themselves dictionaries + of attribute names/values, exactly the same as the + attribute_names parameter of the scalar put_attributes + call. + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + return self.connection.batch_put_attributes(self, items, replace) + + def get_attributes(self, item_name, attribute_name=None, item=None): + """ + Retrieve attributes for a given item. + + :type item_name: string + :param item_name: The name of the item whose attributes are being retrieved. + + :type attribute_names: string or list of strings + :param attribute_names: An attribute name or list of attribute names. This + parameter is optional. If not supplied, all attributes + will be retrieved for the item. + + :rtype: :class:`boto.sdb.item.Item` + :return: An Item mapping type containing the requested attribute name/values + """ + return self.connection.get_attributes(self, item_name, attribute_name, item) + + def delete_attributes(self, item_name, attributes=None): + """ + Delete attributes from a given item. + + :type item_name: string + :param item_name: The name of the item whose attributes are being deleted. + + :type attributes: dict, list or :class:`boto.sdb.item.Item` + :param attributes: Either a list containing attribute names which will cause + all values associated with that attribute name to be deleted or + a dict or Item containing the attribute names and keys and list + of values to delete as the value. If no value is supplied, + all attribute name/values for the item will be deleted. + + :rtype: bool + :return: True if successful + """ + return self.connection.delete_attributes(self, item_name, attributes) + + def query(self, query='', max_items=None, attr_names=None): + """ + Returns a list of items within domain that match the query. + + :type query: string + :param query: The SimpleDB query to be performed. + + :type max_items: int + :param max_items: The maximum number of items to return. If not + supplied, the default is None which returns all + items matching the query. + + :type attr_names: list + :param attr_names: Either None, meaning return all attributes + or a list of attribute names which means to return + only those attributes. + + :rtype: iter + :return: An iterator containing the results. This is actually a generator + function that will iterate across all search results, not just the + first page. + """ + return iter(QueryResultSet(self, query, max_items, attr_names)) + + def select(self, query='', next_token=None, max_items=None): + """ + Returns a set of Attributes for item names within domain_name that match the query. + The query must be expressed in using the SELECT style syntax rather than the + original SimpleDB query language. + + :type query: string + :param query: The SimpleDB query to be performed. + + :type max_items: int + :param max_items: The maximum number of items to return. + + :rtype: iter + :return: An iterator containing the results. This is actually a generator + function that will iterate across all search results, not just the + first page. + """ + return SelectResultSet(self, query, max_items=max_items, + next_token=next_token) + + def get_item(self, item_name): + item = self.get_attributes(item_name) + if item: + item.domain = self + return item + else: + return None + + def new_item(self, item_name): + return self.connection.item_cls(self, item_name) + + def delete_item(self, item): + self.delete_attributes(item.name) + + def to_xml(self, f=None): + """Get this domain as an XML DOM Document + :param f: Optional File to dump directly to + :type f: File or Stream + + :return: File object where the XML has been dumped to + :rtype: file + """ + if not f: + from tempfile import TemporaryFile + f = TemporaryFile() + print >>f, '' + print >>f, '' % self.name + for item in self: + print >>f, '\t' % item.name + for k in item: + print >>f, '\t\t' % k + values = item[k] + if not isinstance(values, list): + values = [values] + for value in values: + print >>f, '\t\t\t>f, ']]>' + print >>f, '\t\t' + print >>f, '\t' + print >>f, '' + f.flush() + f.seek(0) + return f + + + def from_xml(self, doc): + """Load this domain based on an XML document""" + import xml.sax + handler = DomainDumpParser(self) + xml.sax.parse(doc, handler) + return handler + + +class DomainMetaData: + + def __init__(self, domain=None): + self.domain = domain + self.item_count = None + self.item_names_size = None + self.attr_name_count = None + self.attr_names_size = None + self.attr_value_count = None + self.attr_values_size = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'ItemCount': + self.item_count = int(value) + elif name == 'ItemNamesSizeBytes': + self.item_names_size = int(value) + elif name == 'AttributeNameCount': + self.attr_name_count = int(value) + elif name == 'AttributeNamesSizeBytes': + self.attr_names_size = int(value) + elif name == 'AttributeValueCount': + self.attr_value_count = int(value) + elif name == 'AttributeValuesSizeBytes': + self.attr_values_size = int(value) + elif name == 'Timestamp': + self.timestamp = value + else: + setattr(self, name, value) + +import sys +from xml.sax.handler import ContentHandler +class DomainDumpParser(ContentHandler): + """ + SAX parser for a domain that has been dumped + """ + + def __init__(self, domain): + self.uploader = UploaderThread(domain.name) + self.item_id = None + self.attrs = {} + self.attribute = None + self.value = "" + self.domain = domain + + def startElement(self, name, attrs): + if name == "Item": + self.item_id = attrs['id'] + self.attrs = {} + elif name == "attribute": + self.attribute = attrs['id'] + elif name == "value": + self.value = "" + + def characters(self, ch): + self.value += ch + + def endElement(self, name): + if name == "value": + if self.value and self.attribute: + value = self.value.strip() + attr_name = self.attribute.strip() + if self.attrs.has_key(attr_name): + self.attrs[attr_name].append(value) + else: + self.attrs[attr_name] = [value] + elif name == "Item": + self.uploader.items[self.item_id] = self.attrs + # Every 40 items we spawn off the uploader + if len(self.uploader.items) >= 40: + self.uploader.start() + self.uploader = UploaderThread(self.domain.name) + elif name == "Domain": + # If we're done, spawn off our last Uploader Thread + self.uploader.start() + +from threading import Thread +class UploaderThread(Thread): + """Uploader Thread""" + + def __init__(self, domain_name): + import boto + self.sdb = boto.connect_sdb() + self.db = self.sdb.get_domain(domain_name) + self.items = {} + Thread.__init__(self) + + def run(self): + try: + self.db.batch_put_attributes(self.items) + except: + print "Exception using batch put, trying regular put instead" + for item_name in self.items: + self.db.put_attributes(item_name, self.items[item_name]) + print ".", + sys.stdout.flush() diff --git a/storefront/boto/sdb/item.py b/storefront/boto/sdb/item.py new file mode 100644 index 0000000..b81e715 --- /dev/null +++ b/storefront/boto/sdb/item.py @@ -0,0 +1,97 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SDB Item +""" + +import base64 + +class Item(dict): + + def __init__(self, domain, name='', active=False): + dict.__init__(self) + self.domain = domain + self.name = name + self.active = active + self.request_id = None + self.encoding = None + self.in_attribute = False + self.converter = self.domain.connection.converter + + def startElement(self, name, attrs, connection): + if name == 'Attribute': + self.in_attribute = True + self.encoding = attrs.get('encoding', None) + return None + + def decode_value(self, value): + if self.encoding == 'base64': + self.encoding = None + return base64.decodestring(value) + else: + return value + + def endElement(self, name, value, connection): + if name == 'ItemName': + self.name = self.decode_value(value) + elif name == 'Name': + if self.in_attribute: + self.last_key = self.decode_value(value) + else: + self.name = self.decode_value(value) + elif name == 'Value': + if self.has_key(self.last_key): + if not isinstance(self[self.last_key], list): + self[self.last_key] = [self[self.last_key]] + value = self.decode_value(value) + if self.converter: + value = self.converter.decode(value) + self[self.last_key].append(value) + else: + value = self.decode_value(value) + if self.converter: + value = self.converter.decode(value) + self[self.last_key] = value + elif name == 'BoxUsage': + try: + connection.box_usage += float(value) + except: + pass + elif name == 'RequestId': + self.request_id = value + elif name == 'Attribute': + self.in_attribute = False + else: + setattr(self, name, value) + + def load(self): + self.domain.get_attributes(self.name, item=self) + + def save(self, replace=True): + self.domain.put_attributes(self.name, self, replace) + + def delete(self): + self.domain.delete_item(self) + + + + diff --git a/storefront/boto/sdb/persist/__init__.py b/storefront/boto/sdb/persist/__init__.py new file mode 100644 index 0000000..2f2b0c1 --- /dev/null +++ b/storefront/boto/sdb/persist/__init__.py @@ -0,0 +1,83 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.utils import find_class + +class Manager(object): + + DefaultDomainName = boto.config.get('Persist', 'default_domain', None) + + def __init__(self, domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0): + self.domain_name = domain_name + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + self.domain = None + self.sdb = None + self.s3 = None + if not self.domain_name: + self.domain_name = self.DefaultDomainName + if self.domain_name: + boto.log.info('No SimpleDB domain set, using default_domain: %s' % self.domain_name) + else: + boto.log.warning('No SimpleDB domain set, persistance is disabled') + if self.domain_name: + self.sdb = boto.connect_sdb(aws_access_key_id=self.aws_access_key_id, + aws_secret_access_key=self.aws_secret_access_key, + debug=debug) + self.domain = self.sdb.lookup(self.domain_name) + if not self.domain: + self.domain = self.sdb.create_domain(self.domain_name) + + def get_s3_connection(self): + if not self.s3: + self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key) + return self.s3 + +def get_manager(domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0): + return Manager(domain_name, aws_access_key_id, aws_secret_access_key, debug=debug) + +def set_domain(domain_name): + Manager.DefaultDomainName = domain_name + +def get_domain(): + return Manager.DefaultDomainName + +def revive_object_from_id(id, manager): + if not manager.domain: + return None + attrs = manager.domain.get_attributes(id, ['__module__', '__type__', '__lineage__']) + try: + cls = find_class(attrs['__module__'], attrs['__type__']) + return cls(id, manager=manager) + except ImportError: + return None + +def object_lister(cls, query_lister, manager): + for item in query_lister: + if cls: + yield cls(item.name) + else: + o = revive_object_from_id(item.name, manager) + if o: + yield o + + diff --git a/storefront/boto/sdb/persist/checker.py b/storefront/boto/sdb/persist/checker.py new file mode 100644 index 0000000..147ea47 --- /dev/null +++ b/storefront/boto/sdb/persist/checker.py @@ -0,0 +1,303 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from datetime import datetime +import boto +from boto.s3.key import Key +from boto.s3.bucket import Bucket +from boto.sdb.persist import revive_object_from_id +from boto.exception import SDBPersistenceError +from boto.utils import Password + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +class ValueChecker: + + def check(self, value): + """ + Checks a value to see if it is of the right type. + + Should raise a TypeError exception if an in appropriate value is passed in. + """ + raise TypeError + + def from_string(self, str_value, obj): + """ + Takes a string as input and returns the type-specific value represented by that string. + + Should raise a ValueError if the value cannot be converted to the appropriate type. + """ + raise ValueError + + def to_string(self, value): + """ + Convert a value to it's string representation. + + Should raise a ValueError if the value cannot be converted to a string representation. + """ + raise ValueError + +class StringChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('maxlength'): + self.maxlength = params['maxlength'] + else: + self.maxlength = 1024 + if params.has_key('default'): + self.check(params['default']) + self.default = params['default'] + else: + self.default = '' + + def check(self, value): + if isinstance(value, str) or isinstance(value, unicode): + if len(value) > self.maxlength: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting String, got %s' % type(value) + + def from_string(self, str_value, obj): + return str_value + + def to_string(self, value): + self.check(value) + return value + +class PasswordChecker(StringChecker): + def check(self, value): + if isinstance(value, str) or isinstance(value, unicode) or isinstance(value, Password): + if len(value) > self.maxlength: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting String, got %s' % type(value) + +class IntegerChecker(ValueChecker): + + __sizes__ = { 'small' : (65535, 32767, -32768, 5), + 'medium' : (4294967295, 2147483647, -2147483648, 10), + 'large' : (18446744073709551615, 9223372036854775807, -9223372036854775808, 20)} + + def __init__(self, **params): + self.size = params.get('size', 'medium') + if self.size not in self.__sizes__.keys(): + raise ValueError, 'size must be one of %s' % self.__sizes__.keys() + self.signed = params.get('signed', True) + self.default = params.get('default', 0) + self.format_string = '%%0%dd' % self.__sizes__[self.size][-1] + + def check(self, value): + if not isinstance(value, int) and not isinstance(value, long): + raise TypeError, 'Expecting int or long, got %s' % type(value) + if self.signed: + min = self.__sizes__[self.size][2] + max = self.__sizes__[self.size][1] + else: + min = 0 + max = self.__sizes__[self.size][0] + if value > max: + raise ValueError, 'Maximum value is %d' % max + if value < min: + raise ValueError, 'Minimum value is %d' % min + + def from_string(self, str_value, obj): + val = int(str_value) + if self.signed: + val = val + self.__sizes__[self.size][2] + return val + + def to_string(self, value): + self.check(value) + if self.signed: + value += -self.__sizes__[self.size][2] + return self.format_string % value + +class BooleanChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('default'): + self.default = params['default'] + else: + self.default = False + + def check(self, value): + if not isinstance(value, bool): + raise TypeError, 'Expecting bool, got %s' % type(value) + + def from_string(self, str_value, obj): + if str_value.lower() == 'true': + return True + else: + return False + + def to_string(self, value): + self.check(value) + if value == True: + return 'true' + else: + return 'false' + +class DateTimeChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('maxlength'): + self.maxlength = params['maxlength'] + else: + self.maxlength = 1024 + if params.has_key('default'): + self.default = params['default'] + else: + self.default = datetime.now() + + def check(self, value): + if not isinstance(value, datetime): + raise TypeError, 'Expecting datetime, got %s' % type(value) + + def from_string(self, str_value, obj): + try: + return datetime.strptime(str_value, ISO8601) + except: + raise ValueError, 'Unable to convert %s to DateTime' % str_value + + def to_string(self, value): + self.check(value) + return value.strftime(ISO8601) + +class ObjectChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + self.ref_class = params.get('ref_class', None) + if self.ref_class == None: + raise SDBPersistenceError('ref_class parameter is required') + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + # ugly little hack - sometimes I want to just stick a UUID string + # in here rather than instantiate an object. + # This does a bit of hand waving to "type check" the string + t = value.split('-') + if len(t) != 5: + raise ValueError + else: + try: + obj_lineage = value.get_lineage() + cls_lineage = self.ref_class.get_lineage() + if obj_lineage.startswith(cls_lineage): + return + raise TypeError, '%s not instance of %s' % (obj_lineage, cls_lineage) + except: + raise ValueError, '%s is not an SDBObject' % value + + def from_string(self, str_value, obj): + if not str_value: + return None + try: + return revive_object_from_id(str_value, obj._manager) + except: + raise ValueError, 'Unable to convert %s to Object' % str_value + + def to_string(self, value): + self.check(value) + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return value.id + +class S3KeyChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + try: + bucket_name, key_name = value.split('/', 1) + except: + raise ValueError + elif not isinstance(value, Key): + raise TypeError, 'Expecting Key, got %s' % type(value) + + def from_string(self, str_value, obj): + if not str_value: + return None + if str_value == 'None': + return None + try: + bucket_name, key_name = str_value.split('/', 1) + if obj: + s3 = obj._manager.get_s3_connection() + bucket = s3.get_bucket(bucket_name) + key = bucket.get_key(key_name) + if not key: + key = bucket.new_key(key_name) + return key + except: + raise ValueError, 'Unable to convert %s to S3Key' % str_value + + def to_string(self, value): + self.check(value) + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return '%s/%s' % (value.bucket.name, value.name) + +class S3BucketChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + return + elif not isinstance(value, Bucket): + raise TypeError, 'Expecting Bucket, got %s' % type(value) + + def from_string(self, str_value, obj): + if not str_value: + return None + if str_value == 'None': + return None + try: + if obj: + s3 = obj._manager.get_s3_connection() + bucket = s3.get_bucket(str_value) + return bucket + except: + raise ValueError, 'Unable to convert %s to S3Bucket' % str_value + + def to_string(self, value): + self.check(value) + if value == None: + return '' + else: + return '%s' % value.name + diff --git a/storefront/boto/sdb/persist/object.py b/storefront/boto/sdb/persist/object.py new file mode 100644 index 0000000..3646d43 --- /dev/null +++ b/storefront/boto/sdb/persist/object.py @@ -0,0 +1,207 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager, object_lister +from boto.sdb.persist.property import * +import uuid + +class SDBBase(type): + "Metaclass for all SDBObjects" + def __init__(cls, name, bases, dict): + super(SDBBase, cls).__init__(name, bases, dict) + # Make sure this is a subclass of SDBObject - mainly copied from django ModelBase (thanks!) + try: + if filter(lambda b: issubclass(b, SDBObject), bases): + # look for all of the Properties and set their names + for key in dict.keys(): + if isinstance(dict[key], Property): + property = dict[key] + property.set_name(key) + prop_names = [] + props = cls.properties() + for prop in props: + prop_names.append(prop.name) + setattr(cls, '_prop_names', prop_names) + except NameError: + # 'SDBObject' isn't defined yet, meaning we're looking at our own + # SDBObject class, defined below. + pass + +class SDBObject(object): + __metaclass__ = SDBBase + + _manager = get_manager() + + @classmethod + def get_lineage(cls): + l = [c.__name__ for c in cls.mro()] + l.reverse() + return '.'.join(l) + + @classmethod + def get(cls, id=None, **params): + if params.has_key('manager'): + manager = params['manager'] + else: + manager = cls._manager + if manager.domain and id: + a = cls._manager.domain.get_attributes(id, '__type__') + if a.has_key('__type__'): + return cls(id, manager) + else: + raise SDBPersistenceError('%s object with id=%s does not exist' % (cls.__name__, id)) + else: + rs = cls.find(**params) + try: + obj = rs.next() + except StopIteration: + raise SDBPersistenceError('%s object matching query does not exist' % cls.__name__) + try: + rs.next() + except StopIteration: + return obj + raise SDBPersistenceError('Query matched more than 1 item') + + @classmethod + def find(cls, **params): + if params.has_key('manager'): + manager = params['manager'] + del params['manager'] + else: + manager = cls._manager + keys = params.keys() + if len(keys) > 4: + raise SDBPersistenceError('Too many fields, max is 4') + parts = ["['__type__'='%s'] union ['__lineage__'starts-with'%s']" % (cls.__name__, cls.get_lineage())] + properties = cls.properties() + for key in keys: + found = False + for property in properties: + if property.name == key: + found = True + if isinstance(property, ScalarProperty): + checker = property.checker + parts.append("['%s' = '%s']" % (key, checker.to_string(params[key]))) + else: + raise SDBPersistenceError('%s is not a searchable field' % key) + if not found: + raise SDBPersistenceError('%s is not a valid field' % key) + query = ' intersection '.join(parts) + if manager.domain: + rs = manager.domain.query(query) + else: + rs = [] + return object_lister(None, rs, manager) + + @classmethod + def list(cls, max_items=None, manager=None): + if not manager: + manager = cls._manager + if manager.domain: + rs = manager.domain.query("['__type__' = '%s']" % cls.__name__, max_items=max_items) + else: + rs = [] + return object_lister(cls, rs, manager) + + @classmethod + def properties(cls): + properties = [] + while cls: + for key in cls.__dict__.keys(): + if isinstance(cls.__dict__[key], Property): + properties.append(cls.__dict__[key]) + if len(cls.__bases__) > 0: + cls = cls.__bases__[0] + else: + cls = None + return properties + + # for backwards compatibility + find_properties = properties + + def __init__(self, id=None, manager=None): + if manager: + self._manager = manager + self.id = id + if self.id: + self._auto_update = True + if self._manager.domain: + attrs = self._manager.domain.get_attributes(self.id, '__type__') + if len(attrs.keys()) == 0: + raise SDBPersistenceError('Object %s: not found' % self.id) + else: + self.id = str(uuid.uuid4()) + self._auto_update = False + + def __setattr__(self, name, value): + if name in self._prop_names: + object.__setattr__(self, name, value) + elif name.startswith('_'): + object.__setattr__(self, name, value) + elif name == 'id': + object.__setattr__(self, name, value) + else: + self._persist_attribute(name, value) + object.__setattr__(self, name, value) + + def __getattr__(self, name): + if not name.startswith('_'): + a = self._manager.domain.get_attributes(self.id, name) + if a.has_key(name): + object.__setattr__(self, name, a[name]) + return a[name] + raise AttributeError + + def __repr__(self): + return '%s<%s>' % (self.__class__.__name__, self.id) + + def _persist_attribute(self, name, value): + if self.id: + self._manager.domain.put_attributes(self.id, {name : value}, replace=True) + + def _get_sdb_item(self): + return self._manager.domain.get_item(self.id) + + def save(self): + attrs = {'__type__' : self.__class__.__name__, + '__module__' : self.__class__.__module__, + '__lineage__' : self.get_lineage()} + for property in self.properties(): + attrs[property.name] = property.to_string(self) + if self._manager.domain: + self._manager.domain.put_attributes(self.id, attrs, replace=True) + self._auto_update = True + + def delete(self): + if self._manager.domain: + self._manager.domain.delete_attributes(self.id) + + def get_related_objects(self, ref_name, ref_cls=None): + if self._manager.domain: + query = "['%s' = '%s']" % (ref_name, self.id) + if ref_cls: + query += " intersection ['__type__'='%s']" % ref_cls.__name__ + rs = self._manager.domain.query(query) + else: + rs = [] + return object_lister(ref_cls, rs, self._manager) + diff --git a/storefront/boto/sdb/persist/property.py b/storefront/boto/sdb/persist/property.py new file mode 100644 index 0000000..6eea765 --- /dev/null +++ b/storefront/boto/sdb/persist/property.py @@ -0,0 +1,370 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import SDBPersistenceError +from boto.sdb.persist.checker import * +from boto.utils import Password + +class Property(object): + + def __init__(self, checker_class, **params): + self.name = '' + self.checker = checker_class(**params) + self.slot_name = '__' + + def set_name(self, name): + self.name = name + self.slot_name = '__' + self.name + +class ScalarProperty(Property): + + def save(self, obj): + domain = obj._manager.domain + domain.put_attributes(obj.id, {self.name : self.to_string(obj)}, replace=True) + + def to_string(self, obj): + return self.checker.to_string(getattr(obj, self.name)) + + def load(self, obj): + domain = obj._manager.domain + a = domain.get_attributes(obj.id, self.name) + # try to get the attribute value from SDB + if self.name in a: + value = self.checker.from_string(a[self.name], obj) + setattr(obj, self.slot_name, value) + # if it's not there, set the value to the default value + else: + self.__set__(obj, self.checker.default) + + def __get__(self, obj, objtype): + if obj: + try: + value = getattr(obj, self.slot_name) + except AttributeError: + if obj._auto_update: + self.load(obj) + value = getattr(obj, self.slot_name) + else: + value = self.checker.default + setattr(obj, self.slot_name, self.checker.default) + return value + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class StringProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, StringChecker, **params) + +class PasswordProperty(ScalarProperty): + """ + Hashed password + """ + + def __init__(self, **params): + ScalarProperty.__init__(self, PasswordChecker, **params) + + def __set__(self, obj, value): + p = Password() + p.set(value) + ScalarProperty.__set__(self, obj, p) + + def __get__(self, obj, objtype): + return Password(ScalarProperty.__get__(self, obj, objtype)) + +class SmallPositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class SmallIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class PositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class IntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class LargePositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class LargeIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class BooleanProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, BooleanChecker, **params) + +class DateTimeProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, DateTimeChecker, **params) + +class ObjectProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, ObjectChecker, **params) + +class S3KeyProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, S3KeyChecker, **params) + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + if isinstance(value, str): + value = self.checker.from_string(value, obj) + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class S3BucketProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, S3BucketChecker, **params) + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + if isinstance(value, str): + value = self.checker.from_string(value, obj) + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class MultiValueProperty(Property): + + def __init__(self, checker_class, **params): + Property.__init__(self, checker_class, **params) + + def __get__(self, obj, objtype): + if obj: + try: + value = getattr(obj, self.slot_name) + except AttributeError: + if obj._auto_update: + self.load(obj) + value = getattr(obj, self.slot_name) + else: + value = MultiValue(self, obj, []) + setattr(obj, self.slot_name, value) + return value + + def load(self, obj): + if obj != None: + _list = [] + domain = obj._manager.domain + a = domain.get_attributes(obj.id, self.name) + if self.name in a: + lst = a[self.name] + if not isinstance(lst, list): + lst = [lst] + for value in lst: + value = self.checker.from_string(value, obj) + _list.append(value) + setattr(obj, self.slot_name, MultiValue(self, obj, _list)) + + def __set__(self, obj, value): + if not isinstance(value, list): + raise SDBPersistenceError('Value must be a list') + setattr(obj, self.slot_name, MultiValue(self, obj, value)) + str_list = self.to_string(obj) + domain = obj._manager.domain + if obj._auto_update: + if len(str_list) == 1: + domain.put_attributes(obj.id, {self.name : str_list[0]}, replace=True) + else: + try: + self.__delete__(obj) + except: + pass + domain.put_attributes(obj.id, {self.name : str_list}, replace=True) + setattr(obj, self.slot_name, MultiValue(self, obj, value)) + + def __delete__(self, obj): + if obj._auto_update: + domain = obj._manager.domain + domain.delete_attributes(obj.id, [self.name]) + setattr(obj, self.slot_name, MultiValue(self, obj, [])) + + def to_string(self, obj): + str_list = [] + for value in self.__get__(obj, type(obj)): + str_list.append(self.checker.to_string(value)) + return str_list + +class StringListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, StringChecker, **params) + +class SmallIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class SmallPositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class IntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class PositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class LargeIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class LargePositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class BooleanListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, BooleanChecker, **params) + +class ObjectListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, ObjectChecker, **params) + +class HasManyProperty(Property): + + def set_name(self, name): + self.name = name + self.slot_name = '__' + self.name + + def __get__(self, obj, objtype): + return self + + +class MultiValue: + """ + Special Multi Value for boto persistence layer to allow us to do + obj.list.append(foo) + """ + def __init__(self, property, obj, _list): + self.checker = property.checker + self.name = property.name + self.object = obj + self._list = _list + + def __repr__(self): + return repr(self._list) + + def __getitem__(self, key): + return self._list.__getitem__(key) + + def __delitem__(self, key): + item = self[key] + self._list.__delitem__(key) + domain = self.object._manager.domain + domain.delete_attributes(self.object.id, {self.name: [self.checker.to_string(item)]}) + + def __len__(self): + return len(self._list) + + def append(self, value): + self.checker.check(value) + self._list.append(value) + domain = self.object._manager.domain + domain.put_attributes(self.object.id, {self.name: self.checker.to_string(value)}, replace=False) + + def index(self, value): + for x in self._list: + if x.id == value.id: + return self._list.index(x) + + def remove(self, value): + del(self[self.index(value)]) diff --git a/storefront/boto/sdb/persist/test_persist.py b/storefront/boto/sdb/persist/test_persist.py new file mode 100644 index 0000000..3207e58 --- /dev/null +++ b/storefront/boto/sdb/persist/test_persist.py @@ -0,0 +1,138 @@ +from boto.sdb.persist.object import SDBObject +from boto.sdb.persist.property import * +from boto.sdb.persist import Manager +from datetime import datetime +import time + +# +# This will eventually be moved to the boto.tests module and become a real unit test +# but for now it will live here. It shows examples of each of the Property types in +# use and tests the basic operations. +# +class TestScalar(SDBObject): + + name = StringProperty() + description = StringProperty() + size = PositiveIntegerProperty() + offset = IntegerProperty() + foo = BooleanProperty() + date = DateTimeProperty() + file = S3KeyProperty() + +class TestRef(SDBObject): + + name = StringProperty() + ref = ObjectProperty(ref_class=TestScalar) + +class TestSubClass1(TestRef): + + answer = PositiveIntegerProperty() + +class TestSubClass2(TestScalar): + + flag = BooleanProperty() + +class TestList(SDBObject): + + names = StringListProperty() + numbers = PositiveIntegerListProperty() + bools = BooleanListProperty() + objects = ObjectListProperty(ref_class=TestScalar) + +def test1(): + s = TestScalar() + s.name = 'foo' + s.description = 'This is foo' + s.size = 42 + s.offset = -100 + s.foo = True + s.date = datetime.now() + s.save() + return s + +def test2(ref_name): + s = TestRef() + s.name = 'testref' + rs = TestScalar.find(name=ref_name) + s.ref = rs.next() + s.save() + return s + +def test3(): + s = TestScalar() + s.name = 'bar' + s.description = 'This is bar' + s.size = 24 + s.foo = False + s.date = datetime.now() + s.save() + return s + +def test4(ref1, ref2): + s = TestList() + s.names.append(ref1.name) + s.names.append(ref2.name) + s.numbers.append(ref1.size) + s.numbers.append(ref2.size) + s.bools.append(ref1.foo) + s.bools.append(ref2.foo) + s.objects.append(ref1) + s.objects.append(ref2) + s.save() + return s + +def test5(ref): + s = TestSubClass1() + s.answer = 42 + s.ref = ref + s.save() + # test out free form attribute + s.fiddlefaddle = 'this is fiddlefaddle' + s._fiddlefaddle = 'this is not fiddlefaddle' + return s + +def test6(): + s = TestSubClass2() + s.name = 'fie' + s.description = 'This is fie' + s.size = 4200 + s.offset = -820 + s.foo = False + s.date = datetime.now() + s.flag = True + s.save() + return s + +def test(domain_name): + print 'Initialize the Persistance system' + Manager.DefaultDomainName = domain_name + print 'Call test1' + s1 = test1() + # now create a new instance and read the saved data from SDB + print 'Now sleep to wait for things to converge' + time.sleep(5) + print 'Now lookup the object and compare the fields' + s2 = TestScalar(s1.id) + assert s1.name == s2.name + assert s1.description == s2.description + assert s1.size == s2.size + assert s1.offset == s2.offset + assert s1.foo == s2.foo + #assert s1.date == s2.date + print 'Call test2' + s2 = test2(s1.name) + print 'Call test3' + s3 = test3() + print 'Call test4' + s4 = test4(s1, s3) + print 'Call test5' + s6 = test6() + s5 = test5(s6) + domain = s5._manager.domain + item1 = domain.get_item(s1.id) + item2 = domain.get_item(s2.id) + item3 = domain.get_item(s3.id) + item4 = domain.get_item(s4.id) + item5 = domain.get_item(s5.id) + item6 = domain.get_item(s6.id) + return [(s1, item1), (s2, item2), (s3, item3), (s4, item4), (s5, item5), (s6, item6)] diff --git a/storefront/boto/sdb/queryresultset.py b/storefront/boto/sdb/queryresultset.py new file mode 100644 index 0000000..a9430f4 --- /dev/null +++ b/storefront/boto/sdb/queryresultset.py @@ -0,0 +1,92 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.sdb.item import Item + +def query_lister(domain, query='', max_items=None, attr_names=None): + more_results = True + num_results = 0 + next_token = None + while more_results: + rs = domain.connection.query_with_attributes(domain, query, attr_names, + next_token=next_token) + for item in rs: + if max_items: + if num_results == max_items: + raise StopIteration + yield item + num_results += 1 + next_token = rs.next_token + more_results = next_token != None + +class QueryResultSet: + + def __init__(self, domain=None, query='', max_items=None, attr_names=None): + self.max_items = max_items + self.domain = domain + self.query = query + self.attr_names = attr_names + + def __iter__(self): + return query_lister(self.domain, self.query, self.max_items, self.attr_names) + +def select_lister(domain, query='', max_items=None): + more_results = True + num_results = 0 + next_token = None + while more_results: + rs = domain.connection.select(domain, query, next_token=next_token) + for item in rs: + if max_items: + if num_results == max_items: + raise StopIteration + yield item + num_results += 1 + next_token = rs.next_token + more_results = next_token != None + +class SelectResultSet(object): + + def __init__(self, domain=None, query='', max_items=None, + next_token=None): + self.domain = domain + self.query = query + self.max_items = max_items + self.next_token = next_token + + def __iter__(self): + more_results = True + num_results = 0 + while more_results: + rs = self.domain.connection.select(self.domain, self.query, + next_token=self.next_token) + for item in rs: + if self.max_items and num_results >= self.max_items: + raise StopIteration + yield item + num_results += 1 + self.next_token = rs.next_token + if self.max_items and num_results >= self.max_items: + raise StopIteration + more_results = self.next_token != None + + def next(self): + return self.__iter__().next() diff --git a/storefront/boto/sdb/regioninfo.py b/storefront/boto/sdb/regioninfo.py new file mode 100644 index 0000000..bff9dea --- /dev/null +++ b/storefront/boto/sdb/regioninfo.py @@ -0,0 +1,40 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from boto.ec2.regioninfo import RegionInfo + +class SDBRegionInfo(RegionInfo): + + def connect(self, **kw_params): + """ + Connect to this Region's endpoint. Returns an SDBConnection + object pointing to the endpoint associated with this region. + You may pass any of the arguments accepted by the SDBConnection + object's constructor as keyword arguments and they will be + passed along to the SDBConnection object. + + :rtype: :class:`boto.sdb.connection.SDBConnection` + :return: The connection to this regions endpoint + """ + from boto.sdb.connection import SDBConnection + return SDBConnection(region=self, **kw_params) + diff --git a/storefront/boto/services/__init__.py b/storefront/boto/services/__init__.py new file mode 100644 index 0000000..449bd16 --- /dev/null +++ b/storefront/boto/services/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/storefront/boto/services/bs.py b/storefront/boto/services/bs.py new file mode 100755 index 0000000..aafe867 --- /dev/null +++ b/storefront/boto/services/bs.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +from optparse import OptionParser +from boto.services.servicedef import ServiceDef +from boto.services.message import ServiceMessage +from boto.services.submit import Submitter +from boto.services.result import ResultProcessor +import boto +import sys, os, StringIO + +class BS(object): + + Usage = "usage: %prog [options] config_file command" + + Commands = {'reset' : 'Clear input queue and output bucket', + 'submit' : 'Submit local files to the service', + 'start' : 'Start the service', + 'status' : 'Report on the status of the service buckets and queues', + 'retrieve' : 'Retrieve output generated by a batch', + 'batches' : 'List all batches stored in current output_domain'} + + def __init__(self): + self.service_name = None + self.parser = OptionParser(usage=self.Usage) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option("-a", "--access-key", action="store", type="string", + help="your AWS Access Key") + self.parser.add_option("-s", "--secret-key", action="store", type="string", + help="your AWS Secret Access Key") + self.parser.add_option("-p", "--path", action="store", type="string", dest="path", + help="the path to local directory for submit and retrieve") + self.parser.add_option("-k", "--keypair", action="store", type="string", dest="keypair", + help="the SSH keypair used with launched instance(s)") + self.parser.add_option("-l", "--leave", action="store_true", dest="leave", + help="leave the files (don't retrieve) files during retrieve command") + self.parser.set_defaults(leave=False) + self.parser.add_option("-n", "--num-instances", action="store", type="string", dest="num_instances", + help="the number of launched instance(s)") + self.parser.set_defaults(num_instances=1) + self.parser.add_option("-i", "--ignore-dirs", action="append", type="string", dest="ignore", + help="directories that should be ignored by submit command") + self.parser.add_option("-b", "--batch-id", action="store", type="string", dest="batch", + help="batch identifier required by the retrieve command") + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_reset(self): + iq = self.sd.get_obj('input_queue') + if iq: + print 'clearing out input queue' + i = 0 + m = iq.read() + while m: + i += 1 + iq.delete_message(m) + m = iq.read() + print 'deleted %d messages' % i + ob = self.sd.get_obj('output_bucket') + ib = self.sd.get_obj('input_bucket') + if ob: + if ib and ob.name == ib.name: + return + print 'delete generated files in output bucket' + i = 0 + for k in ob: + i += 1 + k.delete() + print 'deleted %d keys' % i + + def do_submit(self): + if not self.options.path: + self.parser.error('No path provided') + if not os.path.exists(self.options.path): + self.parser.error('Invalid path (%s)' % self.options.path) + s = Submitter(self.sd) + t = s.submit_path(self.options.path, None, self.options.ignore, None, + None, True, self.options.path) + print 'A total of %d files were submitted' % t[1] + print 'Batch Identifier: %s' % t[0] + + def do_start(self): + ami_id = self.sd.get('ami_id') + instance_type = self.sd.get('instance_type', 'm1.small') + security_group = self.sd.get('security_group', 'default') + if not ami_id: + self.parser.error('ami_id option is required when starting the service') + ec2 = boto.connect_ec2() + if not self.sd.has_section('Credentials'): + self.sd.add_section('Credentials') + self.sd.set('Credentials', 'aws_access_key_id', ec2.aws_access_key_id) + self.sd.set('Credentials', 'aws_secret_access_key', ec2.aws_secret_access_key) + s = StringIO.StringIO() + self.sd.write(s) + rs = ec2.get_all_images([ami_id]) + img = rs[0] + r = img.run(user_data=s.getvalue(), key_name=self.options.keypair, + max_count=self.options.num_instances, + instance_type=instance_type, + security_groups=[security_group]) + print 'Starting AMI: %s' % ami_id + print 'Reservation %s contains the following instances:' % r.id + for i in r.instances: + print '\t%s' % i.id + + def do_status(self): + iq = self.sd.get_obj('input_queue') + if iq: + print 'The input_queue (%s) contains approximately %s messages' % (iq.id, iq.count()) + ob = self.sd.get_obj('output_bucket') + ib = self.sd.get_obj('input_bucket') + if ob: + if ib and ob.name == ib.name: + return + total = 0 + for k in ob: + total += 1 + print 'The output_bucket (%s) contains %d keys' % (ob.name, total) + + def do_retrieve(self): + if not self.options.path: + self.parser.error('No path provided') + if not os.path.exists(self.options.path): + self.parser.error('Invalid path (%s)' % self.options.path) + if not self.options.batch: + self.parser.error('batch identifier is required for retrieve command') + s = ResultProcessor(self.options.batch, self.sd) + s.get_results(self.options.path, get_file=(not self.options.leave)) + + def do_batches(self): + d = self.sd.get_obj('output_domain') + if d: + print 'Available Batches:' + rs = d.query("['type'='Batch']") + for item in rs: + print ' %s' % item.name + else: + self.parser.error('No output_domain specified for service') + + def main(self): + self.options, self.args = self.parser.parse_args() + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) != 2: + self.parser.error("config_file and command are required") + self.config_file = self.args[0] + self.sd = ServiceDef(self.config_file) + self.command = self.args[1] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + +if __name__ == "__main__": + bs = BS() + bs.main() diff --git a/storefront/boto/services/message.py b/storefront/boto/services/message.py new file mode 100644 index 0000000..6bb2e58 --- /dev/null +++ b/storefront/boto/services/message.py @@ -0,0 +1,59 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.sqs.message import MHMessage +from boto.utils import get_ts +from socket import gethostname +import os, mimetypes, time + +class ServiceMessage(MHMessage): + + def for_key(self, key, params=None, bucket_name=None): + if params: + self.update(params) + if key.path: + t = os.path.split(key.path) + self['OriginalLocation'] = t[0] + self['OriginalFileName'] = t[1] + mime_type = mimetypes.guess_type(t[1])[0] + if mime_type == None: + mime_type = 'application/octet-stream' + self['Content-Type'] = mime_type + s = os.stat(key.path) + t = time.gmtime(s[7]) + self['FileAccessedDate'] = get_ts(t) + t = time.gmtime(s[8]) + self['FileModifiedDate'] = get_ts(t) + t = time.gmtime(s[9]) + self['FileCreateDate'] = get_ts(t) + else: + self['OriginalFileName'] = key.name + self['OriginalLocation'] = key.bucket.name + self['ContentType'] = key.content_type + self['Host'] = gethostname() + if bucket_name: + self['Bucket'] = bucket_name + else: + self['Bucket'] = key.bucket.name + self['InputKey'] = key.name + self['Size'] = key.size + diff --git a/storefront/boto/services/result.py b/storefront/boto/services/result.py new file mode 100644 index 0000000..240085b --- /dev/null +++ b/storefront/boto/services/result.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import getopt, sys, os, time, mimetypes +from datetime import datetime, timedelta +from boto.services.servicedef import ServiceDef +from boto.utils import parse_ts +import boto + +class ResultProcessor: + + LogFileName = 'log.csv' + + def __init__(self, batch_name, sd, mimetype_files=None): + self.sd = sd + self.batch = batch_name + self.log_fp = None + self.num_files = 0 + self.total_time = 0 + self.min_time = timedelta.max + self.max_time = timedelta.min + self.earliest_time = datetime.max + self.latest_time = datetime.min + self.queue = self.sd.get_obj('output_queue') + self.domain = self.sd.get_obj('output_domain') + + def calculate_stats(self, msg): + start_time = parse_ts(msg['Service-Read']) + end_time = parse_ts(msg['Service-Write']) + elapsed_time = end_time - start_time + if elapsed_time > self.max_time: + self.max_time = elapsed_time + if elapsed_time < self.min_time: + self.min_time = elapsed_time + self.total_time += elapsed_time.seconds + if start_time < self.earliest_time: + self.earliest_time = start_time + if end_time > self.latest_time: + self.latest_time = end_time + + def log_message(self, msg, path): + keys = msg.keys() + keys.sort() + if not self.log_fp: + self.log_fp = open(os.path.join(path, self.LogFileName), 'w') + line = ','.join(keys) + self.log_fp.write(line+'\n') + values = [] + for key in keys: + value = msg[key] + if value.find(',') > 0: + value = '"%s"' % value + values.append(value) + line = ','.join(values) + self.log_fp.write(line+'\n') + + def process_record(self, record, path, get_file=True): + self.log_message(record, path) + self.calculate_stats(record) + outputs = record['OutputKey'].split(',') + if record.has_key('OutputBucket'): + bucket = boto.lookup('s3', record['OutputBucket']) + else: + bucket = boto.lookup('s3', record['Bucket']) + for output in outputs: + if get_file: + key_name, type = output.split(';') + if type: + mimetype = type.split('=')[1] + key = bucket.lookup(key_name) + file_name = os.path.join(path, key_name) + print 'retrieving file: %s to %s' % (key_name, file_name) + key.get_contents_to_filename(file_name) + self.num_files += 1 + + def get_results_from_queue(self, path, get_file=True, delete_msg=True): + m = self.queue.read() + while m: + if m.has_key('Batch') and m['Batch'] == self.batch: + self.process_record(m, path, get_file) + if delete_msg: + self.queue.delete_message(m) + m = self.queue.read() + + def get_results_from_domain(self, path, get_file=True): + rs = self.domain.query("['Batch'='%s']" % self.batch) + for item in rs: + self.process_record(item, path, get_file) + + def get_results_from_bucket(self, path): + bucket = self.sd.get_obj('output_bucket') + if bucket: + print 'No output queue or domain, just retrieving files from output_bucket' + for key in bucket: + file_name = os.path.join(path, key_name) + print 'retrieving file: %s to %s' % (key_name, file_name) + key.get_contents_to_filename(file_name) + self.num_files + 1 + + def get_results(self, path, get_file=True, delete_msg=True): + if not os.path.isdir(path): + os.mkdir(path) + if self.queue: + self.get_results_from_queue(path, get_file) + elif self.domain: + self.get_results_from_domain(path, get_file) + else: + self.get_results_from_bucket(path) + if self.log_fp: + self.log_fp.close() + print '%d results successfully retrieved.' % self.num_files + if self.num_files > 0: + self.avg_time = float(self.total_time)/self.num_files + print 'Minimum Processing Time: %d' % self.min_time.seconds + print 'Maximum Processing Time: %d' % self.max_time.seconds + print 'Average Processing Time: %f' % self.avg_time + self.elapsed_time = self.latest_time-self.earliest_time + print 'Elapsed Time: %d' % self.elapsed_time.seconds + tput = 1.0 / ((self.elapsed_time.seconds/60.0) / self.num_files) + print 'Throughput: %f transactions / minute' % tput + diff --git a/storefront/boto/services/service.py b/storefront/boto/services/service.py new file mode 100644 index 0000000..942c47f --- /dev/null +++ b/storefront/boto/services/service.py @@ -0,0 +1,163 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.services.message import ServiceMessage +from boto.services.servicedef import ServiceDef +from boto.pyami.scriptbase import ScriptBase +from boto.exception import S3ResponseError +from boto.utils import get_ts +import StringIO +import time +import os +import sys, traceback +import mimetypes + +class Service(ScriptBase): + + # Time required to process a transaction + ProcessingTime = 60 + + def __init__(self, config_file=None, mimetype_files=None): + ScriptBase.__init__(self, config_file) + self.name = self.__class__.__name__ + self.working_dir = boto.config.get('Pyami', 'working_dir') + self.sd = ServiceDef(config_file) + self.retry_count = self.sd.getint('retry_count', 5) + self.loop_delay = self.sd.getint('loop_delay', 30) + self.processing_time = self.sd.getint('processing_time', 60) + self.input_queue = self.sd.get_obj('input_queue') + self.output_queue = self.sd.get_obj('output_queue') + self.output_domain = self.sd.get_obj('output_domain') + if mimetype_files: + mimetypes.init(mimetype_files) + + def split_key(key): + if key.find(';') < 0: + t = (key, '') + else: + key, type = key.split(';') + label, mtype = type.split('=') + t = (key, mtype) + return t + + def read_message(self): + boto.log.info('read_message') + message = self.input_queue.read(self.processing_time) + if message: + boto.log.info(message.get_body()) + key = 'Service-Read' + message[key] = get_ts() + return message + + # retrieve the source file from S3 + def get_file(self, message): + bucket_name = message['Bucket'] + key_name = message['InputKey'] + file_name = os.path.join(self.working_dir, message.get('OriginalFileName', 'in_file')) + boto.log.info('get_file: %s/%s to %s' % (bucket_name, key_name, file_name)) + bucket = boto.lookup('s3', bucket_name) + key = bucket.new_key(key_name) + key.get_contents_to_filename(os.path.join(self.working_dir, file_name)) + return file_name + + # process source file, return list of output files + def process_file(self, in_file_name, msg): + return [] + + # store result file in S3 + def put_file(self, bucket_name, file_path, key_name=None): + boto.log.info('putting file %s as %s.%s' % (file_path, bucket_name, key_name)) + bucket = boto.lookup('s3', bucket_name) + key = bucket.new_key(key_name) + key.set_contents_from_filename(file_path) + return key + + def save_results(self, results, input_message, output_message): + output_keys = [] + for file, type in results: + if input_message.has_key('OutputBucket'): + output_bucket = input_message['OutputBucket'] + else: + output_bucket = input_message['Bucket'] + key_name = os.path.split(file)[1] + key = self.put_file(output_bucket, file, key_name) + output_keys.append('%s;type=%s' % (key.name, type)) + output_message['OutputKey'] = ','.join(output_keys) + + # write message to each output queue + def write_message(self, message): + message['Service-Write'] = get_ts() + message['Server'] = self.name + if os.environ.has_key('HOSTNAME'): + message['Host'] = os.environ['HOSTNAME'] + else: + message['Host'] = 'unknown' + message['Instance-ID'] = self.instance_id + if self.output_queue: + boto.log.info('Writing message to SQS queue: %s' % self.output_queue.id) + self.output_queue.write(message) + if self.output_domain: + boto.log.info('Writing message to SDB domain: %s' % self.output_domain.name) + item_name = '/'.join([message['Service-Write'], message['Bucket'], message['InputKey']]) + self.output_domain.put_attributes(item_name, message) + + # delete message from input queue + def delete_message(self, message): + boto.log.info('deleting message from %s' % self.input_queue.id) + self.input_queue.delete_message(message) + + # to clean up any files, etc. after each iteration + def cleanup(self): + pass + + def shutdown(self): + on_completion = self.sd.get('on_completion', 'shutdown') + if on_completion == 'shutdown': + if self.instance_id: + time.sleep(60) + c = boto.connect_ec2() + c.terminate_instances([self.instance_id]) + + def main(self, notify=False): + self.notify('Service: %s Starting' % self.name) + empty_reads = 0 + while self.retry_count < 0 or empty_reads < self.retry_count: + try: + input_message = self.read_message() + if input_message: + empty_reads = 0 + output_message = ServiceMessage(None, input_message.get_body()) + input_file = self.get_file(input_message) + results = self.process_file(input_file, output_message) + self.save_results(results, input_message, output_message) + self.write_message(output_message) + self.delete_message(input_message) + self.cleanup() + else: + empty_reads += 1 + time.sleep(self.loop_delay) + except Exception, e: + boto.log.exception('Service Failed') + empty_reads += 1 + self.notify('Service: %s Shutting Down' % self.name) + self.shutdown() + diff --git a/storefront/boto/services/servicedef.py b/storefront/boto/services/servicedef.py new file mode 100644 index 0000000..1cb01aa --- /dev/null +++ b/storefront/boto/services/servicedef.py @@ -0,0 +1,91 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.pyami.config import Config +from boto.services.message import ServiceMessage +import boto + +class ServiceDef(Config): + + def __init__(self, config_file, aws_access_key_id=None, aws_secret_access_key=None): + Config.__init__(self, config_file) + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + script = Config.get(self, 'Pyami', 'scripts') + if script: + self.name = script.split('.')[-1] + else: + self.name = None + + + def get(self, name, default=None): + return Config.get(self, self.name, name, default) + + def has_option(self, option): + return Config.has_option(self, self.name, option) + + def getint(self, option, default=0): + try: + val = Config.get(self, self.name, option) + val = int(val) + except: + val = int(default) + return val + + def getbool(self, option, default=False): + try: + val = Config.get(self, self.name, option) + if val.lower() == 'true': + val = True + else: + val = False + except: + val = default + return val + + def get_obj(self, name): + """ + Returns the AWS object associated with a given option. + + The heuristics used are a bit lame. If the option name contains + the word 'bucket' it is assumed to be an S3 bucket, if the name + contains the word 'queue' it is assumed to be an SQS queue and + if it contains the word 'domain' it is assumed to be a SimpleDB + domain. If the option name specified does not exist in the + config file or if the AWS object cannot be retrieved this + returns None. + """ + val = self.get(name) + if not val: + return None + if name.find('queue') >= 0: + obj = boto.lookup('sqs', val) + if obj: + obj.set_message_class(ServiceMessage) + elif name.find('bucket') >= 0: + obj = boto.lookup('s3', val) + elif name.find('domain') >= 0: + obj = boto.lookup('sdb', val) + else: + obj = None + return obj + + diff --git a/storefront/boto/services/sonofmmm.cfg b/storefront/boto/services/sonofmmm.cfg new file mode 100644 index 0000000..d70d379 --- /dev/null +++ b/storefront/boto/services/sonofmmm.cfg @@ -0,0 +1,43 @@ +# +# Your AWS Credentials +# You only need to supply these in this file if you are not using +# the boto tools to start your service +# +#[Credentials] +#aws_access_key_id = +#aws_secret_access_key = + +# +# Fill out this section if you want emails from the service +# when it starts and stops +# +#[Notification] +#smtp_host = +#smtp_user = +#smtp_pass = +#smtp_from = +#smtp_to = + +[Pyami] +scripts = boto.services.sonofmmm.SonOfMMM + +[SonOfMMM] +# id of the AMI to be launched +ami_id = ami-dc799cb5 +# number of times service will read an empty queue before exiting +# a negative value will cause the service to run forever +retry_count = 5 +# seconds to wait after empty queue read before reading again +loop_delay = 10 +# average time it takes to process a transaction +# controls invisibility timeout of messages +processing_time = 60 +ffmpeg_args = -y -i %%s -f mov -r 29.97 -b 1200kb -mbd 2 -flags +4mv+trell -aic 2 -cmp 2 -subcmp 2 -ar 48000 -ab 19200 -s 320x240 -vcodec mpeg4 -acodec libfaac %%s +output_mimetype = video/quicktime +output_ext = .mov +input_bucket = +output_bucket = +output_domain = +output_queue = +input_queue = + diff --git a/storefront/boto/services/sonofmmm.py b/storefront/boto/services/sonofmmm.py new file mode 100644 index 0000000..5b94f90 --- /dev/null +++ b/storefront/boto/services/sonofmmm.py @@ -0,0 +1,80 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.services.service import Service +from boto.services.message import ServiceMessage +import os, time, mimetypes + +class SonOfMMM(Service): + + def __init__(self, config_file=None): + Service.__init__(self, config_file) + self.log_file = '%s.log' % self.instance_id + self.log_path = os.path.join(self.working_dir, self.log_file) + boto.set_file_logger(self.name, self.log_path) + if self.sd.has_option('ffmpeg_args'): + self.command = '/usr/local/bin/ffmpeg ' + self.sd.get('ffmpeg_args') + else: + self.command = '/usr/local/bin/ffmpeg -y -i %s %s' + self.output_mimetype = self.sd.get('output_mimetype') + if self.sd.has_option('output_ext'): + self.output_ext = self.sd.get('output_ext') + else: + self.output_ext = mimetypes.guess_extension(self.output_mimetype) + self.output_bucket = self.sd.get_obj('output_bucket') + self.input_bucket = self.sd.get_obj('input_bucket') + # check to see if there are any messages queue + # if not, create messages for all files in input_bucket + m = self.input_queue.read(1) + if not m: + self.queue_files() + + def queue_files(self): + boto.log.info('Queueing files from %s' % self.input_bucket.name) + for key in self.input_bucket: + boto.log.info('Queueing %s' % key.name) + m = ServiceMessage() + if self.output_bucket: + d = {'OutputBucket' : self.output_bucket.name} + else: + d = None + m.for_key(key, d) + self.input_queue.write(m) + + def process_file(self, in_file_name, msg): + base, ext = os.path.splitext(in_file_name) + out_file_name = os.path.join(self.working_dir, + base+self.output_ext) + command = self.command % (in_file_name, out_file_name) + boto.log.info('running:\n%s' % command) + status = self.run(command) + if status == 0: + return [(out_file_name, self.output_mimetype)] + else: + return [] + + def shutdown(self): + if os.path.isfile(self.log_path): + if self.output_bucket: + key = self.output_bucket.new_key(self.log_file) + key.set_contents_from_filename(self.log_path) + Service.shutdown(self) diff --git a/storefront/boto/services/submit.py b/storefront/boto/services/submit.py new file mode 100644 index 0000000..dfa71f2 --- /dev/null +++ b/storefront/boto/services/submit.py @@ -0,0 +1,87 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +import time, os + +class Submitter: + + def __init__(self, sd): + self.sd = sd + self.input_bucket = self.sd.get_obj('input_bucket') + self.output_bucket = self.sd.get_obj('output_bucket') + self.output_domain = self.sd.get_obj('output_domain') + self.queue = self.sd.get_obj('input_queue') + + def get_key_name(self, fullpath, prefix): + key_name = fullpath[len(prefix):] + l = key_name.split(os.sep) + return '/'.join(l) + + def write_message(self, key, metadata): + if self.queue: + m = self.queue.new_message() + m.for_key(key, metadata) + if self.output_bucket: + m['OutputBucket'] = self.output_bucket.name + self.queue.write(m) + + def submit_file(self, path, metadata=None, cb=None, num_cb=0, prefix='/'): + if not metadata: + metadata = {} + key_name = self.get_key_name(path, prefix) + k = self.input_bucket.new_key(key_name) + k.update_metadata(metadata) + k.set_contents_from_filename(path, replace=False, cb=cb, num_cb=num_cb) + self.write_message(k, metadata) + + def submit_path(self, path, tags=None, ignore_dirs=None, cb=None, num_cb=0, status=False, prefix='/'): + path = os.path.expanduser(path) + path = os.path.expandvars(path) + path = os.path.abspath(path) + total = 0 + metadata = {} + if tags: + metadata['Tags'] = tags + l = [] + for t in time.gmtime(): + l.append(str(t)) + metadata['Batch'] = '_'.join(l) + if self.output_domain: + self.output_domain.put_attributes(metadata['Batch'], {'type' : 'Batch'}) + if os.path.isdir(path): + for root, dirs, files in os.walk(path): + if ignore_dirs: + for ignore in ignore_dirs: + if ignore in dirs: + dirs.remove(ignore) + for file in files: + fullpath = os.path.join(root, file) + if status: + print 'Submitting %s' % fullpath + self.submit_file(fullpath, metadata, cb, num_cb, prefix) + total += 1 + elif os.path.isfile(path): + self.submit_file(path, metadata, cb, num_cb) + total += 1 + else: + print 'problem with %s' % path + return (metadata['Batch'], total) diff --git a/storefront/boto/sqs/20070501/__init__.py b/storefront/boto/sqs/20070501/__init__.py new file mode 100644 index 0000000..561f9cf --- /dev/null +++ b/storefront/boto/sqs/20070501/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from connection import SQSConnection as Connection +from queue import Queue +from message import Message, MHMessage + +__all__ = ['Connection', 'Queue', 'Message', 'MHMessage'] diff --git a/storefront/boto/sqs/20070501/attributes.py b/storefront/boto/sqs/20070501/attributes.py new file mode 100644 index 0000000..b13370d --- /dev/null +++ b/storefront/boto/sqs/20070501/attributes.py @@ -0,0 +1,45 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Attribute Name/Value set +""" + +class Attributes(dict): + + def __init__(self): + self.current_key = None + self.current_value = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'AttributedValue': + self[self.current_key] = self.current_value + elif name == 'Attribute': + self.current_key = value + elif name == 'Value': + self.current_value = value + else: + setattr(self, name, value) + + diff --git a/storefront/boto/sqs/20070501/connection.py b/storefront/boto/sqs/20070501/connection.py new file mode 100644 index 0000000..7890aa2 --- /dev/null +++ b/storefront/boto/sqs/20070501/connection.py @@ -0,0 +1,389 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.connection import AWSAuthConnection, AWSQueryConnection +import xml.sax +from boto.sqs.queue import Queue +from boto.sqs.message import Message +from boto.sqs.attributes import Attributes +from boto import handler +from boto.resultset import ResultSet +from boto.exception import SQSError + +PERM_ReceiveMessage = 'ReceiveMessage' +PERM_SendMessage = 'SendMessage' +PERM_FullControl = 'FullControl' + +AllPermissions = [PERM_ReceiveMessage, PERM_SendMessage, PERM_FullControl] + +class SQSQueryConnection(AWSQueryConnection): + + """ + This class uses the Query API (boo!) to SQS to access some of the + new features which have not yet been added to the REST api (yeah!). + """ + + DefaultHost = 'queue.amazonaws.com' + APIVersion = '2007-05-01' + SignatureVersion = '1' + DefaultContentType = 'text/plain' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0, https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + host, debug, https_connection_factory) + + def get_queue_attributes(self, queue_url, attribute='All'): + params = {'Attribute' : attribute} + response = self.make_request('GetQueueAttributes', params, queue_url) + body = response.read() + if response.status == 200: + attrs = Attributes() + h = handler.XmlHandler(attrs, self) + xml.sax.parseString(body, h) + return attrs + else: + raise SQSError(response.status, response.reason, body) + + def set_queue_attribute(self, queue_url, attribute, value): + params = {'Attribute' : attribute, 'Value' : value} + response = self.make_request('SetQueueAttributes', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def change_message_visibility(self, queue_url, message_id, vtimeout): + params = {'MessageId' : message_id, + 'VisibilityTimeout' : vtimeout} + response = self.make_request('ChangeMessageVisibility', params, + queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def add_grant(self, queue_url, permission, email_address=None, user_id=None): + params = {'Permission' : permission} + if user_id: + params['Grantee.ID'] = user_id + if email_address: + params['Grantee.EmailAddress'] = email_address + response = self.make_request('AddGrant', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def remove_grant(self, queue_url, permission, email_address=None, user_id=None): + params = {'Permission' : permission} + if user_id: + params['Grantee.ID'] = user_id + if email_address: + params['Grantee.EmailAddress'] = email_address + response = self.make_request('RemoveGrant', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def list_grants(self, queue_url, permission=None, email_address=None, user_id=None): + params = {} + if user_id: + params['Grantee.ID'] = user_id + if email_address: + params['Grantee.EmailAddress'] = email_address + if permission: + params['Permission'] = permission + response = self.make_request('ListGrants', params, queue_url) + body = response.read() + if response.status == 200: + return body + else: + raise SQSError(response.status, response.reason, body) + + def receive_message(self, queue_url, number_messages=1, + visibility_timeout=None, message_class=Message): + """ + This provides the same functionality as the read and get_messages methods + of the queue object. The only reason this is included here is that there is + currently a bug in SQS that makes it impossible to read a message from a queue + owned by someone else (even if you have been granted appropriate permissions) + via the REST interface. As it turns out, I need to be able to do this so until + the REST interface gets fixed this is the workaround. + """ + params = {'NumberOfMessages' : number_messages} + if visibility_timeout: + params['VisibilityTimeout'] = visibility_timeout + response = self.make_request('ReceiveMessage', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet([('Message', message_class)]) + h = handler.XmlHandler(rs, queue_url) + xml.sax.parseString(body, h) + if len(rs) == 1: + return rs[0] + else: + return rs + else: + raise SQSError(response.status, response.reason, body) + + def delete_message(self, queue_url, message_id): + """ + Because we have to use the Query interface to read messages from queues that + we don't own, we also have to provide a way to delete those messages via Query. + """ + params = {'MessageId' : message_id} + response = self.make_request('DeleteMessage', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + +class SQSConnection(AWSAuthConnection): + + DefaultHost = 'queue.amazonaws.com' + APIVersion = '2007-05-01' + DefaultContentType = 'text/plain' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0, https_connection_factory=None): + AWSAuthConnection.__init__(self, host, + aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, debug, + https_connection_factory) + self.query_conn = None + + def make_request(self, method, path, headers=None, data=''): + # add auth header + if headers == None: + headers = {} + + if not headers.has_key('AWS-Version'): + headers['AWS-Version'] = self.APIVersion + + if not headers.has_key('Content-Type'): + headers['Content-Type'] = self.DefaultContentType + + return AWSAuthConnection.make_request(self, method, path, + headers, data) + + def get_query_connection(self): + if not self.query_conn: + self.query_conn = SQSQueryConnection(self.aws_access_key_id, + self.aws_secret_access_key, + self.is_secure, self.port, + self.proxy, self.proxy_port, + self.server, self.debug, + self.https_connection_factory) + return self.query_conn + + def get_all_queues(self, prefix=''): + if prefix: + path = '/?QueueNamePrefix=%s' % prefix + else: + path = '/' + response = self.make_request('GET', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet([('QueueUrl', Queue)]) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def get_queue(self, queue_name): + i = 0 + rs = self.get_all_queues(queue_name) + for q in rs: + i += 1 + if i != 1: + return None + else: + return q + + def get_queue_attributes(self, queue_url, attribute='All'): + """ + Performs a GetQueueAttributes request and returns an Attributes + instance (subclass of a Dictionary) holding the requested + attribute name/value pairs. + Inputs: + queue_url - the URL of the desired SQS queue + attribute - All|ApproximateNumberOfMessages|VisibilityTimeout + Default value is "All" + Returns: + An Attribute object which is a mapping type holding the + requested name/value pairs + """ + qc = self.get_query_connection() + return qc.get_queue_attributes(queue_url, attribute) + + def set_queue_attribute(self, queue_url, attribute, value): + """ + Performs a SetQueueAttributes request. + Inputs: + queue_url - The URL of the desired SQS queue + attribute - The name of the attribute you want to set. The + only valid value at this time is: VisibilityTimeout + value - The new value for the attribute. + For VisibilityTimeout the value must be an + integer number of seconds from 0 to 86400. + Returns: + Boolean True if successful, otherwise False. + """ + qc = self.get_query_connection() + return qc.set_queue_attribute(queue_url, attribute, value) + + def change_message_visibility(self, queue_url, message_id, vtimeout): + """ + Change the VisibilityTimeout for an individual message. + Inputs: + queue_url - The URL of the desired SQS queue + message_id - The ID of the message whose timeout will be changed + vtimeout - The new VisibilityTimeout value, in seconds + Returns: + Boolean True if successful, otherwise False + Note: This functionality is also available as a method of the + Message object. + """ + qc = self.get_query_connection() + return qc.change_message_visibility(queue_url, message_id, vtimeout) + + def add_grant(self, queue_url, permission, email_address=None, user_id=None): + """ + Add a grant to a queue. + Inputs: + queue_url - The URL of the desired SQS queue + permission - The permission being granted. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + qc = self.get_query_connection() + return qc.add_grant(queue_url, permission, email_address, user_id) + + def remove_grant(self, queue_url, permission, email_address=None, user_id=None): + """ + Remove a grant from a queue. + Inputs: + queue_url - The URL of the desired SQS queue + permission - The permission being removed. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + qc = self.get_query_connection() + return qc.remove_grant(queue_url, permission, email_address, user_id) + + def list_grants(self, queue_url, permission=None, email_address=None, user_id=None): + """ + List the grants to a queue. + Inputs: + queue_url - The URL of the desired SQS queue + permission - The permission granted. One of "ReceiveMessage", "SendMessage" or "FullControl". + If supplied, only grants that allow this permission will be returned. + email_address - the email address of the grantee. If supplied, only grants related to this email + address will be returned + user_id - The ID of the grantee. If supplied, only grants related to his user_id will be returned. + Returns: + A string containing the XML Response elements describing the grants. + """ + qc = self.get_query_connection() + return qc.list_grants(queue_url, permission, email_address, user_id) + + def create_queue(self, queue_name, visibility_timeout=None): + """ + Create a new queue. + Inputs: + queue_name - The name of the new queue + visibility_timeout - (Optional) The default visibility + timeout for the new queue. + Returns: + A new Queue object representing the newly created queue. + """ + path = '/?QueueName=%s' % queue_name + if visibility_timeout: + path = path + '&DefaultVisibilityTimeout=%d' % visibility_timeout + response = self.make_request('POST', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + q = Queue(self) + h = handler.XmlHandler(q, self) + xml.sax.parseString(body, h) + return q + + def delete_queue(self, queue, force_deletion=False): + """ + Delete an SQS Queue. + Inputs: + queue - a Queue object representing the SQS queue to be deleted. + force_deletion - (Optional) Normally, SQS will not delete a + queue that contains messages. However, if + the force_deletion argument is True, the + queue will be deleted regardless of whether + there are messages in the queue or not. + USE WITH CAUTION. This will delete all + messages in the queue as well. + Returns: + An empty ResultSet object. Not sure why, actually. It + should probably return a Boolean indicating success or + failure. + """ + method = 'DELETE' + path = queue.id + if force_deletion: + path = path + '?ForceDeletion=true' + response = self.make_request(method, path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + diff --git a/storefront/boto/sqs/20070501/message.py b/storefront/boto/sqs/20070501/message.py new file mode 100644 index 0000000..0c45b31 --- /dev/null +++ b/storefront/boto/sqs/20070501/message.py @@ -0,0 +1,180 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Message +""" + +import base64 +import StringIO + +class RawMessage: + """ + Base class for SQS messages. RawMessage does not encode the message + in any way. Whatever you store in the body of the message is what + will be written to SQS and whatever is returned from SQS is stored + directly into the body of the message. + """ + + def __init__(self, queue=None, body=''): + self.queue = queue + self._body = '' + self.set_body(body) + self.id = None + + def __len__(self): + return len(self._body) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'MessageBody': + self.set_body(value) + elif name == 'MessageId': + self.id = value + else: + setattr(self, name, value) + + def set_body(self, body): + """ + Set the body of the message. You should always call this method + rather than setting the attribute directly. + """ + self._body = body + + def get_body(self): + """ + Retrieve the body of the message. + """ + return self._body + + def get_body_encoded(self): + """ + This method is really a semi-private method used by the Queue.write + method when writing the contents of the message to SQS. The + RawMessage class does not encode the message in any way so this + just calls get_body(). You probably shouldn't need to call this + method in the normal course of events. + """ + return self.get_body() + + def change_visibility(self, vtimeout): + """ + Convenience function to allow you to directly change the + invisibility timeout for an individual message that has been + read from an SQS queue. This won't affect the default visibility + timeout of the queue. + """ + return self.queue.connection.change_message_visibility(self.queue.id, + self.id, + vtimeout) +class Message(RawMessage): + """ + The default Message class used for SQS queues. This class automatically + encodes/decodes the message body using Base64 encoding to avoid any + illegal characters in the message body. See: + + http://developer.amazonwebservices.com/connect/thread.jspa?messageID=49680%EC%88%90 + + for details on why this is a good idea. The encode/decode is meant to + be transparent to the end-user. + """ + + def endElement(self, name, value, connection): + if name == 'MessageBody': + # Decode the message body returned from SQS using base64 + self.set_body(base64.b64decode(value)) + elif name == 'MessageId': + self.id = value + else: + setattr(self, name, value) + + def get_body_encoded(self): + """ + Because the Message class encodes the message body in base64 + this private method used by queue.write needs to perform the + encoding. + """ + return base64.b64encode(self.get_body()) + +class MHMessage(Message): + """ + The MHMessage class provides a message that provides RFC821-like + headers like this: + + HeaderName: HeaderValue + + The encoding/decoding of this is handled automatically and after + the message body has been read, the message instance can be treated + like a mapping object, i.e. m['HeaderName'] would return 'HeaderValue'. + """ + + def __init__(self, queue=None, body='', xml_attrs=None): + self._dict = {} + Message.__init__(self, queue, body) + + def set_body(self, body): + fp = StringIO.StringIO(body) + line = fp.readline() + while line: + delim = line.find(':') + key = line[0:delim] + value = line[delim+1:].strip() + self._dict[key.strip()] = value.strip() + line = fp.readline() + + def get_body(self): + s = '' + for key,value in self._dict.items(): + s = s + '%s: %s\n' % (key, value) + return s + + def __len__(self): + return len(self.get_body()) + + def __getitem__(self, key): + if self._dict.has_key(key): + return self._dict[key] + else: + raise KeyError(key) + + def __setitem__(self, key, value): + self._dict[key] = value + + def keys(self): + return self._dict.keys() + + def values(self): + return self._dict.values() + + def items(self): + return self._dict.items() + + def has_key(self, key): + return self._dict.has_key(key) + + def update(self, d): + return self._dict.update(d) + + def get(self, key, default=None): + return self._dict.get(key, default) + diff --git a/storefront/boto/sqs/20070501/queue.py b/storefront/boto/sqs/20070501/queue.py new file mode 100644 index 0000000..64289ef --- /dev/null +++ b/storefront/boto/sqs/20070501/queue.py @@ -0,0 +1,343 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Queue +""" + +import xml.sax +import urlparse +from boto.exception import SQSError +from boto.handler import XmlHandler +from boto.sqs.message import Message +from boto.resultset import ResultSet + +class Queue: + + def __init__(self, connection=None, url=None, message_class=Message): + self.connection = connection + self.url = url + self.message_class = message_class + self.visibility_timeout = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'QueueUrl': + self.url = value + if value: + self.id = urlparse.urlparse(value)[2] + elif name == 'VisibilityTimeout': + self.visibility_timeout = int(value) + else: + setattr(self, name, value) + + def set_message_class(self, message_class): + """ + Set the message class that should be used when instantiating messages read + from the queue. By default, the class boto.sqs.message.Message is used but + this can be overriden with any class that behaves like a message. + Inputs: + message_class - The new message class + Returns: + Nothing + """ + self.message_class = message_class + + def get_attributes(self, attributes='All'): + """ + Retrieves attributes about this queue object and returns + them in an Attribute instance (subclass of a Dictionary). + Inputs: + attributes - A string containing + All|ApproximateNumberOfMessages|VisibilityTimeout + Default value is "All" + Returns: + An Attribute object which is a mapping type holding the + requested name/value pairs + """ + return self.connection.get_queue_attributes(self.id, attributes) + + def set_attribute(self, attribute, value): + """ + Set a new value for an attribute of the Queue. + Inputs: + attribute - The name of the attribute you want to set. The + only valid value at this time is: VisibilityTimeout + value - The new value for the attribute. + For VisibilityTimeout the value must be an + integer number of seconds from 0 to 86400. + Returns: + Boolean True if successful, otherwise False. + """ + return self.connection.set_queue_attribute(self.id, attribute, value) + + def add_grant(self, permission, email_address=None, user_id=None): + """ + Add a grant to this queue. + Inputs: + permission - The permission being granted. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + return self.connection.add_grant(self.id, permission, email_address, user_id) + + def remove_grant(self, permission, email_address=None, user_id=None): + """ + Remove a grant from this queue. + Inputs: + permission - The permission being removed. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + return self.connection.remove_grant(self.id, permission, email_address, user_id) + + def list_grants(self, permission=None, email_address=None, user_id=None): + """ + List the grants to this queue. + Inputs: + permission - The permission granted. One of "ReceiveMessage", "SendMessage" or "FullControl". + If supplied, only grants that allow this permission will be returned. + email_address - the email address of the grantee. If supplied, only grants related to this email + address will be returned + user_id - The ID of the grantee. If supplied, only grants related to his user_id will be returned. + Returns: + A string containing the XML Response elements describing the grants. + """ + return self.connection.list_grants(self.id, permission, email_address, user_id) + + def get_timeout(self): + """ + Get the visibility timeout for the queue. + Inputs: + None + Returns: + The number of seconds as an integer. + """ + a = self.get_attributes('VisibilityTimeout') + return int(a['VisibilityTimeout']) + + def set_timeout(self, visibility_timeout): + """ + Set the visibility timeout for the queue. + Inputs: + visibility_timeout - The desired timeout in seconds + Returns: + Nothing + """ + retval = self.set_attribute('VisibilityTimeout', visibility_timeout) + if retval: + self.visibility_timeout = visibility_timeout + return retval + + def read(self, visibility_timeout=None): + """ + Read a single message from the queue. + Inputs: + visibility_timeout - The timeout for this message in seconds + Returns: + A single message or None if queue is empty + """ + rs = self.get_messages(1, visibility_timeout) + if len(rs) == 1: + return rs[0] + else: + return None + + def write(self, message): + """ + Add a single message to the queue. + Inputs: + message - The message to be written to the queue + Returns: + None + """ + path = '%s/back' % self.id + message.queue = self + response = self.connection.make_request('PUT', path, None, + message.get_body_encoded()) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + handler = XmlHandler(message, self.connection) + xml.sax.parseString(body, handler) + return None + + def new_message(self, body=''): + return self.message_class(self, body) + + # get a variable number of messages, returns a list of messages + def get_messages(self, num_messages=1, visibility_timeout=None): + path = '%s/front?NumberOfMessages=%d' % (self.id, num_messages) + if visibility_timeout: + path = '%s&VisibilityTimeout=%d' % (path, visibility_timeout) + response = self.connection.make_request('GET', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet([('Message', self.message_class)]) + h = XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def delete_message(self, message): + path = '%s/%s' % (self.id, message.id) + response = self.connection.make_request('DELETE', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet() + h = XmlHandler(rs, self.connection) + xml.sax.parseString(body, h) + return rs + + def clear(self, page_size=100, vtimeout=10): + """Utility function to remove all messages from a queue""" + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + self.delete_message(m) + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def count(self, page_size=100, vtimeout=10): + """ + Utility function to count the number of messages in a queue. + Note: This function now calls GetQueueAttributes to obtain + an 'approximate' count of the number of messages in a queue. + """ + a = self.get_attributes('ApproximateNumberOfMessages') + return int(a['ApproximateNumberOfMessages']) + + def count_slow(self, page_size=100, vtimeout=10): + """ + Deprecated. This is the old 'count' method that actually counts + the messages by reading them all. This gives an accurate count but + is very slow for queues with non-trivial number of messasges. + Instead, use get_attribute('ApproximateNumberOfMessages') to take + advantage of the new SQS capability. This is retained only for + the unit tests. + """ + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def dump(self, file_name, page_size=100, vtimeout=10, sep='\n'): + """Utility function to dump the messages in a queue to a file""" + fp = open(file_name, 'wb') + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + fp.write(m.get_body()) + if sep: + fp.write(sep) + n += 1 + l = self.get_messages(page_size, vtimeout) + fp.close() + return n + + def save(self, file_name, sep='\n'): + """ + Read all messages from the queue and persist them to local file. + Messages are written to the file and the 'sep' string is written + in between messages. Messages are deleted from the queue after + being written to the file. + Returns the number of messages saved. + """ + fp = open(file_name, 'wb') + n = 0 + m = self.read() + while m: + n += 1 + fp.write(m.get_body()) + if sep: + fp.write(sep) + self.delete_message(m) + m = self.read() + fp.close() + return n + + def save_to_s3(self, bucket): + """ + Read all messages from the queue and persist them to S3. + Messages are stored in the S3 bucket using a naming scheme of: + / + Messages are deleted from the queue after being saved to S3. + Returns the number of messages saved. + """ + n = 0 + m = self.read() + while m: + n += 1 + key = bucket.new_key('%s/%s' % (self.id, m.id)) + key.set_contents_from_string(m.get_body()) + self.delete_message(m) + m = self.read() + return n + + def load_from_s3(self, bucket, prefix=None): + """ + Load messages previously saved to S3. + """ + n = 0 + if prefix: + prefix = '%s/' % prefix + else: + prefix = '%s/' % self.id + rs = bucket.list(prefix=prefix) + for key in rs: + n += 1 + m = self.new_message(key.get_contents_as_string()) + self.write(m) + return n + + def load(self, file_name, sep='\n'): + """Utility function to load messages from a file to a queue""" + fp = open(file_name, 'rb') + n = 0 + body = '' + l = fp.readline() + while l: + if l == sep: + m = Message(self, body) + self.write(m) + n += 1 + print 'writing message %d' % n + body = '' + else: + body = body + l + l = fp.readline() + fp.close() + return n + + diff --git a/storefront/boto/sqs/20070501/readme.txt b/storefront/boto/sqs/20070501/readme.txt new file mode 100644 index 0000000..7132579 --- /dev/null +++ b/storefront/boto/sqs/20070501/readme.txt @@ -0,0 +1,6 @@ +The main SQS implementation now uses the 2008-01-01 API verson. To use the older API version +(2007-05-01) you need to edit your /etc/boto.cfg or ~/.boto file to add the following line: + +boto.sqs_extend = 20070501 + +This will allow the code in the boto.sqs.20070501 module to override the code in boto.sqs. diff --git a/storefront/boto/sqs/__init__.py b/storefront/boto/sqs/__init__.py new file mode 100644 index 0000000..0b3924c --- /dev/null +++ b/storefront/boto/sqs/__init__.py @@ -0,0 +1,46 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto + +boto.check_extensions(__name__, __path__) + +from queue import Queue +from message import Message, MHMessage, EncodedMHMessage +from regioninfo import SQSRegionInfo + +def regions(): + """ + Get all available regions for the SQS service. + + :rtype: list + :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` + """ + return [SQSRegionInfo(name='us-east-1', endpoint='queue.amazonaws.com'), + SQSRegionInfo(name='eu-west-1', endpoint='eu-west-1.queue.amazonaws.com'), + SQSRegionInfo(name='us-west-1', endpoint='us-west-1.queue.amazonaws.com')] + +def connect_to_region(region_name): + for region in regions(): + if region.name == region_name: + return region.connect() + return None diff --git a/storefront/boto/sqs/attributes.py b/storefront/boto/sqs/attributes.py new file mode 100644 index 0000000..26c7204 --- /dev/null +++ b/storefront/boto/sqs/attributes.py @@ -0,0 +1,46 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Attribute Name/Value set +""" + +class Attributes(dict): + + def __init__(self, parent): + self.parent = parent + self.current_key = None + self.current_value = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'Attribute': + self[self.current_key] = self.current_value + elif name == 'Name': + self.current_key = value + elif name == 'Value': + self.current_value = value + else: + setattr(self, name, value) + + diff --git a/storefront/boto/sqs/connection.py b/storefront/boto/sqs/connection.py new file mode 100644 index 0000000..fd13d2a --- /dev/null +++ b/storefront/boto/sqs/connection.py @@ -0,0 +1,257 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.connection import AWSQueryConnection +import xml.sax +from boto.sqs.regioninfo import SQSRegionInfo +from boto.sqs.queue import Queue +from boto.sqs.message import Message +from boto.sqs.attributes import Attributes +from boto import handler +from boto.resultset import ResultSet +from boto.exception import SQSError + +class SQSConnection(AWSQueryConnection): + """ + A Connection to the SQS Service. + """ + DefaultRegionName = 'us-east-1' + DefaultRegionEndpoint = 'queue.amazonaws.com' + APIVersion = '2009-02-01' + SignatureVersion = '2' + DefaultContentType = 'text/plain' + ResponseError = SQSError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/'): + if not region: + region = SQSRegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + self.region.endpoint, debug, https_connection_factory, path) + + def create_queue(self, queue_name, visibility_timeout=None): + """ + Create an SQS Queue. + + :type queue_name: str or unicode + :param queue_name: The name of the new queue. Names are scoped to an account and need to + be unique within that account. Calling this method on an existing + queue name will not return an error from SQS unless the value for + visibility_timeout is different than the value of the existing queue + of that name. This is still an expensive operation, though, and not + the preferred way to check for the existence of a queue. See the + :func:`boto.sqs.connection.SQSConnection.lookup` method. + + :type visibility_timeout: int + :param visibility_timeout: The default visibility timeout for all messages written in the + queue. This can be overridden on a per-message. + + :rtype: :class:`boto.sqs.queue.Queue` + :return: The newly created queue. + + """ + params = {'QueueName': queue_name} + if visibility_timeout: + params['DefaultVisibilityTimeout'] = '%d' % (visibility_timeout,) + return self.get_object('CreateQueue', params, Queue) + + def delete_queue(self, queue, force_deletion=False): + """ + Delete an SQS Queue. + + :type queue: A Queue object + :param queue: The SQS queue to be deleted + + :type force_deletion: Boolean + :param force_deletion: Normally, SQS will not delete a queue that contains messages. + However, if the force_deletion argument is True, the + queue will be deleted regardless of whether there are messages in + the queue or not. USE WITH CAUTION. This will delete all + messages in the queue as well. + + :rtype: bool + :return: True if the command succeeded, False otherwise + """ + return self.get_status('DeleteQueue', None, queue.id) + + def get_queue_attributes(self, queue, attribute='All'): + """ + Gets one or all attributes of a Queue + + :type queue: A Queue object + :param queue: The SQS queue to be deleted + + :type attribute: str + :type attribute: The specific attribute requested. If not supplied, the default + is to return all attributes. Valid attributes are: + ApproximateNumberOfMessages, + ApproximateNumberOfMessagesNotVisible, + VisibilityTimeout, + CreatedTimestamp, + LastModifiedTimestamp, + Policy + + :rtype: :class:`boto.sqs.attributes.Attributes` + :return: An Attributes object containing request value(s). + """ + params = {'AttributeName' : attribute} + return self.get_object('GetQueueAttributes', params, Attributes, queue.id) + + def set_queue_attribute(self, queue, attribute, value): + params = {'Attribute.Name' : attribute, 'Attribute.Value' : value} + return self.get_status('SetQueueAttributes', params, queue.id) + + def receive_message(self, queue, number_messages=1, visibility_timeout=None, + attributes=None): + """ + Read messages from an SQS Queue. + + :type queue: A Queue object + :param queue: The Queue from which messages are read. + + :type number_messages: int + :param number_messages: The maximum number of messages to read (default=1) + + :type visibility_timeout: int + :param visibility_timeout: The number of seconds the message should remain invisible + to other queue readers (default=None which uses the Queues default) + + :type attributes: list of strings + :param attributes: A list of additional attributes that will be returned + with the response. Valid values: + All + SenderId + SentTimestamp + ApproximateReceiveCount + ApproximateFirstReceiveTimestamp + + """ + params = {'MaxNumberOfMessages' : number_messages} + if visibility_timeout: + params['VisibilityTimeout'] = visibility_timeout + if attributes: + self.build_list_params(self, params, attributes, 'AttributeName') + return self.get_list('ReceiveMessage', params, [('Message', queue.message_class)], + queue.id, queue) + + def delete_message(self, queue, message): + params = {'ReceiptHandle' : message.receipt_handle} + return self.get_status('DeleteMessage', params, queue.id) + + def send_message(self, queue, message_content): + params = {'MessageBody' : message_content} + return self.get_object('SendMessage', params, Message, queue.id, verb='POST') + + def change_message_visibility(self, queue, receipt_handle, visibility_timeout): + """ + Extends the read lock timeout for the specified message from the specified queue + to the specified value. + + :type queue: A :class:`boto.sqs.queue.Queue` object + :param queue: The Queue from which messages are read. + + :type receipt_handle: str + :param queue: The receipt handle associated with the message whose + visibility timeout will be changed. + + :type visibility_timeout: int + :param visibility_timeout: The new value of the message's visibility timeout + in seconds. + """ + params = {'ReceiptHandle' : receipt_handle, + 'VisibilityTimeout' : visibility_timeout} + return self.get_status('ChangeMessageVisibility', params, queue.id) + + def get_all_queues(self, prefix=''): + params = {} + if prefix: + params['QueueNamePrefix'] = prefix + return self.get_list('ListQueues', params, [('QueueUrl', Queue)]) + + def get_queue(self, queue_name): + rs = self.get_all_queues(queue_name) + for q in rs: + if q.url.endswith(queue_name): + return q + return None + + lookup = get_queue + + # + # Permissions methods + # + + def add_permission(self, queue, label, aws_account_id, action_name): + """ + Add a permission to a queue. + + :type queue: :class:`boto.sqs.queue.Queue` + :param queue: The queue object + + :type label: str or unicode + :param label: A unique identification of the permission you are setting. + Maximum of 80 characters ``[0-9a-zA-Z_-]`` + Example, AliceSendMessage + + :type aws_account_id: str or unicode + :param principal_id: The AWS account number of the principal who will be given + permission. The principal must have an AWS account, but + does not need to be signed up for Amazon SQS. For information + about locating the AWS account identification. + + :type action_name: str or unicode + :param action_name: The action. Valid choices are: + \*|SendMessage|ReceiveMessage|DeleteMessage| + ChangeMessageVisibility|GetQueueAttributes + + :rtype: bool + :return: True if successful, False otherwise. + + """ + params = {'Label': label, + 'AWSAccountId' : aws_account_id, + 'ActionName' : action_name} + return self.get_status('AddPermission', params, queue.id) + + def remove_permission(self, queue, label): + """ + Remove a permission from a queue. + + :type queue: :class:`boto.sqs.queue.Queue` + :param queue: The queue object + + :type label: str or unicode + :param label: The unique label associated with the permission being removed. + + :rtype: bool + :return: True if successful, False otherwise. + """ + params = {'Label': label} + return self.get_status('RemovePermission', params, queue.id) + + + + + diff --git a/storefront/boto/sqs/jsonmessage.py b/storefront/boto/sqs/jsonmessage.py new file mode 100644 index 0000000..ab05a60 --- /dev/null +++ b/storefront/boto/sqs/jsonmessage.py @@ -0,0 +1,42 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.sqs.message import MHMessage +from boto.exception import SQSDecodeError +import base64 +import simplejson + +class JSONMessage(MHMessage): + """ + Acts like a dictionary but encodes it's data as a Base64 encoded JSON payload. + """ + + def decode(self, value): + try: + value = base64.b64decode(value) + value = simplejson.loads(value) + except: + raise SQSDecodeError('Unable to decode message', self) + return value + + def encode(self, value): + value = simplejson.dumps(value) + return base64.b64encode(value) diff --git a/storefront/boto/sqs/message.py b/storefront/boto/sqs/message.py new file mode 100644 index 0000000..da1ba68 --- /dev/null +++ b/storefront/boto/sqs/message.py @@ -0,0 +1,251 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +SQS Message + +A Message represents the data stored in an SQS queue. The rules for what is allowed within an SQS +Message are here: + + http://docs.amazonwebservices.com/AWSSimpleQueueService/2008-01-01/SQSDeveloperGuide/Query_QuerySendMessage.html + +So, at it's simplest level a Message just needs to allow a developer to store bytes in it and get the bytes +back out. However, to allow messages to have richer semantics, the Message class must support the +following interfaces: + +The constructor for the Message class must accept a keyword parameter "queue" which is an instance of a +boto Queue object and represents the queue that the message will be stored in. The default value for +this parameter is None. + +The constructor for the Message class must accept a keyword parameter "body" which represents the +content or body of the message. The format of this parameter will depend on the behavior of the +particular Message subclass. For example, if the Message subclass provides dictionary-like behavior to the +user the body passed to the constructor should be a dict-like object that can be used to populate +the initial state of the message. + +The Message class must provide an encode method that accepts a value of the same type as the body +parameter of the constructor and returns a string of characters that are able to be stored in an +SQS message body (see rules above). + +The Message class must provide a decode method that accepts a string of characters that can be +stored (and probably were stored!) in an SQS message and return an object of a type that is consistent +with the "body" parameter accepted on the class constructor. + +The Message class must provide a __len__ method that will return the size of the encoded message +that would be stored in SQS based on the current state of the Message object. + +The Message class must provide a get_body method that will return the body of the message in the +same format accepted in the constructor of the class. + +The Message class must provide a set_body method that accepts a message body in the same format +accepted by the constructor of the class. This method should alter to the internal state of the +Message object to reflect the state represented in the message body parameter. + +The Message class must provide a get_body_encoded method that returns the current body of the message +in the format in which it would be stored in SQS. +""" + +import base64 +import StringIO +from boto.sqs.attributes import Attributes +from boto.exception import SQSDecodeError + +class RawMessage: + """ + Base class for SQS messages. RawMessage does not encode the message + in any way. Whatever you store in the body of the message is what + will be written to SQS and whatever is returned from SQS is stored + directly into the body of the message. + """ + + def __init__(self, queue=None, body=''): + self.queue = queue + self.set_body(body) + self.id = None + self.receipt_handle = None + self.md5 = None + self.attributes = Attributes(self) + + def __len__(self): + return len(self.encode(self._body)) + + def startElement(self, name, attrs, connection): + if name == 'Attribute': + return self.attributes + return None + + def endElement(self, name, value, connection): + if name == 'Body': + self.set_body(self.decode(value)) + elif name == 'MessageId': + self.id = value + elif name == 'ReceiptHandle': + self.receipt_handle = value + elif name == 'MD5OfMessageBody': + self.md5 = value + else: + setattr(self, name, value) + + def encode(self, value): + """Transform body object into serialized byte array format.""" + return value + + def decode(self, value): + """Transform seralized byte array into any object.""" + return value + + def set_body(self, body): + """Override the current body for this object, using decoded format.""" + self._body = body + + def get_body(self): + return self._body + + def get_body_encoded(self): + """ + This method is really a semi-private method used by the Queue.write + method when writing the contents of the message to SQS. + You probably shouldn't need to call this method in the normal course of events. + """ + return self.encode(self.get_body()) + + def delete(self): + if self.queue: + return self.queue.delete_message(self) + + def change_visibility(self, visibility_timeout): + if self.queue: + self.queue.connection.change_message_visibility(self.queue, + self.receipt_handle, + visibility_timeout) + +class Message(RawMessage): + """ + The default Message class used for SQS queues. This class automatically + encodes/decodes the message body using Base64 encoding to avoid any + illegal characters in the message body. See: + + http://developer.amazonwebservices.com/connect/thread.jspa?messageID=49680%EC%88%90 + + for details on why this is a good idea. The encode/decode is meant to + be transparent to the end-user. + """ + + def encode(self, value): + return base64.b64encode(value) + + def decode(self, value): + try: + value = base64.b64decode(value) + except: + raise SQSDecodeError('Unable to decode message', self) + return value + +class MHMessage(Message): + """ + The MHMessage class provides a message that provides RFC821-like + headers like this: + + HeaderName: HeaderValue + + The encoding/decoding of this is handled automatically and after + the message body has been read, the message instance can be treated + like a mapping object, i.e. m['HeaderName'] would return 'HeaderValue'. + """ + + def __init__(self, queue=None, body=None, xml_attrs=None): + if body == None or body == '': + body = {} + Message.__init__(self, queue, body) + + def decode(self, value): + try: + msg = {} + fp = StringIO.StringIO(value) + line = fp.readline() + while line: + delim = line.find(':') + key = line[0:delim] + value = line[delim+1:].strip() + msg[key.strip()] = value.strip() + line = fp.readline() + except: + raise SQSDecodeError('Unable to decode message', self) + return msg + + def encode(self, value): + s = '' + for item in value.items(): + s = s + '%s: %s\n' % (item[0], item[1]) + return s + + def __getitem__(self, key): + if self._body.has_key(key): + return self._body[key] + else: + raise KeyError(key) + + def __setitem__(self, key, value): + self._body[key] = value + self.set_body(self._body) + + def keys(self): + return self._body.keys() + + def values(self): + return self._body.values() + + def items(self): + return self._body.items() + + def has_key(self, key): + return self._body.has_key(key) + + def update(self, d): + self._body.update(d) + self.set_body(self._body) + + def get(self, key, default=None): + return self._body.get(key, default) + +class EncodedMHMessage(MHMessage): + """ + The EncodedMHMessage class provides a message that provides RFC821-like + headers like this: + + HeaderName: HeaderValue + + This variation encodes/decodes the body of the message in base64 automatically. + The message instance can be treated like a mapping object, + i.e. m['HeaderName'] would return 'HeaderValue'. + """ + + def decode(self, value): + try: + value = base64.b64decode(value) + except: + raise SQSDecodeError('Unable to decode message', self) + return MHMessage.decode(self, value) + + def encode(self, value): + value = MHMessage.encode(value) + return base64.b64encode(self, value) + diff --git a/storefront/boto/sqs/queue.py b/storefront/boto/sqs/queue.py new file mode 100644 index 0000000..48b6115 --- /dev/null +++ b/storefront/boto/sqs/queue.py @@ -0,0 +1,415 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Queue +""" + +import xml.sax +import urlparse +from boto.exception import SQSError +from boto.handler import XmlHandler +from boto.sqs.message import Message +from boto.resultset import ResultSet + +class Queue: + + def __init__(self, connection=None, url=None, message_class=Message): + self.connection = connection + self.url = url + self.message_class = message_class + self.visibility_timeout = None + + def _id(self): + if self.url: + val = urlparse.urlparse(self.url)[2] + else: + val = self.url + return val + id = property(_id) + + def _name(self): + if self.url: + val = urlparse.urlparse(self.url)[2].split('/')[2] + else: + val = self.url + return val + name = property(_name) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'QueueUrl': + self.url = value + elif name == 'VisibilityTimeout': + self.visibility_timeout = int(value) + else: + setattr(self, name, value) + + def set_message_class(self, message_class): + """ + Set the message class that should be used when instantiating messages read + from the queue. By default, the class boto.sqs.message.Message is used but + this can be overriden with any class that behaves like a message. + + :type message_class: Message-like class + :param message_class: The new Message class + """ + self.message_class = message_class + + def get_attributes(self, attributes='All'): + """ + Retrieves attributes about this queue object and returns + them in an Attribute instance (subclass of a Dictionary). + + :type attributes: string + :param attributes: String containing one of: + ApproximateNumberOfMessages, + ApproximateNumberOfMessagesNotVisible, + VisibilityTimeout, + CreatedTimestamp, + LastModifiedTimestamp, + Policy + :rtype: Attribute object + :return: An Attribute object which is a mapping type holding the + requested name/value pairs + """ + return self.connection.get_queue_attributes(self, attributes) + + def set_attribute(self, attribute, value): + """ + Set a new value for an attribute of the Queue. + + :type attribute: String + :param attribute: The name of the attribute you want to set. The + only valid value at this time is: VisibilityTimeout + :type value: int + :param value: The new value for the attribute. + For VisibilityTimeout the value must be an + integer number of seconds from 0 to 86400. + + :rtype: bool + :return: True if successful, otherwise False. + """ + return self.connection.set_queue_attribute(self, attribute, value) + + def get_timeout(self): + """ + Get the visibility timeout for the queue. + + :rtype: int + :return: The number of seconds as an integer. + """ + a = self.get_attributes('VisibilityTimeout') + return int(a['VisibilityTimeout']) + + def set_timeout(self, visibility_timeout): + """ + Set the visibility timeout for the queue. + + :type visibility_timeout: int + :param visibility_timeout: The desired timeout in seconds + """ + retval = self.set_attribute('VisibilityTimeout', visibility_timeout) + if retval: + self.visibility_timeout = visibility_timeout + return retval + + def add_permission(self, label, aws_account_id, action_name): + """ + Add a permission to a queue. + + :type label: str or unicode + :param label: A unique identification of the permission you are setting. + Maximum of 80 characters ``[0-9a-zA-Z_-]`` + Example, AliceSendMessage + + :type aws_account_id: str or unicode + :param principal_id: The AWS account number of the principal who will be given + permission. The principal must have an AWS account, but + does not need to be signed up for Amazon SQS. For information + about locating the AWS account identification. + + :type action_name: str or unicode + :param action_name: The action. Valid choices are: + \*|SendMessage|ReceiveMessage|DeleteMessage| + ChangeMessageVisibility|GetQueueAttributes + + :rtype: bool + :return: True if successful, False otherwise. + + """ + return self.connection.add_permission(self, label, aws_account_id, action_name) + + def remove_permission(self, label): + """ + Remove a permission from a queue. + + :type label: str or unicode + :param label: The unique label associated with the permission being removed. + + :rtype: bool + :return: True if successful, False otherwise. + """ + return self.connection.remove_permission(self, label) + + def read(self, visibility_timeout=None): + """ + Read a single message from the queue. + + :type visibility_timeout: int + :param visibility_timeout: The timeout for this message in seconds + + :rtype: :class:`boto.sqs.message.Message` + :return: A single message or None if queue is empty + """ + rs = self.get_messages(1, visibility_timeout) + if len(rs) == 1: + return rs[0] + else: + return None + + def write(self, message): + """ + Add a single message to the queue. + + :type message: Message + :param message: The message to be written to the queue + + :rtype: :class:`boto.sqs.message.Message` + :return: The :class:`boto.sqs.message.Message` object that was written. + """ + new_msg = self.connection.send_message(self, message.get_body_encoded()) + message.id = new_msg.id + message.md5 = new_msg.md5 + return message + + def new_message(self, body=''): + """ + Create new message of appropriate class. + + :type body: message body + :param body: The body of the newly created message (optional). + + :rtype: :class:`boto.sqs.message.Message` + :return: A new Message object + """ + m = self.message_class(self, body) + m.queue = self + return m + + # get a variable number of messages, returns a list of messages + def get_messages(self, num_messages=1, visibility_timeout=None, + attributes=None): + """ + Get a variable number of messages. + + :type num_messages: int + :param num_messages: The maximum number of messages to read from the queue. + + :type visibility_timeout: int + :param visibility_timeout: The VisibilityTimeout for the messages read. + + :type attributes: list of strings + :param attributes: A list of additional attributes that will be returned + with the response. Valid values: + All + SenderId + SentTimestamp + ApproximateReceiveCount + ApproximateFirstReceiveTimestamp + :rtype: list + :return: A list of :class:`boto.sqs.message.Message` objects. + """ + return self.connection.receive_message(self, number_messages=num_messages, + visibility_timeout=visibility_timeout, + attributes=attributes) + + def delete_message(self, message): + """ + Delete a message from the queue. + + :type message: :class:`boto.sqs.message.Message` + :param message: The :class:`boto.sqs.message.Message` object to delete. + + :rtype: bool + :return: True if successful, False otherwise + """ + return self.connection.delete_message(self, message) + + def delete(self): + """ + Delete the queue. + """ + return self.connection.delete_queue(self) + + def clear(self, page_size=10, vtimeout=10): + """Utility function to remove all messages from a queue""" + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + self.delete_message(m) + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def count(self, page_size=10, vtimeout=10): + """ + Utility function to count the number of messages in a queue. + Note: This function now calls GetQueueAttributes to obtain + an 'approximate' count of the number of messages in a queue. + """ + a = self.get_attributes('ApproximateNumberOfMessages') + return int(a['ApproximateNumberOfMessages']) + + def count_slow(self, page_size=10, vtimeout=10): + """ + Deprecated. This is the old 'count' method that actually counts + the messages by reading them all. This gives an accurate count but + is very slow for queues with non-trivial number of messasges. + Instead, use get_attribute('ApproximateNumberOfMessages') to take + advantage of the new SQS capability. This is retained only for + the unit tests. + """ + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def dump_(self, file_name, page_size=10, vtimeout=10, sep='\n'): + """Utility function to dump the messages in a queue to a file + NOTE: Page size must be < 10 else SQS errors""" + fp = open(file_name, 'wb') + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + fp.write(m.get_body()) + if sep: + fp.write(sep) + n += 1 + l = self.get_messages(page_size, vtimeout) + fp.close() + return n + + def save_to_file(self, fp, sep='\n'): + """ + Read all messages from the queue and persist them to file-like object. + Messages are written to the file and the 'sep' string is written + in between messages. Messages are deleted from the queue after + being written to the file. + Returns the number of messages saved. + """ + n = 0 + m = self.read() + while m: + n += 1 + fp.write(m.get_body()) + if sep: + fp.write(sep) + self.delete_message(m) + m = self.read() + return n + + def save_to_filename(self, file_name, sep='\n'): + """ + Read all messages from the queue and persist them to local file. + Messages are written to the file and the 'sep' string is written + in between messages. Messages are deleted from the queue after + being written to the file. + Returns the number of messages saved. + """ + fp = open(file_name, 'wb') + n = self.save_to_file(fp, sep) + fp.close() + return n + + # for backwards compatibility + save = save_to_filename + + def save_to_s3(self, bucket): + """ + Read all messages from the queue and persist them to S3. + Messages are stored in the S3 bucket using a naming scheme of:: + + / + + Messages are deleted from the queue after being saved to S3. + Returns the number of messages saved. + """ + n = 0 + m = self.read() + while m: + n += 1 + key = bucket.new_key('%s/%s' % (self.id, m.id)) + key.set_contents_from_string(m.get_body()) + self.delete_message(m) + m = self.read() + return n + + def load_from_s3(self, bucket, prefix=None): + """ + Load messages previously saved to S3. + """ + n = 0 + if prefix: + prefix = '%s/' % prefix + else: + prefix = '%s/' % self.id[1:] + rs = bucket.list(prefix=prefix) + for key in rs: + n += 1 + m = self.new_message(key.get_contents_as_string()) + self.write(m) + return n + + def load_from_file(self, fp, sep='\n'): + """Utility function to load messages from a file-like object to a queue""" + n = 0 + body = '' + l = fp.readline() + while l: + if l == sep: + m = Message(self, body) + self.write(m) + n += 1 + print 'writing message %d' % n + body = '' + else: + body = body + l + l = fp.readline() + return n + + def load_from_filename(self, file_name, sep='\n'): + """Utility function to load messages from a local filename to a queue""" + fp = open(file_name, 'rb') + n = self.load_file_file(fp, sep) + fp.close() + return n + + # for backward compatibility + load = load_from_filename + diff --git a/storefront/boto/sqs/regioninfo.py b/storefront/boto/sqs/regioninfo.py new file mode 100644 index 0000000..1d13a40 --- /dev/null +++ b/storefront/boto/sqs/regioninfo.py @@ -0,0 +1,40 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from boto.ec2.regioninfo import RegionInfo + +class SQSRegionInfo(RegionInfo): + + def connect(self, **kw_params): + """ + Connect to this Region's endpoint. Returns an SQSConnection + object pointing to the endpoint associated with this region. + You may pass any of the arguments accepted by the SQSConnection + object's constructor as keyword arguments and they will be + passed along to the SQSConnection object. + + :rtype: :class:`boto.sqs.connection.SQSConnection` + :return: The connection to this regions endpoint + """ + from boto.sqs.connection import SQSConnection + return SQSConnection(region=self, **kw_params) + diff --git a/storefront/boto/tests/__init__.py b/storefront/boto/tests/__init__.py new file mode 100644 index 0000000..449bd16 --- /dev/null +++ b/storefront/boto/tests/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/storefront/boto/tests/devpay_s3.py b/storefront/boto/tests/devpay_s3.py new file mode 100644 index 0000000..bb91125 --- /dev/null +++ b/storefront/boto/tests/devpay_s3.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the S3Connection +""" + +import time +import os +import urllib + +from boto.s3.connection import S3Connection +from boto.exception import S3PermissionsError + +# this test requires a devpay product and user token to run: + +AMAZON_USER_TOKEN = '{UserToken}...your token here...' +DEVPAY_HEADERS = { 'x-amz-security-token': AMAZON_USER_TOKEN } + +print '--- running S3Connection tests (DevPay) ---' +c = S3Connection() +# create a new, empty bucket +bucket_name = 'test-%d' % int(time.time()) +bucket = c.create_bucket(bucket_name, headers=DEVPAY_HEADERS) +# now try a get_bucket call and see if it's really there +bucket = c.get_bucket(bucket_name, headers=DEVPAY_HEADERS) +# test logging +logging_bucket = c.create_bucket(bucket_name + '-log', headers=DEVPAY_HEADERS) +logging_bucket.set_as_logging_target(headers=DEVPAY_HEADERS) +bucket.enable_logging(target_bucket=logging_bucket, target_prefix=bucket.name, headers=DEVPAY_HEADERS) +bucket.disable_logging(headers=DEVPAY_HEADERS) +c.delete_bucket(logging_bucket, headers=DEVPAY_HEADERS) +# create a new key and store it's content from a string +k = bucket.new_key() +k.name = 'foobar' +s1 = 'This is a test of file upload and download' +s2 = 'This is a second string to test file upload and download' +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +fp = open('foobar', 'wb') +# now get the contents from s3 to a local file +k.get_contents_to_file(fp, headers=DEVPAY_HEADERS) +fp.close() +fp = open('foobar') +# check to make sure content read from s3 is identical to original +assert s1 == fp.read(), 'corrupted file' +fp.close() +# test generated URLs +url = k.generate_url(3600, headers=DEVPAY_HEADERS) +file = urllib.urlopen(url) +assert s1 == file.read(), 'invalid URL %s' % url +url = k.generate_url(3600, force_http=True, headers=DEVPAY_HEADERS) +file = urllib.urlopen(url) +assert s1 == file.read(), 'invalid URL %s' % url +bucket.delete_key(k, headers=DEVPAY_HEADERS) +# test a few variations on get_all_keys - first load some data +# for the first one, let's override the content type +phony_mimetype = 'application/x-boto-test' +headers = {'Content-Type': phony_mimetype} +headers.update(DEVPAY_HEADERS) +k.name = 'foo/bar' +k.set_contents_from_string(s1, headers) +k.name = 'foo/bas' +k.set_contents_from_filename('foobar', headers=DEVPAY_HEADERS) +k.name = 'foo/bat' +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +k.name = 'fie/bar' +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +k.name = 'fie/bas' +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +k.name = 'fie/bat' +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +# try resetting the contents to another value +md5 = k.md5 +k.set_contents_from_string(s2, headers=DEVPAY_HEADERS) +assert k.md5 != md5 +os.unlink('foobar') +all = bucket.get_all_keys(headers=DEVPAY_HEADERS) +assert len(all) == 6 +rs = bucket.get_all_keys(prefix='foo', headers=DEVPAY_HEADERS) +assert len(rs) == 3 +rs = bucket.get_all_keys(prefix='', delimiter='/', headers=DEVPAY_HEADERS) +assert len(rs) == 2 +rs = bucket.get_all_keys(maxkeys=5, headers=DEVPAY_HEADERS) +assert len(rs) == 5 +# test the lookup method +k = bucket.lookup('foo/bar', headers=DEVPAY_HEADERS) +assert isinstance(k, bucket.key_class) +assert k.content_type == phony_mimetype +k = bucket.lookup('notthere', headers=DEVPAY_HEADERS) +assert k == None +# try some metadata stuff +k = bucket.new_key() +k.name = 'has_metadata' +mdkey1 = 'meta1' +mdval1 = 'This is the first metadata value' +k.set_metadata(mdkey1, mdval1) +mdkey2 = 'meta2' +mdval2 = 'This is the second metadata value' +k.set_metadata(mdkey2, mdval2) +k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) +k = bucket.lookup('has_metadata', headers=DEVPAY_HEADERS) +assert k.get_metadata(mdkey1) == mdval1 +assert k.get_metadata(mdkey2) == mdval2 +k = bucket.new_key() +k.name = 'has_metadata' +k.get_contents_as_string(headers=DEVPAY_HEADERS) +assert k.get_metadata(mdkey1) == mdval1 +assert k.get_metadata(mdkey2) == mdval2 +bucket.delete_key(k, headers=DEVPAY_HEADERS) +# test list and iterator +rs1 = bucket.list(headers=DEVPAY_HEADERS) +num_iter = 0 +for r in rs1: + num_iter = num_iter + 1 +rs = bucket.get_all_keys(headers=DEVPAY_HEADERS) +num_keys = len(rs) +assert num_iter == num_keys +# try a key with a funny character +k = bucket.new_key() +k.name = 'testnewline\n' +k.set_contents_from_string('This is a test', headers=DEVPAY_HEADERS) +rs = bucket.get_all_keys(headers=DEVPAY_HEADERS) +assert len(rs) == num_keys + 1 +bucket.delete_key(k, headers=DEVPAY_HEADERS) +rs = bucket.get_all_keys(headers=DEVPAY_HEADERS) +assert len(rs) == num_keys +# try some acl stuff +bucket.set_acl('public-read', headers=DEVPAY_HEADERS) +policy = bucket.get_acl(headers=DEVPAY_HEADERS) +assert len(policy.acl.grants) == 2 +bucket.set_acl('private', headers=DEVPAY_HEADERS) +policy = bucket.get_acl(headers=DEVPAY_HEADERS) +assert len(policy.acl.grants) == 1 +k = bucket.lookup('foo/bar', headers=DEVPAY_HEADERS) +k.set_acl('public-read', headers=DEVPAY_HEADERS) +policy = k.get_acl(headers=DEVPAY_HEADERS) +assert len(policy.acl.grants) == 2 +k.set_acl('private', headers=DEVPAY_HEADERS) +policy = k.get_acl(headers=DEVPAY_HEADERS) +assert len(policy.acl.grants) == 1 +# try the convenience methods for grants +# this doesn't work with devpay +#bucket.add_user_grant('FULL_CONTROL', +# 'c1e724fbfa0979a4448393c59a8c055011f739b6d102fb37a65f26414653cd67', +# headers=DEVPAY_HEADERS) +try: + bucket.add_email_grant('foobar', 'foo@bar.com', headers=DEVPAY_HEADERS) +except S3PermissionsError: + pass +# now delete all keys in bucket +for k in all: + bucket.delete_key(k, headers=DEVPAY_HEADERS) +# now delete bucket + +c.delete_bucket(bucket, headers=DEVPAY_HEADERS) + +print '--- tests completed ---' diff --git a/storefront/boto/tests/test.py b/storefront/boto/tests/test.py new file mode 100755 index 0000000..c6175ca --- /dev/null +++ b/storefront/boto/tests/test.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +do the unit tests! +""" + +import sys, os, unittest +import getopt, sys +import boto + +from boto.tests.test_sqsconnection import SQSConnectionTest +from boto.tests.test_s3connection import S3ConnectionTest +from boto.tests.test_ec2connection import EC2ConnectionTest +from boto.tests.test_sdbconnection import SDBConnectionTest + +def usage(): + print 'test.py [-t testsuite] [-v verbosity]' + print ' -t run specific testsuite (s3|sqs|ec2|sdb|all)' + print ' -v verbosity (0|1|2)' + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'ht:v:', + ['help', 'testsuite', 'verbosity']) + except: + usage() + sys.exit(2) + testsuite = 'all' + verbosity = 1 + for o, a in opts: + if o in ('-h', '--help'): + usage() + sys.exit() + if o in ('-t', '--testsuite'): + testsuite = a + if o in ('-v', '--verbosity'): + verbosity = int(a) + if len(args) != 0: + usage() + sys.exit() + suite = unittest.TestSuite() + if testsuite == 'all': + suite.addTest(unittest.makeSuite(SQSConnectionTest)) + suite.addTest(unittest.makeSuite(S3ConnectionTest)) + suite.addTest(unittest.makeSuite(EC2ConnectionTest)) + suite.addTest(unittest.makeSuite(SDBConnectionTest)) + elif testsuite == 's3': + suite.addTest(unittest.makeSuite(S3ConnectionTest)) + elif testsuite == 'sqs': + suite.addTest(unittest.makeSuite(SQSConnectionTest)) + elif testsuite == 'ec2': + suite.addTest(unittest.makeSuite(EC2ConnectionTest)) + elif testsuite == 'sdb': + suite.addTest(unittest.makeSuite(SDBConnectionTest)) + else: + usage() + sys.exit() + unittest.TextTestRunner(verbosity=verbosity).run(suite) + +if __name__ == "__main__": + main() diff --git a/storefront/boto/tests/test_ec2connection.py b/storefront/boto/tests/test_ec2connection.py new file mode 100644 index 0000000..8f1fb59 --- /dev/null +++ b/storefront/boto/tests/test_ec2connection.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the EC2Connection +""" + +import unittest +import time +import os +from boto.ec2.connection import EC2Connection +import telnetlib +import socket + +class EC2ConnectionTest (unittest.TestCase): + + def test_1_basic(self): + # this is my user_id, if you want to run these tests you should + # replace this with yours or they won't work + user_id = '084307701560' + print '--- running EC2Connection tests ---' + c = EC2Connection() + # get list of private AMI's + rs = c.get_all_images(owners=[user_id]) + assert len(rs) > 0 + # now pick the first one + image = rs[0] + # temporarily make this image runnable by everyone + status = image.set_launch_permissions(group_names=['all']) + assert status + d = image.get_launch_permissions() + assert d.has_key('groups') + assert len(d['groups']) > 0 + # now remove that permission + status = image.remove_launch_permissions(group_names=['all']) + assert status + d = image.get_launch_permissions() + assert not d.has_key('groups') + + # create a new security group + group_name = 'test-%d' % int(time.time()) + group_desc = 'This is a security group created during unit testing' + group = c.create_security_group(group_name, group_desc) + # now get a listing of all security groups and look for our new one + rs = c.get_all_security_groups() + found = False + for g in rs: + if g.name == group_name: + found = True + assert found + # now pass arg to filter results to only our new group + rs = c.get_all_security_groups([group_name]) + assert len(rs) == 1 + group = rs[0] + # + # now delete the security group + status = c.delete_security_group(group_name) + # now make sure it's really gone + rs = c.get_all_security_groups() + found = False + for g in rs: + if g.name == group_name: + found = True + assert not found + # now create it again for use with the instance test + group = c.create_security_group(group_name, group_desc) + + # now try to launch apache image with our new security group + rs = c.get_all_images() + img_loc = 'ec2-public-images/fedora-core4-apache.manifest.xml' + for image in rs: + if image.location == img_loc: + break + reservation = image.run(security_groups=[group.name]) + instance = reservation.instances[0] + while instance.state != 'running': + print '\tinstance is %s' % instance.state + time.sleep(30) + instance.update() + # instance in now running, try to telnet to port 80 + t = telnetlib.Telnet() + try: + t.open(instance.dns_name, 80) + except socket.error: + pass + # now open up port 80 and try again, it should work + group.authorize('tcp', 80, 80, '0.0.0.0/0') + t.open(instance.dns_name, 80) + t.close() + # now revoke authorization and try again + group.revoke('tcp', 80, 80, '0.0.0.0/0') + try: + t.open(instance.dns_name, 80) + except socket.error: + pass + # now kill the instance and delete the security group + instance.stop() + # unfortunately, I can't delete the sg within this script + #sg.delete() + + # create a new key pair + key_name = 'test-%d' % int(time.time()) + status = c.create_key_pair(key_name) + assert status + # now get a listing of all key pairs and look for our new one + rs = c.get_all_key_pairs() + found = False + for k in rs: + if k.name == key_name: + found = True + assert found + # now pass arg to filter results to only our new key pair + rs = c.get_all_key_pairs([key_name]) + assert len(rs) == 1 + key_pair = rs[0] + # now delete the key pair + status = c.delete_key_pair(key_name) + # now make sure it's really gone + rs = c.get_all_key_pairs() + found = False + for k in rs: + if k.name == key_name: + found = True + assert not found + + # short test around Paid AMI capability + demo_paid_ami_id = 'ami-bd9d78d4' + demo_paid_ami_product_code = 'A79EC0DB' + l = c.get_all_images([demo_paid_ami_id]) + assert len(l) == 1 + assert len(l[0].product_codes) == 1 + assert l[0].product_codes[0] == demo_paid_ami_product_code + + print '--- tests completed ---' diff --git a/storefront/boto/tests/test_s3connection.py b/storefront/boto/tests/test_s3connection.py new file mode 100644 index 0000000..7afc8d2 --- /dev/null +++ b/storefront/boto/tests/test_s3connection.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the S3Connection +""" + +import unittest +import time +import os +import urllib +from boto.s3.connection import S3Connection +from boto.exception import S3PermissionsError + +class S3ConnectionTest (unittest.TestCase): + + def test_1_basic(self): + print '--- running S3Connection tests ---' + c = S3Connection() + # create a new, empty bucket + bucket_name = 'test-%d' % int(time.time()) + bucket = c.create_bucket(bucket_name) + # now try a get_bucket call and see if it's really there + bucket = c.get_bucket(bucket_name) + # test logging + logging_bucket = c.create_bucket(bucket_name + '-log') + logging_bucket.set_as_logging_target() + bucket.enable_logging(target_bucket=logging_bucket, target_prefix=bucket.name) + bucket.disable_logging() + c.delete_bucket(logging_bucket) + k = bucket.new_key() + k.name = 'foobar' + s1 = 'This is a test of file upload and download' + s2 = 'This is a second string to test file upload and download' + k.set_contents_from_string(s1) + fp = open('foobar', 'wb') + # now get the contents from s3 to a local file + k.get_contents_to_file(fp) + fp.close() + fp = open('foobar') + # check to make sure content read from s3 is identical to original + assert s1 == fp.read(), 'corrupted file' + fp.close() + # test generated URLs + url = k.generate_url(3600) + file = urllib.urlopen(url) + assert s1 == file.read(), 'invalid URL %s' % url + url = k.generate_url(3600, force_http=True) + file = urllib.urlopen(url) + assert s1 == file.read(), 'invalid URL %s' % url + bucket.delete_key(k) + # test a few variations on get_all_keys - first load some data + # for the first one, let's override the content type + phony_mimetype = 'application/x-boto-test' + headers = {'Content-Type': phony_mimetype} + k.name = 'foo/bar' + k.set_contents_from_string(s1, headers) + k.name = 'foo/bas' + k.set_contents_from_filename('foobar') + k.name = 'foo/bat' + k.set_contents_from_string(s1) + k.name = 'fie/bar' + k.set_contents_from_string(s1) + k.name = 'fie/bas' + k.set_contents_from_string(s1) + k.name = 'fie/bat' + k.set_contents_from_string(s1) + # try resetting the contents to another value + md5 = k.md5 + k.set_contents_from_string(s2) + assert k.md5 != md5 + os.unlink('foobar') + all = bucket.get_all_keys() + assert len(all) == 6 + rs = bucket.get_all_keys(prefix='foo') + assert len(rs) == 3 + rs = bucket.get_all_keys(prefix='', delimiter='/') + assert len(rs) == 2 + rs = bucket.get_all_keys(maxkeys=5) + assert len(rs) == 5 + # test the lookup method + k = bucket.lookup('foo/bar') + assert isinstance(k, bucket.key_class) + assert k.content_type == phony_mimetype + k = bucket.lookup('notthere') + assert k == None + # try some metadata stuff + k = bucket.new_key() + k.name = 'has_metadata' + mdkey1 = 'meta1' + mdval1 = 'This is the first metadata value' + k.set_metadata(mdkey1, mdval1) + mdkey2 = 'meta2' + mdval2 = 'This is the second metadata value' + k.set_metadata(mdkey2, mdval2) + k.set_contents_from_string(s1) + k = bucket.lookup('has_metadata') + assert k.get_metadata(mdkey1) == mdval1 + assert k.get_metadata(mdkey2) == mdval2 + k = bucket.new_key() + k.name = 'has_metadata' + k.get_contents_as_string() + assert k.get_metadata(mdkey1) == mdval1 + assert k.get_metadata(mdkey2) == mdval2 + bucket.delete_key(k) + # test list and iterator + rs1 = bucket.list() + num_iter = 0 + for r in rs1: + num_iter = num_iter + 1 + rs = bucket.get_all_keys() + num_keys = len(rs) + assert num_iter == num_keys + # try a key with a funny character + k = bucket.new_key() + k.name = 'testnewline\n' + k.set_contents_from_string('This is a test') + rs = bucket.get_all_keys() + assert len(rs) == num_keys + 1 + bucket.delete_key(k) + rs = bucket.get_all_keys() + assert len(rs) == num_keys + # try some acl stuff + bucket.set_acl('public-read') + policy = bucket.get_acl() + assert len(policy.acl.grants) == 2 + bucket.set_acl('private') + policy = bucket.get_acl() + assert len(policy.acl.grants) == 1 + k = bucket.lookup('foo/bar') + k.set_acl('public-read') + policy = k.get_acl() + assert len(policy.acl.grants) == 2 + k.set_acl('private') + policy = k.get_acl() + assert len(policy.acl.grants) == 1 + # try the convenience methods for grants + bucket.add_user_grant('FULL_CONTROL', + 'c1e724fbfa0979a4448393c59a8c055011f739b6d102fb37a65f26414653cd67') + try: + bucket.add_email_grant('foobar', 'foo@bar.com') + except S3PermissionsError: + pass + # now delete all keys in bucket + for k in all: + bucket.delete_key(k) + # now delete bucket + c.delete_bucket(bucket) + print '--- tests completed ---' diff --git a/storefront/boto/tests/test_sdbconnection.py b/storefront/boto/tests/test_sdbconnection.py new file mode 100644 index 0000000..c2bb74e --- /dev/null +++ b/storefront/boto/tests/test_sdbconnection.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the SDBConnection +""" + +import unittest +import time +from boto.sdb.connection import SDBConnection +from boto.exception import SDBResponseError + +class SDBConnectionTest (unittest.TestCase): + + def test_1_basic(self): + print '--- running SDBConnection tests ---' + c = SDBConnection() + rs = c.get_all_domains() + num_domains = len(rs) + + # try illegal name + try: + domain = c.create_domain('bad:domain:name') + except SDBResponseError: + pass + + # now create one that should work and should be unique (i.e. a new one) + domain_name = 'test%d' % int(time.time()) + domain = c.create_domain(domain_name) + rs = c.get_all_domains() + assert len(rs) == num_domains+1 + + # now let's a couple of items and attributes + item_1 = 'item1' + same_value = 'same_value' + attrs_1 = {'name1' : same_value, 'name2' : 'diff_value_1'} + domain.put_attributes(item_1, attrs_1) + item_2 = 'item2' + attrs_2 = {'name1' : same_value, 'name2' : 'diff_value_2'} + domain.put_attributes(item_2, attrs_2) + time.sleep(10) + + # try to get the attributes and see if they match + item = domain.get_attributes(item_1) + assert len(item.keys()) == len(attrs_1.keys()) + assert item['name1'] == attrs_1['name1'] + assert item['name2'] == attrs_1['name2'] + + # try a search or two + rs = domain.query("['name1'='%s']" % same_value) + n = 0 + for item in rs: + n += 1 + assert n == 2 + rs = domain.query("['name2'='diff_value_2']") + n = 0 + for item in rs: + n += 1 + assert n == 1 + + # delete all attributes associated with item_1 + stat = domain.delete_attributes(item_1) + assert stat + + # now try a batch put operation on the domain + item3 = {'name3_1' : 'value3_1', + 'name3_2' : 'value3_2', + 'name3_3' : ['value3_3_1', 'value3_3_2']} + + item4 = {'name4_1' : 'value4_1', + 'name4_2' : ['value4_2_1', 'value4_2_2'], + 'name4_3' : 'value4_3'} + items = {'item3' : item3, 'item4' : item4} + domain.batch_put_attributes(items) + time.sleep(10) + item = domain.get_attributes('item3') + assert item['name3_2'] == 'value3_2' + + # now delete the domain + stat = c.delete_domain(domain) + assert stat + + print '--- tests completed ---' + diff --git a/storefront/boto/tests/test_sqsconnection.py b/storefront/boto/tests/test_sqsconnection.py new file mode 100644 index 0000000..f24ad32 --- /dev/null +++ b/storefront/boto/tests/test_sqsconnection.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the SQSConnection +""" + +import unittest +import time +from boto.sqs.connection import SQSConnection +from boto.sqs.message import MHMessage +from boto.exception import SQSError + +class SQSConnectionTest (unittest.TestCase): + + def test_1_basic(self): + print '--- running SQSConnection tests ---' + c = SQSConnection() + rs = c.get_all_queues() + num_queues = 0 + for q in rs: + num_queues += 1 + + # try illegal name + try: + queue = c.create_queue('bad_queue_name') + except SQSError: + pass + + # now create one that should work and should be unique (i.e. a new one) + queue_name = 'test%d' % int(time.time()) + timeout = 60 + queue = c.create_queue(queue_name, timeout) + time.sleep(60) + rs = c.get_all_queues() + i = 0 + for q in rs: + i += 1 + assert i == num_queues+1 + assert queue.count_slow() == 0 + + # check the visibility timeout + t = queue.get_timeout() + assert t == timeout, '%d != %d' % (t, timeout) + + # now try to get queue attributes + a = q.get_attributes() + assert a.has_key('ApproximateNumberOfMessages') + assert a.has_key('VisibilityTimeout') + a = q.get_attributes('ApproximateNumberOfMessages') + assert a.has_key('ApproximateNumberOfMessages') + assert not a.has_key('VisibilityTimeout') + a = q.get_attributes('VisibilityTimeout') + assert not a.has_key('ApproximateNumberOfMessages') + assert a.has_key('VisibilityTimeout') + + # now change the visibility timeout + timeout = 45 + queue.set_timeout(timeout) + time.sleep(60) + t = queue.get_timeout() + assert t == timeout, '%d != %d' % (t, timeout) + + # now add a message + message_body = 'This is a test\n' + message = queue.new_message(message_body) + queue.write(message) + time.sleep(30) + assert queue.count_slow() == 1 + time.sleep(30) + + # now read the message from the queue with a 10 second timeout + message = queue.read(visibility_timeout=10) + assert message + assert message.get_body() == message_body + + # now immediately try another read, shouldn't find anything + message = queue.read() + assert message == None + + # now wait 10 seconds and try again + time.sleep(10) + message = queue.read() + assert message + + if c.APIVersion == '2007-05-01': + # now terminate the visibility timeout for this message + message.change_visibility(0) + # now see if we can read it in the queue + message = queue.read() + assert message + + # now delete the message + queue.delete_message(message) + time.sleep(30) + assert queue.count_slow() == 0 + + # create another queue so we can test force deletion + # we will also test MHMessage with this queue + queue_name = 'test%d' % int(time.time()) + timeout = 60 + queue = c.create_queue(queue_name, timeout) + queue.set_message_class(MHMessage) + time.sleep(30) + + # now add a couple of messages + message = queue.new_message() + message['foo'] = 'bar' + queue.write(message) + message_body = {'fie' : 'baz', 'foo' : 'bar'} + message = queue.new_message(body=message_body) + queue.write(message) + time.sleep(30) + + m = queue.read() + assert m['foo'] == 'bar' + + # now delete that queue and messages + c.delete_queue(queue, True) + + print '--- tests completed ---' + diff --git a/storefront/boto/utils.py b/storefront/boto/utils.py new file mode 100644 index 0000000..db16d30 --- /dev/null +++ b/storefront/boto/utils.py @@ -0,0 +1,560 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# +# Parts of this code were copied or derived from sample code supplied by AWS. +# The following notice applies to that code. +# +# This software code is made available "AS IS" without warranties of any +# kind. You may copy, display, modify and redistribute the software +# code either by itself or as incorporated into your code; provided that +# you do not remove any proprietary notices. Your use of this software +# code is at your own risk and you waive any claim against Amazon +# Digital Services, Inc. or its affiliates with respect to your use of +# this software code. (c) 2006 Amazon Digital Services, Inc. or its +# affiliates. + +""" +Some handy utility functions used by several classes. +""" + +import base64 +import hmac +import re +import urllib, urllib2 +import imp +import subprocess, os, StringIO +import time, datetime +import logging.handlers +import boto +import tempfile +import smtplib +import datetime +from email.MIMEMultipart import MIMEMultipart +from email.MIMEBase import MIMEBase +from email.MIMEText import MIMEText +from email.Utils import formatdate +from email import Encoders + +try: + import hashlib + _hashfn = hashlib.sha512 +except ImportError: + import md5 + _hashfn = md5.md5 + +METADATA_PREFIX = 'x-amz-meta-' +AMAZON_HEADER_PREFIX = 'x-amz-' + +# generates the aws canonical string for the given parameters +def canonical_string(method, path, headers, expires=None): + interesting_headers = {} + for key in headers: + lk = key.lower() + if lk in ['content-md5', 'content-type', 'date'] or lk.startswith(AMAZON_HEADER_PREFIX): + interesting_headers[lk] = headers[key].strip() + + # these keys get empty strings if they don't exist + if not interesting_headers.has_key('content-type'): + interesting_headers['content-type'] = '' + if not interesting_headers.has_key('content-md5'): + interesting_headers['content-md5'] = '' + + # just in case someone used this. it's not necessary in this lib. + if interesting_headers.has_key('x-amz-date'): + interesting_headers['date'] = '' + + # if you're using expires for query string auth, then it trumps date + # (and x-amz-date) + if expires: + interesting_headers['date'] = str(expires) + + sorted_header_keys = interesting_headers.keys() + sorted_header_keys.sort() + + buf = "%s\n" % method + for key in sorted_header_keys: + if key.startswith(AMAZON_HEADER_PREFIX): + buf += "%s:%s\n" % (key, interesting_headers[key]) + else: + buf += "%s\n" % interesting_headers[key] + + # don't include anything after the first ? in the resource... + buf += "%s" % path.split('?')[0] + + # ...unless there is an acl or torrent parameter + if re.search("[&?]acl($|=|&)", path): + buf += "?acl" + elif re.search("[&?]logging($|=|&)", path): + buf += "?logging" + elif re.search("[&?]torrent($|=|&)", path): + buf += "?torrent" + elif re.search("[&?]location($|=|&)", path): + buf += "?location" + elif re.search("[&?]requestPayment($|=|&)", path): + buf += "?requestPayment" + + return buf + +def merge_meta(headers, metadata): + final_headers = headers.copy() + for k in metadata.keys(): + if k.lower() in ['cache-control', 'content-md5', 'content-type', + 'content-encoding', 'content-disposition', + 'date', 'expires']: + final_headers[k] = metadata[k] + else: + final_headers[METADATA_PREFIX + k] = metadata[k] + + return final_headers + +def get_aws_metadata(headers): + metadata = {} + for hkey in headers.keys(): + if hkey.lower().startswith(METADATA_PREFIX): + metadata[hkey[len(METADATA_PREFIX):]] = headers[hkey] + del headers[hkey] + return metadata + +def retry_url(url, retry_on_404=True): + for i in range(0, 10): + try: + req = urllib2.Request(url) + resp = urllib2.urlopen(req) + return resp.read() + except urllib2.HTTPError, e: + # in 2.6 you use getcode(), in 2.5 and earlier you use code + if hasattr(e, 'getcode'): + code = e.getcode() + else: + code = e.code + if code == 404 and not retry_on_404: + return '' + except: + pass + boto.log.exception('Caught exception reading instance data') + time.sleep(2**i) + boto.log.error('Unable to read instance data, giving up') + return '' + +def _get_instance_metadata(url): + d = {} + data = retry_url(url) + if data: + fields = data.split('\n') + for field in fields: + if field.endswith('/'): + d[field[0:-1]] = _get_instance_metadata(url + field) + else: + p = field.find('=') + if p > 0: + key = field[p+1:] + resource = field[0:p] + '/openssh-key' + else: + key = resource = field + val = retry_url(url + resource) + p = val.find('\n') + if p > 0: + val = val.split('\n') + d[key] = val + return d + +def get_instance_metadata(version='latest'): + """ + Returns the instance metadata as a nested Python dictionary. + Simple values (e.g. local_hostname, hostname, etc.) will be + stored as string values. Values such as ancestor-ami-ids will + be stored in the dict as a list of string values. More complex + fields such as public-keys and will be stored as nested dicts. + """ + url = 'http://169.254.169.254/%s/meta-data/' % version + return _get_instance_metadata(url) + +def get_instance_userdata(version='latest', sep=None): + url = 'http://169.254.169.254/%s/user-data' % version + user_data = retry_url(url, retry_on_404=False) + if user_data: + if sep: + l = user_data.split(sep) + user_data = {} + for nvpair in l: + t = nvpair.split('=') + user_data[t[0].strip()] = t[1].strip() + return user_data + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +def get_ts(ts=None): + if not ts: + ts = time.gmtime() + return time.strftime(ISO8601, ts) + +def parse_ts(ts): + return datetime.datetime.strptime(ts, ISO8601) + +def find_class(module_name, class_name=None): + if class_name: + module_name = "%s.%s" % (module_name, class_name) + modules = module_name.split('.') + path = None + c = None + + try: + for m in modules[1:]: + if c: + c = getattr(c, m) + else: + c = getattr(__import__(".".join(modules[0:-1])), m) + return c + except: + return None + +def update_dme(username, password, dme_id, ip_address): + """ + Update your Dynamic DNS record with DNSMadeEasy.com + """ + dme_url = 'https://www.dnsmadeeasy.com/servlet/updateip' + dme_url += '?username=%s&password=%s&id=%s&ip=%s' + s = urllib2.urlopen(dme_url % (username, password, dme_id, ip_address)) + return s.read() + +def fetch_file(uri, file=None, username=None, password=None): + """ + Fetch a file based on the URI provided. If you do not pass in a file pointer + a tempfile.NamedTemporaryFile, or None if the file could not be + retrieved is returned. + The URI can be either an HTTP url, or "s3://bucket_name/key_name" + """ + boto.log.info('Fetching %s' % uri) + if file == None: + file = tempfile.NamedTemporaryFile() + try: + working_dir = boto.config.get("General", "working_dir") + if uri.startswith('s3://'): + bucket_name, key_name = uri[len('s3://'):].split('/', 1) + c = boto.connect_s3() + bucket = c.get_bucket(bucket_name) + key = bucket.get_key(key_name) + key.get_contents_to_file(file) + else: + if username and password: + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + passman.add_password(None, uri, username, password) + authhandler = urllib2.HTTPBasicAuthHandler(passman) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) + s = urllib2.urlopen(uri) + file.write(s.read()) + file.seek(0) + except: + raise + boto.log.exception('Problem Retrieving file: %s' % uri) + file = None + return file + +class ShellCommand(object): + + def __init__(self, command, wait=True): + self.exit_code = 0 + self.command = command + self.log_fp = StringIO.StringIO() + self.wait = wait + self.run() + + def run(self): + boto.log.info('running:%s' % self.command) + self.process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if(self.wait): + while self.process.poll() == None: + time.sleep(1) + t = self.process.communicate() + self.log_fp.write(t[0]) + self.log_fp.write(t[1]) + boto.log.info(self.log_fp.getvalue()) + self.exit_code = self.process.returncode + return self.exit_code + + def setReadOnly(self, value): + raise AttributeError + + def getStatus(self): + return self.exit_code + + status = property(getStatus, setReadOnly, None, 'The exit code for the command') + + def getOutput(self): + return self.log_fp.getvalue() + + output = property(getOutput, setReadOnly, None, 'The STDIN and STDERR output of the command') + +class AuthSMTPHandler(logging.handlers.SMTPHandler): + """ + This class extends the SMTPHandler in the standard Python logging module + to accept a username and password on the constructor and to then use those + credentials to authenticate with the SMTP server. To use this, you could + add something like this in your boto config file: + + [handler_hand07] + class=boto.utils.AuthSMTPHandler + level=WARN + formatter=form07 + args=('localhost', 'username', 'password', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject') + """ + + def __init__(self, mailhost, username, password, fromaddr, toaddrs, subject): + """ + Initialize the handler. + + We have extended the constructor to accept a username/password + for SMTP authentication. + """ + logging.handlers.SMTPHandler.__init__(self, mailhost, fromaddr, toaddrs, subject) + self.username = username + self.password = password + + def emit(self, record): + """ + Emit a record. + + Format the record and send it to the specified addressees. + It would be really nice if I could add authorization to this class + without having to resort to cut and paste inheritance but, no. + """ + try: + import smtplib + try: + from email.Utils import formatdate + except: + formatdate = self.date_time + port = self.mailport + if not port: + port = smtplib.SMTP_PORT + smtp = smtplib.SMTP(self.mailhost, port) + smtp.login(self.username, self.password) + msg = self.format(record) + msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % ( + self.fromaddr, + string.join(self.toaddrs, ","), + self.getSubject(record), + formatdate(), msg) + smtp.sendmail(self.fromaddr, self.toaddrs, msg) + smtp.quit() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handleError(record) + +class LRUCache(dict): + """A dictionary-like object that stores only a certain number of items, and + discards its least recently used item when full. + + >>> cache = LRUCache(3) + >>> cache['A'] = 0 + >>> cache['B'] = 1 + >>> cache['C'] = 2 + >>> len(cache) + 3 + + >>> cache['A'] + 0 + + Adding new items to the cache does not increase its size. Instead, the least + recently used item is dropped: + + >>> cache['D'] = 3 + >>> len(cache) + 3 + >>> 'B' in cache + False + + Iterating over the cache returns the keys, starting with the most recently + used: + + >>> for key in cache: + ... print key + D + A + C + + This code is based on the LRUCache class from Genshi which is based on + Mighty's LRUCache from ``myghtyutils.util``, written + by Mike Bayer and released under the MIT license (Genshi uses the + BSD License). See: + + http://svn.myghty.org/myghtyutils/trunk/lib/myghtyutils/util.py + """ + + class _Item(object): + def __init__(self, key, value): + self.previous = self.next = None + self.key = key + self.value = value + def __repr__(self): + return repr(self.value) + + def __init__(self, capacity): + self._dict = dict() + self.capacity = capacity + self.head = None + self.tail = None + + def __contains__(self, key): + return key in self._dict + + def __iter__(self): + cur = self.head + while cur: + yield cur.key + cur = cur.next + + def __len__(self): + return len(self._dict) + + def __getitem__(self, key): + item = self._dict[key] + self._update_item(item) + return item.value + + def __setitem__(self, key, value): + item = self._dict.get(key) + if item is None: + item = self._Item(key, value) + self._dict[key] = item + self._insert_item(item) + else: + item.value = value + self._update_item(item) + self._manage_size() + + def __repr__(self): + return repr(self._dict) + + def _insert_item(self, item): + item.previous = None + item.next = self.head + if self.head is not None: + self.head.previous = item + else: + self.tail = item + self.head = item + self._manage_size() + + def _manage_size(self): + while len(self._dict) > self.capacity: + olditem = self._dict[self.tail.key] + del self._dict[self.tail.key] + if self.tail != self.head: + self.tail = self.tail.previous + self.tail.next = None + else: + self.head = self.tail = None + + def _update_item(self, item): + if self.head == item: + return + + previous = item.previous + previous.next = item.next + if item.next is not None: + item.next.previous = previous + else: + self.tail = previous + + item.previous = None + item.next = self.head + self.head.previous = self.head = item + +class Password(object): + """ + Password object that stores itself as SHA512 hashed. + """ + def __init__(self, str=None): + """ + Load the string from an initial value, this should be the raw SHA512 hashed password + """ + self.str = str + + def set(self, value): + self.str = _hashfn(value).hexdigest() + + def __str__(self): + return str(self.str) + + def __eq__(self, other): + if other == None: + return False + return str(_hashfn(other).hexdigest()) == str(self.str) + + def __len__(self): + if self.str: + return len(self.str) + else: + return 0 + +def notify(subject, body=None, html_body=None, to_string=None, attachments=[], append_instance_id=True): + if append_instance_id: + subject = "[%s] %s" % (boto.config.get_value("Instance", "instance-id"), subject) + if not to_string: + to_string = boto.config.get_value('Notification', 'smtp_to', None) + if to_string: + try: + from_string = boto.config.get_value('Notification', 'smtp_from', 'boto') + msg = MIMEMultipart() + msg['From'] = from_string + msg['To'] = to_string + msg['Date'] = formatdate(localtime=True) + msg['Subject'] = subject + + if body: + msg.attach(MIMEText(body)) + + if html_body: + part = MIMEBase('text', 'html') + part.set_payload(html_body) + Encoders.encode_base64(part) + msg.attach(part) + + for part in attachments: + msg.attach(part) + + smtp_host = boto.config.get_value('Notification', 'smtp_host', 'localhost') + + # Alternate port support + if boto.config.get_value("Notification", "smtp_port"): + server = smtplib.SMTP(smtp_host, int(boto.config.get_value("Notification", "smtp_port"))) + else: + server = smtplib.SMTP(smtp_host) + + # TLS support + if boto.config.getbool("Notification", "smtp_tls"): + server.ehlo() + server.starttls() + server.ehlo() + smtp_user = boto.config.get_value('Notification', 'smtp_user', '') + smtp_pass = boto.config.get_value('Notification', 'smtp_pass', '') + if smtp_user: + server.login(smtp_user, smtp_pass) + server.sendmail(from_string, to_string, msg.as_string()) + server.quit() + except: + boto.log.exception('notify failed') + diff --git a/storefront/boto/vpc/__init__.py b/storefront/boto/vpc/__init__.py new file mode 100644 index 0000000..80b0073 --- /dev/null +++ b/storefront/boto/vpc/__init__.py @@ -0,0 +1,478 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a connection to the EC2 service. +""" + +import urllib +import base64 +import boto +from boto import config +from boto.ec2.connection import EC2Connection +from boto.resultset import ResultSet +from boto.vpc.vpc import VPC +from boto.vpc.customergateway import CustomerGateway +from boto.vpc.vpngateway import VpnGateway, Attachment +from boto.vpc.dhcpoptions import DhcpOptions +from boto.vpc.subnet import Subnet +from boto.vpc.vpnconnection import VpnConnection + +class VPCConnection(EC2Connection): + + # VPC methods + + def get_all_vpcs(self, vpc_ids=None, filters=None): + """ + Retrieve information about your VPCs. You can filter results to + return information only about those VPCs that match your search + parameters. Otherwise, all VPCs associated with your account + are returned. + + :type vpc_ids: list + :param vpc_ids: A list of strings with the desired VPC ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the VPC (pending or available) + - *cidrBlock*, CIDR block of the VPC + - *dhcpOptionsId*, the ID of a set of DHCP options + + :rtype: list + :return: A list of :class:`boto.vpc.vpc.VPC` + """ + params = {} + if vpc_ids: + self.build_list_params(params, vpc_ids, 'VpcId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeVpcs', params, [('item', VPC)]) + + def create_vpc(self, cidr_block): + """ + Create a new Virtual Private Cloud. + + :type cidr_block: str + :param cidr_block: A valid CIDR block + + :rtype: The newly created VPC + :return: A :class:`boto.vpc.vpc.VPC` object + """ + params = {'CidrBlock' : cidr_block} + return self.get_object('CreateVpc', params, VPC) + + def delete_vpc(self, vpc_id): + """ + Delete a Virtual Private Cloud. + + :type vpc_id: str + :param vpc_id: The ID of the vpc to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'VpcId': vpc_id} + return self.get_status('DeleteVpc', params) + + # Customer Gateways + + def get_all_customer_gateways(self, customer_gateway_ids=None, filters=None): + """ + Retrieve information about your CustomerGateways. You can filter results to + return information only about those CustomerGateways that match your search + parameters. Otherwise, all CustomerGateways associated with your account + are returned. + + :type customer_gateway_ids: list + :param customer_gateway_ids: A list of strings with the desired CustomerGateway ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the CustomerGateway + (pending,available,deleting,deleted) + - *type*, the type of customer gateway (ipsec.1) + - *ipAddress* the IP address of customer gateway's + internet-routable external inteface + + :rtype: list + :return: A list of :class:`boto.vpc.customergateway.CustomerGateway` + """ + params = {} + if customer_gateway_ids: + self.build_list_params(params, customer_gateway_ids, 'CustomerGatewayId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeCustomerGateways', params, [('item', CustomerGateway)]) + + def create_customer_gateway(self, type, ip_address, bgp_asn): + """ + Create a new Customer Gateway + + :type type: str + :param type: Type of VPN Connection. Only valid valid currently is 'ipsec.1' + + :type ip_address: str + :param ip_address: Internet-routable IP address for customer's gateway. + Must be a static address. + + :type bgp_asn: str + :param bgp_asn: Customer gateway's Border Gateway Protocol (BGP) + Autonomous System Number (ASN) + + :rtype: The newly created CustomerGateway + :return: A :class:`boto.vpc.customergateway.CustomerGateway` object + """ + params = {'Type' : type, + 'IpAddress' : ip_address, + 'BgpAsn' : bgp_asn} + return self.get_object('CreateCustomerGateway', params, CustomerGateway) + + def delete_customer_gateway(self, customer_gateway_id): + """ + Delete a Customer Gateway. + + :type customer_gateway_id: str + :param customer_gateway_id: The ID of the customer_gateway to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'CustomerGatewayId': customer_gateway_id} + return self.get_status('DeleteCustomerGateway', params) + + # VPN Gateways + + def get_all_vpn_gateways(self, vpn_gateway_ids=None, filters=None): + """ + Retrieve information about your VpnGateways. You can filter results to + return information only about those VpnGateways that match your search + parameters. Otherwise, all VpnGateways associated with your account + are returned. + + :type vpn_gateway_ids: list + :param vpn_gateway_ids: A list of strings with the desired VpnGateway ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the VpnGateway + (pending,available,deleting,deleted) + - *type*, the type of customer gateway (ipsec.1) + - *availabilityZone*, the Availability zone the + VPN gateway is in. + + :rtype: list + :return: A list of :class:`boto.vpc.customergateway.VpnGateway` + """ + params = {} + if vpn_gateway_ids: + self.build_list_params(params, vpn_gateway_ids, 'VpnGatewayId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeVpnGateways', params, [('item', VpnGateway)]) + + def create_vpn_gateway(self, type, availability_zone=None): + """ + Create a new Vpn Gateway + + :type type: str + :param type: Type of VPN Connection. Only valid valid currently is 'ipsec.1' + + :type availability_zone: str + :param availability_zone: The Availability Zone where you want the VPN gateway. + + :rtype: The newly created VpnGateway + :return: A :class:`boto.vpc.vpngateway.VpnGateway` object + """ + params = {'Type' : type} + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('CreateVpnGateway', params, VpnGateway) + + def delete_vpn_gateway(self, vpn_gateway_id): + """ + Delete a Vpn Gateway. + + :type vpn_gateway_id: str + :param vpn_gateway_id: The ID of the vpn_gateway to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'VpnGatewayId': vpn_gateway_id} + return self.get_status('DeleteVpnGateway', params) + + def attach_vpn_gateway(self, vpn_gateway_id, vpc_id): + """ + Attaches a VPN gateway to a VPC. + + :type vpn_gateway_id: str + :param vpn_gateway_id: The ID of the vpn_gateway to attach + + :type vpc_id: str + :param vpc_id: The ID of the VPC you want to attach the gateway to. + + :rtype: An attachment + :return: a :class:`boto.vpc.vpngateway.Attachment` + """ + params = {'VpnGatewayId': vpn_gateway_id, + 'VpcId' : vpc_id} + return self.get_object('AttachVpnGateway', params, Attachment) + + # Subnets + + def get_all_subnets(self, subnet_ids=None, filters=None): + """ + Retrieve information about your Subnets. You can filter results to + return information only about those Subnets that match your search + parameters. Otherwise, all Subnets associated with your account + are returned. + + :type subnet_ids: list + :param subnet_ids: A list of strings with the desired Subnet ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the Subnet + (pending,available) + - *vpdId*, the ID of teh VPC the subnet is in. + - *cidrBlock*, CIDR block of the subnet + - *availabilityZone*, the Availability Zone + the subnet is in. + + + :rtype: list + :return: A list of :class:`boto.vpc.subnet.Subnet` + """ + params = {} + if subnet_ids: + self.build_list_params(params, subnet_ids, 'SubnetId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeSubnets', params, [('item', Subnet)]) + + def create_subnet(self, vpc_id, cidr_block, availability_zone=None): + """ + Create a new Subnet + + :type vpc_id: str + :param vpc_id: The ID of the VPC where you want to create the subnet. + + :type cidr_block: str + :param cidr_block: The CIDR block you want the subnet to cover. + + :type availability_zone: str + :param availability_zone: The AZ you want the subnet in + + :rtype: The newly created Subnet + :return: A :class:`boto.vpc.customergateway.Subnet` object + """ + params = {'VpcId' : vpc_id, + 'CidrBlock' : cidr_block} + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('CreateSubnet', params, Subnet) + + def delete_subnet(self, subnet_id): + """ + Delete a subnet. + + :type subnet_id: str + :param subnet_id: The ID of the subnet to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'SubnetId': subnet_id} + return self.get_status('DeleteSubnet', params) + + + # DHCP Options + + def get_all_dhcp_options(self, dhcp_options_ids=None): + """ + Retrieve information about your DhcpOptions. + + :type dhcp_options_ids: list + :param dhcp_options_ids: A list of strings with the desired DhcpOption ID's + + :rtype: list + :return: A list of :class:`boto.vpc.dhcpoptions.DhcpOptions` + """ + params = {} + if dhcp_options_ids: + self.build_list_params(params, dhcp_options_ids, 'DhcpOptionsId') + return self.get_list('DescribeDhcpOptions', params, [('item', DhcpOptions)]) + + def create_dhcp_options(self, vpc_id, cidr_block, availability_zone=None): + """ + Create a new DhcpOption + + :type vpc_id: str + :param vpc_id: The ID of the VPC where you want to create the subnet. + + :type cidr_block: str + :param cidr_block: The CIDR block you want the subnet to cover. + + :type availability_zone: str + :param availability_zone: The AZ you want the subnet in + + :rtype: The newly created DhcpOption + :return: A :class:`boto.vpc.customergateway.DhcpOption` object + """ + params = {'VpcId' : vpc_id, + 'CidrBlock' : cidr_block} + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('CreateDhcpOption', params, DhcpOptions) + + def delete_dhcp_options(self, dhcp_options_id): + """ + Delete a DHCP Options + + :type dhcp_options_id: str + :param dhcp_options_id: The ID of the DHCP Options to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'DhcpOptionsId': subnet_id} + return self.get_status('DeleteDhcpOptions', params) + + def associate_dhcp_options(self, dhcp_options_id, vpc_id): + """ + Associate a set of Dhcp Options with a VPC. + + :type dhcp_options_id: str + :param dhcp_options_id: The ID of the Dhcp Options + + :type vpc_id: str + :param vpc_id: The ID of the VPC. + + :rtype: bool + :return: True if successful + """ + params = {'DhcpOptionsId': dhcp_option, + 'VpcId' : vpc_id} + return self.get_status('AssociateDhcpOptions', params) + + # VPN Connection + + def get_all_vpn_connections(self, vpn_connection_ids=None, filters=None): + """ + Retrieve information about your VPN_CONNECTIONs. You can filter results to + return information only about those VPN_CONNECTIONs that match your search + parameters. Otherwise, all VPN_CONNECTIONs associated with your account + are returned. + + :type vpn_connection_ids: list + :param vpn_connection_ids: A list of strings with the desired VPN_CONNECTION ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the VPN_CONNECTION + pending,available,deleting,deleted + - *type*, the type of connection, currently 'ipsec.1' + - *customerGatewayId*, the ID of the customer gateway + associated with the VPN + - *vpnGatewayId*, the ID of the VPN gateway associated + with the VPN connection + + :rtype: list + :return: A list of :class:`boto.vpn_connection.vpnconnection.VpnConnection` + """ + params = {} + if vpn_connection_ids: + self.build_list_params(params, vpn_connection_ids, 'Vpn_ConnectionId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeVpnConnections', params, [('item', VPNConnection)]) + + def create_vpn_connection(self, type, customer_gateway_id, vpn_gateway_id): + """ + Create a new VPN Connection. + + :type type: str + :param type: The type of VPN Connection. Currently only 'ipsec.1' + is supported + + :type customer_gateway_id: str + :param customer_gateway_id: The ID of the customer gateway. + + :type vpn_gateway_id: str + :param vpn_gateway_id: The ID of the VPN gateway. + + :rtype: The newly created VpnConnection + :return: A :class:`boto.vpc.vpnconnection.VpnConnection` object + """ + params = {'Type' : type, + 'CustomerGatewayId' : customer_gateway_id, + 'VpnGatewayId' : vpn_gateway_id} + return self.get_object('CreateVpnConnection', params, VpnConnection) + + def delete_vpn_connection(self, vpn_connection_id): + """ + Delete a VPN Connection. + + :type vpn_connection_id: str + :param vpn_connection_id: The ID of the vpn_connection to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'VpnConnectionId': vpn_connection_id} + return self.get_status('DeleteVpnConnection', params) + + diff --git a/storefront/boto/vpc/customergateway.py b/storefront/boto/vpc/customergateway.py new file mode 100644 index 0000000..c50a616 --- /dev/null +++ b/storefront/boto/vpc/customergateway.py @@ -0,0 +1,54 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Customer Gateway +""" + +from boto.ec2.ec2object import EC2Object + +class CustomerGateway(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.type = None + self.state = None + self.ip_address = None + self.bgp_asn = None + + def __repr__(self): + return 'CustomerGateway:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'customerGatewayId': + self.id = value + elif name == 'ipAddress': + self.ip_address = value + elif name == 'type': + self.type = value + elif name == 'state': + self.state = value + elif name == 'bgpAsn': + self.bgp_asn = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/vpc/dhcpoptions.py b/storefront/boto/vpc/dhcpoptions.py new file mode 100644 index 0000000..4fce7dc --- /dev/null +++ b/storefront/boto/vpc/dhcpoptions.py @@ -0,0 +1,69 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a DHCP Options set +""" + +from boto.ec2.ec2object import EC2Object + +class DhcpValueSet(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'value': + self.append(value) + +class DhcpConfigSet(dict): + + def startElement(self, name, attrs, connection): + if name == 'valueSet': + if not self.has_key(self._name): + self[self._name] = DhcpValueSet() + return self[self._name] + + def endElement(self, name, value, connection): + if name == 'key': + self._name = value + +class DhcpOptions(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.options = None + + def __repr__(self): + return 'DhcpOptions:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'dhcpConfigurationSet': + self.options = DhcpConfigSet() + return self.options + + def endElement(self, name, value, connection): + if name == 'dhcpOptionsId': + self.id = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/vpc/subnet.py b/storefront/boto/vpc/subnet.py new file mode 100644 index 0000000..de8a959 --- /dev/null +++ b/storefront/boto/vpc/subnet.py @@ -0,0 +1,54 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Subnet +""" + +from boto.ec2.ec2object import EC2Object + +class Subnet(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.state = None + self.cidr_block = None + self.available_ip_address_count = 0 + self.availability_zone = None + + def __repr__(self): + return 'Subnet:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'subnetId': + self.id = value + elif name == 'state': + self.state = value + elif name == 'cidrBlock': + self.cidr_block = value + elif name == 'availableIpAddressCount': + self.available_ip_address_count = int(value) + elif name == 'availabilityZone': + self.availability_zone = value + else: + setattr(self, name, value) + diff --git a/storefront/boto/vpc/vpc.py b/storefront/boto/vpc/vpc.py new file mode 100644 index 0000000..152cff3 --- /dev/null +++ b/storefront/boto/vpc/vpc.py @@ -0,0 +1,54 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Virtual Private Cloud. +""" + +from boto.ec2.ec2object import EC2Object + +class VPC(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.dhcp_options_id = None + self.state = None + self.cidr_block = None + + def __repr__(self): + return 'VPC:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'vpcId': + self.id = value + elif name == 'dhcpOptionsId': + self.dhcp_options_id = value + elif name == 'state': + self.state = value + elif name == 'cidrBlock': + self.cidr_block = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_vpc(self.id) + diff --git a/storefront/boto/vpc/vpnconnection.py b/storefront/boto/vpc/vpnconnection.py new file mode 100644 index 0000000..42739d9 --- /dev/null +++ b/storefront/boto/vpc/vpnconnection.py @@ -0,0 +1,60 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a VPN Connectionn +""" + +from boto.ec2.ec2object import EC2Object + +class VpnConnection(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.state = None + self.customer_gateway_configuration = None + self.type = None + self.customer_gateway_id = None + self.vpn_gateway_id = Nonen + + def __repr__(self): + return 'VpnConnection:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'vpnConnectionId': + self.id = value + elif name == 'state': + self.state = value + elif name == 'CustomerGatewayConfiguration': + self.customer_gateway_configuration = value + elif name == 'type': + self.type = value + elif name == 'customerGatewayId': + self.customer_gateway_id = value + elif name == 'vpnGatewayId': + self.vpn_gateway_id = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_vpn_connection(self.id) + diff --git a/storefront/boto/vpc/vpngateway.py b/storefront/boto/vpc/vpngateway.py new file mode 100644 index 0000000..0fa0a9e --- /dev/null +++ b/storefront/boto/vpc/vpngateway.py @@ -0,0 +1,80 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Vpn Gateway +""" + +from boto.ec2.ec2object import EC2Object + +class Attachment(object): + + def __init__(self, connection=None): + self.vpc_id = None + self.state = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'vpcId': + self.vpc_id = value + elif name == 'state': + self.state = value + else: + setattr(self, name, value) + +class VpnGateway(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.type = None + self.state = None + self.availability_zone = None + self.attachments = [] + + def __repr__(self): + return 'VpnGateway:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'item': + att = Attachment() + self.attachments.append(att) + return att + + def endElement(self, name, value, connection): + if name == 'vpnGatewayId': + self.id = value + elif name == 'type': + self.type = value + elif name == 'state': + self.state = value + elif name == 'availabilityZone': + self.availability_zone = value + elif name == 'attachments': + pass + else: + setattr(self, name, value) + + def attach(self, vpc_id): + return self.connection.attach_vpn_gateway(self.id, vpc_id) + diff --git a/storefront/django_cookies.py b/storefront/django_cookies.py new file mode 100644 index 0000000..22bf776 --- /dev/null +++ b/storefront/django_cookies.py @@ -0,0 +1,98 @@ +""" +A two-part middleware which modifies request.COOKIES and adds a set and delete method. + + `set` matches django.http.HttpResponse.set_cookie + `delete` matches django.http.HttpResponse.delete_cookie + +MIDDLEWARE_CLASSES = ( + 'django_cookies.CookiePreHandlerMiddleware', + ... + 'django_cookies.CookiePostHandlerMiddleware', +) + +def my_view(request): + request.COOKIES.set([args]) + ... + return response +""" + +from Cookie import SimpleCookie, Morsel +import copy + +class CookiePreHandlerMiddleware(object): + """ + This middleware modifies request.COOKIES and adds a set and delete method. + + `set` matches django.http.HttpResponse.set_cookie + `delete` matches django.http.HttpResponse.delete_cookie + + This should be the first middleware you load. + """ + def process_request(self, request): + cookies = CookieHandler() + for k, v in request.COOKIES.iteritems(): + cookies[k] = str(v) + request.COOKIES = cookies + request._orig_cookies = copy.deepcopy(request.COOKIES) + +class CookiePostHandlerMiddleware(object): + """ + This middleware modifies updates the response will all modified cookies. + + This should be the last middleware you load. + """ + def process_response(self, request, response): + if hasattr(request, '_orig_cookies') and request.COOKIES != request._orig_cookies: + for k,v in request.COOKIES.iteritems(): + if request._orig_cookies.get(k) != v: + dict.__setitem__(response.cookies, k, v) + return response + +class StringMorsel(Morsel): + def __str__(self): + return self.value + + def __eq__(self, a): + if isinstance(a, str): + return str(self) == a + elif isinstance(a, Morsel): + return a.output() == self.output() + return False + + def __ne__(self, a): + if isinstance(a, str): + return str(self) != a + elif isinstance(a, Morsel): + return a.output() != self.output() + return True + + def __repr__(self): + return str(self) + +class CookieHandler(SimpleCookie): + def __set(self, key, real_value, coded_value): + """Private method for setting a cookie's value""" + M = self.get(key, StringMorsel()) + M.set(key, real_value, coded_value) + dict.__setitem__(self, key, M) + + def __setitem__(self, key, value): + """Dictionary style assignment.""" + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) + + def set(self, key, value='', max_age=None, expires=None, path='/', domain=None, secure=None): + self[key] = value + for var in ('max_age', 'path', 'domain', 'secure', 'expires'): + val = locals()[var] + if val is not None: + self[key][var.replace('_', '-')] = val + + def delete(self, key, path='/', domain=None): + self[key] = '' + if path is not None: + self[key]['path'] = path + if domain is not None: + self[key]['domain'] = domain + self[key]['expires'] = 0 + self[key]['max-age'] = 0 \ No newline at end of file diff --git a/storefront/error_logging.py b/storefront/error_logging.py new file mode 100644 index 0000000..ef7f6f5 --- /dev/null +++ b/storefront/error_logging.py @@ -0,0 +1,11 @@ +import traceback +import sys +class ViewErrorLoggingMiddleware: + + def process_view(self, request, view_func, view_args, view_kwargs): + self.view_name = view_func.__name__ + def process_exception(self, request, exception): + print '=' * 60 + print '[ERROR] exception in view "%s"' % self.view_name + traceback.print_exc(file=sys.stdout) + print '=' * 60 diff --git a/storefront/forms.py b/storefront/forms.py new file mode 100644 index 0000000..4b426a4 --- /dev/null +++ b/storefront/forms.py @@ -0,0 +1,379 @@ +from django import forms +from django.forms.widgets import HiddenInput, Select + +from datetime import datetime, timedelta + +COUNTRIES = [('US', 'United States'), + ('AD', 'Andorra'), + ('AE', 'United Arab Emirates'), + ('AF', 'Afghanistan'), + ('AG', 'Antigua and Barbuda'), + ('AI', 'Anguilla'), + ('AL', 'Albania'), + ('AM', 'Armenia'), + ('AN', 'Netherlands Antilles'), + ('DZ', 'Algeria'), + ('AO', 'Angola'), + ('AQ', 'Antarctica'), + ('AR', 'Argentina'), + ('AS', 'American Samoa'), + ('AT', 'Austria'), + ('AU', 'Australia'), + ('AW', 'Aruba'), + ('AX', 'Aland Island'), + ('AZ', 'Azerbaijan'), + ('BA', 'Bosnia and Herzegovina'), + ('BB', 'Barbados'), + ('BD', 'Bangladesh'), + ('BE', 'Belgium'), + ('BF', 'Burkina Faso'), + ('BG', 'Bulgaria'), + ('BH', 'Bahrain'), + ('BI', 'Burundi'), + ('BJ', 'Benin'), + ('BL', 'Saint Barthelemy'), + ('BM', 'Bermuda'), + ('BN', 'Brunei Darussalam'), + ('BO', 'Bolivia, Plurinational State of'), + ('BR', 'Brazil'), + ('BS', 'Bahamas'), + ('BT', 'Bhutan'), + ('BV', 'Bouvet Island'), + ('BW', 'Botswana'), + ('BY', 'Belarus'), + ('BZ', 'Belize'), + ('CA', 'Canada'), + ('CC', 'Cocos (Keeling) Islands'), + ('CD', 'Congo, the Democratic Republic of the'), + ('CF', 'Central African Republic'), + ('CG', 'Congo'), + ('CH', 'Switzerland'), + ('CI', 'Cote d\'Ivoire'), + ('CK', 'Cook Islands'), + ('CL', 'Chile'), + ('CM', 'Cameroon'), + ('CN', 'China'), + ('CO', 'Colombia'), + ('CR', 'Costa Rica'), + ('CU', 'Cuba'), + ('CV', 'Cape Verde'), + ('CX', 'Christmas Island'), + ('CY', 'Cyprus'), + ('CZ', 'Czech Republic'), + ('DE', 'Germany'), + ('DJ', 'Djibouti'), + ('DK', 'Denmark'), + ('DM', 'Dominica'), + ('DO', 'Dominican Republic'), + ('EC', 'Ecuador'), + ('EE', 'Estonia'), + ('EG', 'Egypt'), + ('EH', 'Western Sahara'), + ('ER', 'Eritrea'), + ('ES', 'Spain'), + ('ET', 'Ethiopia'), + ('FI', 'Finland'), + ('FJ', 'Fiji'), + ('FK', 'Falkland Islands (Malvinas)'), + ('FM', 'Micronesia, Federated States of'), + ('FO', 'Faroe Islands'), + ('FR', 'France'), + ('GA', 'Gabon'), + ('GB', 'United Kingdom'), + ('GD', 'Grenada'), + ('GE', 'Georgia'), + ('GF', 'French Guiana'), + ('GG', 'Guernsey'), + ('GH', 'Ghana'), + ('GI', 'Gibraltar'), + ('GL', 'Greenland'), + ('GM', 'Gambia'), + ('GN', 'Guinea'), + ('GP', 'Guadeloupe'), + ('GQ', 'Equatorial Guinea'), + ('GR', 'Greece'), + ('GS', 'South Georgia and the South Sandwich Islands'), + ('GT', 'Guatemala'), + ('GU', 'Guam'), + ('GW', 'Guinea-Bissau'), + ('GY', 'Guyana'), + ('HK', 'Hong Kong'), + ('HM', 'Heard Island and McDonald Islands'), + ('HN', 'Honduras'), + ('HR', 'Croatia'), + ('HT', 'Haiti'), + ('HU', 'Hungary'), + ('ID', 'Indonesia'), + ('IE', 'Ireland'), + ('IL', 'Israel'), + ('IM', 'Isle of Man'), + ('IN', 'India'), + ('IO', 'British Indian Ocean Territory'), + ('IQ', 'Iraq'), + ('IR', 'Iran, Islamic Republic of'), + ('IS', 'Iceland'), + ('IT', 'Italy'), + ('JE', 'Jersey'), + ('JM', 'Jamaica'), + ('JO', 'Jordan'), + ('JP', 'Japan'), + ('KE', 'Kenya'), + ('KG', 'Kyrgyzstan'), + ('KH', 'Cambodia'), + ('KI', 'Kiribati'), + ('KM', 'Comoros'), + ('KN', 'Saint Kitts and Nevis'), + ('KP', 'Korea, Democratic People\'s Republic of'), + ('KR', 'Korea, Republic of'), + ('KW', 'Kuwait'), + ('KY', 'Cayman Islands'), + ('KZ', 'Kazakhstan'), + ('LA', 'Lao People\'s Democratic Republic'), + ('LB', 'Lebanon'), + ('LC', 'Saint Lucia'), + ('LI', 'Liechtenstein'), + ('LK', 'Sri Lanka'), + ('LR', 'Liberia'), + ('LS', 'Lesotho'), + ('LT', 'Lithuania'), + ('LU', 'Luxembourg'), + ('LV', 'Latvia'), + ('LY', 'Libyan Arab Jamahiriya'), + ('MA', 'Morocco'), + ('MC', 'Monaco'), + ('MD', 'Moldova, Republic of'), + ('ME', 'Montenegro'), + ('MF', 'Saint Martin (French part)'), + ('MG', 'Madagascar'), + ('MH', 'Marshall Islands'), + ('MK', 'Macedonia, the former Yugoslav Republic of'), + ('ML', 'Mali'), + ('MM', 'Myanmar'), + ('MN', 'Mongolia'), + ('MO', 'Macao'), + ('MP', 'Northern Mariana Islands'), + ('MQ', 'Martinique'), + ('MR', 'Mauritania'), + ('MS', 'Montserrat'), + ('MT', 'Malta'), + ('MU', 'Mauritius'), + ('MV', 'Maldives'), + ('MW', 'Malawi'), + ('MX', 'Mexico'), + ('MY', 'Malaysia'), + ('MZ', 'Mozambique'), + ('NA', 'Namibia'), + ('NC', 'New Caledonia'), + ('NE', 'Niger'), + ('NF', 'Norfolk Island'), + ('NG', 'Nigeria'), + ('NI', 'Nicaragua'), + ('NL', 'Netherlands'), + ('NO', 'Norway'), + ('NP', 'Nepal'), + ('NR', 'Nauru'), + ('NU', 'Niue'), + ('NZ', 'New Zealand'), + ('OM', 'Oman'), + ('PA', 'Panama'), + ('PE', 'Peru'), + ('PF', 'French Polynesia'), + ('PG', 'Papua New Guinea'), + ('PH', 'Philippines'), + ('PK', 'Pakistan'), + ('PL', 'Poland'), + ('PM', 'Saint Pierre and Miquelon'), + ('PN', 'Pitcairn'), + ('PR', 'Puerto Rico'), + ('PS', 'Palestinian Territory, Occupied'), + ('PT', 'Portugal'), + ('PW', 'Palau'), + ('PY', 'Paraguay'), + ('QA', 'Qatar'), + ('RE', 'Reunion'), + ('RO', 'Romania'), + ('RS', 'Serbia'), + ('RU', 'Russian Federation'), + ('RW', 'Rwanda'), + ('SA', 'Saudi Arabia'), + ('SB', 'Solomon Islands'), + ('SC', 'Seychelles'), + ('SD', 'Sudan'), + ('SE', 'Sweden'), + ('SG', 'Singapore'), + ('SH', 'Saint Helena, Ascension and Tristan da Cunha'), + ('SI', 'Slovenia'), + ('SJ', 'Svalbard and Jan Mayen'), + ('SK', 'Slovakia'), + ('SL', 'Sierra Leone'), + ('SM', 'San Marino'), + ('SN', 'Senegal'), + ('SO', 'Somalia'), + ('SR', 'Suriname'), + ('ST', 'Sao Tome and Principe'), + ('SV', 'El Salvador'), + ('SY', 'Syrian Arab Republic'), + ('SZ', 'Swaziland'), + ('TC', 'Turks and Caicos Islands'), + ('TD', 'Chad'), + ('TF', 'French Southern Territories'), + ('TG', 'Togo'), + ('TH', 'Thailand'), + ('TJ', 'Tajikistan'), + ('TK', 'Tokelau'), + ('TL', 'Timor-Leste'), + ('TM', 'Turkmenistan'), + ('TN', 'Tunisia'), + ('TO', 'Tonga'), + ('TR', 'Turkey'), + ('TT', 'Trinidad and Tobago'), + ('TV', 'Tuvalu'), + ('TW', 'Taiwan, Province of China'), + ('TZ', 'Tanzania, United Republic of'), + ('UA', 'Ukraine'), + ('UG', 'Uganda'), + ('UM', 'United States Minor Outlying Islands'), + ('UY', 'Uruguay'), + ('UZ', 'Uzbekistan'), + ('VA', 'Holy See (Vatican City State)'), + ('VC', 'Saint Vincent and the Grenadines'), + ('VE', 'Venezuela, Bolivarian Republic of'), + ('VG', 'Virgin Islands, British'), + ('VI', 'Virgin Islands, U.S.'), + ('VN', 'Viet Nam'), + ('VU', 'Vanuatu'), + ('WF', 'Wallis and Futuna'), + ('WS', 'Samoa'), + ('YE', 'Yemen'), + ('YT', 'Mayotte'), + ('ZA', 'South Africa'), + ('ZM', 'Zambia'), + ('ZW', 'Zimbabwe'),] + + +def emptystyle(original_class): + class E(original_class): + def widget_attrs(self, widget): + attrs = { 'class': 'empty', 'emptyvalue': self.label } + if isinstance(widget, forms.PasswordInput): + attrs['ispass'] = 'ispass' + return attrs + return E + +class LoginForm(forms.Form): + email = emptystyle(forms.EmailField)(required=True, label='Email') + password = emptystyle(forms.CharField)(widget=forms.PasswordInput, label='Password') + +class ForgotPassForm(forms.Form): + email = emptystyle(forms.EmailField)(required=True, label='Email') + +class CloseAccountForm(forms.Form): + password = forms.CharField(widget=forms.PasswordInput, label='Password') + +class SignUpForm(forms.Form): + email = forms.EmailField(required=True, label='Email') + password = forms.CharField(widget=forms.PasswordInput, label='Password') + password_again = forms.CharField(widget=forms.PasswordInput, label=('Password again')) + + def clean_password_again(self): + if 'password' in self.cleaned_data: + if self.cleaned_data['password'] != self.cleaned_data['password_again']: + raise forms.ValidationError('Passwords didn\'t match') + +class ChangePassForm(forms.Form): + old_password = emptystyle(forms.CharField)(widget=forms.PasswordInput, label='Current password') + new_password = emptystyle(forms.CharField)(widget=forms.PasswordInput, label='New password') + new_password_again = emptystyle(forms.CharField)(widget=forms.PasswordInput, label=('Repeat new password')) + + def clean_new_password_again(self): + if 'new_password' in self.cleaned_data: + if self.cleaned_data['new_password'] != self.cleaned_data['new_password_again']: + raise forms.ValidationError('Passwords didn\'t match') + +class IndexForm(forms.Form): + name = emptystyle(forms.CharField)(min_length=3, max_length=50, required=True, label='Index Name') + def clean_name(self): + name = self.cleaned_data['name'] + if name is None or len(name) == 0: + raise forms.ValidationError('Must provide an index name') + if not name[0].isalpha(): + raise forms.ValidationError('Index name must start with a letter') + for character in name: + if not (character.isalpha() or character.isdigit() or character == '_'): + raise forms.ValidationError('Invalid character in index name.') + return name + +class ScoreFunctionForm(forms.Form): + name = forms.CharField(widget=HiddenInput(), min_length=1, required=True, label='Numeric Code') + definition = forms.CharField(required=True, label='Formula') + +class BetaTestForm(forms.Form): + email = forms.EmailField(required=True, label='* E-Mail') + site_url = forms.CharField(required=False, label='Site', max_length=200) + textarea_widget = forms.widgets.Textarea(attrs={'rows': 5, 'cols': 20}) + summary = forms.CharField(widget=textarea_widget, required=False, max_length=500, min_length=4, label='Intended use') + +class PaymentInformationForm(forms.Form): + first_name = emptystyle(forms.CharField)(required=True, label='First Name', max_length=50) + last_name = emptystyle(forms.CharField)(required=True, label='Last Name', max_length=50) + + credit_card_number = emptystyle(forms.CharField)(required=True, label='Credit Card Number', max_length=19) + exp_month = emptystyle(forms.CharField)(required=True, label='Expiration (MM/YY)', max_length=5) + #exp_year = emptystyle(forms.CharField)(required=True, label='Exp Year', max_length=2) + + address = emptystyle(forms.CharField)(required=False, label='Address', max_length=60) + city = emptystyle(forms.CharField)(required=False, label='City', max_length=60) + state = emptystyle(forms.CharField)(required=False, label='State', max_length=2) + zip_code = emptystyle(forms.CharField)(required=False, label='ZIP Code', max_length=15) + #select_widget = Select(attrs={'style':'width: 280px;border: none;background: white;font-size: 16px;height: 40px;margin: 1px;text-indent: 5px;'}) + country = emptystyle(forms.ChoiceField)(required=False, choices=COUNTRIES, label='Country') + + def clean_credit_card_number(self): + if 'credit_card_number' in self.cleaned_data: + cc = self.cleaned_data['credit_card_number'] + if not cc.isdigit() or not is_luhn_valid(cc): + raise forms.ValidationError('Invalid credit card number') + return self.cleaned_data['credit_card_number'] + + def clean(self): + month = '' + year = '' + if 'exp_month' in self.cleaned_data: + month = self.cleaned_data['exp_month'] + if not '/' in month: + self._errors['exp_month'] = ['Not a valid expiration (it should be MM/YY)'] + return + + month, year = month.split('/', 1) + + if not month.isdigit() or int(month) > 12: + self._errors['exp_month'] = ['Not a valid expiration month (it should be MM/YY)'] + return + if not year.isdigit() or int(year) > 99: + self._errors['exp_month'] = ['Not a valid expiration year (it should be MM/YY)'] + return + else: + self._errors['exp_month'] = ['Not a valid expiration (it should be MM/YY)'] + return + + expiration = datetime(month=int(month), year=int('20' + year), day=1) + today = datetime.now() + timedelta(days=1) + + checkpoint_month = today.month + 1 if today.month != 12 else 1 + checkpoint_year = today.year if today.month != 12 else today.year + 1 + + checkpoint = datetime(month=checkpoint_month, year=checkpoint_year, day=1) + + if expiration <= checkpoint: + self._errors['exp_month'] = ['Card expired'] + return + + return self.cleaned_data + + +def is_luhn_valid(cc): + num = map(int, cc) + return not sum(num[::-2] + map(lambda d: sum(divmod(d * 2, 10)), num[-2::-2])) % 10 + + + diff --git a/storefront/kill_webapp.sh b/storefront/kill_webapp.sh new file mode 100755 index 0000000..35e822c --- /dev/null +++ b/storefront/kill_webapp.sh @@ -0,0 +1,2 @@ +#!/bin/bash +kill `cat pid` diff --git a/storefront/lib/__init__.py b/storefront/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/storefront/lib/authorizenet.py b/storefront/lib/authorizenet.py new file mode 100644 index 0000000..00bcd6c --- /dev/null +++ b/storefront/lib/authorizenet.py @@ -0,0 +1,153 @@ +from xml.dom.minidom import Document, parseString +import httplib +import urlparse + + +class AuthorizeNet: + """ + Basic client for Authorize.net's Automated Recurring Billing (ARB) service + """ + + def __init__(self): + from django.conf import settings + f = open("authorize.settings.prod") if not settings.DEBUG else open("authorize.settings.debug") + for line in f: + line = line.strip() + if len(line) > 0 and not line.startswith('#'): + parts = line.split('=',1) + var = parts[0].strip() + val = parts[1].strip() + if var in ['host_url','api_login_id','transaction_key']: + cmd = 'self.%s = %s' % (var,val) + exec(cmd) + + def subscription_create(self, refId, name, length, unit, startDate, totalOccurrences, trialOccurrences, + amount, trialAmount, cardNumber, expirationDate, firstName, lastName, company, + address, city, state, zip, country): + doc,root = self._new_doc("ARBCreateSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + subscription = self._add_node(doc, root, 'subscription') + self._add_text_node(doc, subscription, 'name', name) + paymentSchedule = self._add_node(doc, subscription, 'paymentSchedule') + interval = self._add_node(doc, paymentSchedule, 'interval') + self._add_text_node(doc, interval, 'length', length) + self._add_text_node(doc, interval, 'unit', unit) + self._add_text_node(doc, paymentSchedule, 'startDate', startDate) + self._add_text_node(doc, paymentSchedule, 'totalOccurrences', totalOccurrences) + self._add_text_node(doc, paymentSchedule, 'trialOccurrences', trialOccurrences) + self._add_text_node(doc, subscription, 'amount', amount) + self._add_text_node(doc, subscription, 'trialAmount', trialAmount) + payment = self._add_node(doc, subscription, 'payment') + creditcard = self._add_node(doc, payment, 'creditCard') + self._add_text_node(doc, creditcard, 'cardNumber', cardNumber) + self._add_text_node(doc, creditcard, 'expirationDate', expirationDate) + billto = self._add_node(doc, subscription, 'billTo') + self._add_text_node(doc, billto, 'firstName', firstName) + self._add_text_node(doc, billto, 'lastName', lastName) + self._add_text_node(doc, billto, 'company', company) + self._add_text_node(doc, billto, 'address', address) + self._add_text_node(doc, billto, 'city', city) + self._add_text_node(doc, billto, 'state', state) + self._add_text_node(doc, billto, 'zip', zip) + self._add_text_node(doc, billto, 'country', country) + res = self._send_xml(doc.toxml()) + subscriptionId = res.getElementsByTagName('subscriptionId')[0].childNodes[0].nodeValue + return subscriptionId + + + def subscription_update(self, refId, subscriptionId, name, startDate, totalOccurrences, trialOccurrences, + amount, trialAmount, cardNumber, expirationDate, firstName, lastName, company, + address, city, state, zip, country): + doc,root = self._new_doc("ARBUpdateSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + self._add_text_node(doc, root, 'subscriptionId', subscriptionId) + subscription = self._add_node(doc, root, 'subscription') + if name: + self._add_text_node(doc, subscription, 'name', name) + if startDate or totalOccurrences or trialOccurrences: + paymentSchedule = self._add_node(doc, subscription, 'paymentSchedule') + if startDate: + self._add_text_node(doc, paymentSchedule, 'startDate', startDate) + if totalOccurrences: + self._add_text_node(doc, paymentSchedule, 'totalOccurrences', totalOccurrences) + if trialOccurrences: + self._add_text_node(doc, paymentSchedule, 'trialOccurrences', trialOccurrences) + if amount: + self._add_text_node(doc, subscription, 'amount', amount) + if trialAmount: + self._add_text_node(doc, subscription, 'trialAmount', trialAmount) + if cardNumber and expirationDate: + payment = self._add_node(doc, subscription, 'payment') + creditcard = self._add_node(doc, payment, 'creditCard') + self._add_text_node(doc, creditcard, 'cardNumber', cardNumber) + self._add_text_node(doc, creditcard, 'expirationDate', expirationDate) + if firstName and lastName: + billto = self._add_node(doc, subscription, 'billTo') + self._add_text_node(doc, billto, 'firstName', firstName) + self._add_text_node(doc, billto, 'lastName', lastName) + if company: + self._add_text_node(doc, billto, 'company', company) + if address and city and state and zip and country: + self._add_text_node(doc, billto, 'address', address) + self._add_text_node(doc, billto, 'city', city) + self._add_text_node(doc, billto, 'state', state) + self._add_text_node(doc, billto, 'zip', zip) + self._add_text_node(doc, billto, 'country', country) + self._send_xml(doc.toxml()) + + + def subscription_cancel(self, refId, subscriptionId): + doc,root = self._new_doc("ARBCancelSubscriptionRequest") + self._add_text_node(doc, root, 'refId', refId) + self._add_text_node(doc, root, 'subscriptionId', subscriptionId) + self._send_xml(doc.toxml()) + + + def _add_node(self, doc, node, name): + elem = doc.createElement(name) + node.appendChild(elem) + return elem + + def _add_text_node(self, doc, node, name, text): + elem = self._add_node(doc, node, name) + text_node = doc.createTextNode(text) + elem.appendChild(text_node) + return elem + + def _new_doc(self, operation): + doc = Document() + root = doc.createElement(operation) + root.setAttribute('xmlns','AnetApi/xml/v1/schema/AnetApiSchema.xsd') + doc.appendChild(root) + auth = self._add_node(doc, root, 'merchantAuthentication') + self._add_text_node(doc, auth, 'name', self.api_login_id) + self._add_text_node(doc, auth, 'transactionKey', self.transaction_key) + return doc, root + + def _send_xml(self, xml): + splits = urlparse.urlsplit(self.host_url) + print "connection.request('POST', "+self.host_url+", xml, {'Content-Type':'text/xml'})" + print "xml: "+xml + connection = httplib.HTTPSConnection(splits.hostname) + connection.request('POST', self.host_url, xml, {'Content-Type':'text/xml'}) + response = connection.getresponse() + response.body = response.read() + connection.close() + print "resp: "+response.body + res = parseString(response.body) + ok = res.getElementsByTagName('resultCode')[0].childNodes[0].nodeValue == "Ok" + if not ok: + code = res.getElementsByTagName('message')[0].childNodes[0].childNodes[0].nodeValue + msg = res.getElementsByTagName('message')[0].childNodes[1].childNodes[0].nodeValue + " (%s)"%code + raise BillingException(msg,code) + return res + + +class BillingException(Exception): + def __init__(self, msg, code): + self.msg = msg + self.code = code + def __str__(self): + return repr(self.msg) + + diff --git a/storefront/lib/encoder.py b/storefront/lib/encoder.py new file mode 100644 index 0000000..f6bb4dd --- /dev/null +++ b/storefront/lib/encoder.py @@ -0,0 +1,74 @@ +# Short URL Generator + +#DEFAULT_ALPHABET = 'JedR8LNFY2j6MrhkBSADUyfP5amuH9xQCX4VqbgpsGtnW7vc3TwKE' +DEFAULT_ALPHABET = 'ed82j6rhkyf5amu9x4qbgpstn7vc3w1ioz' +DEFAULT_BLOCK_SIZE = 22 + +class Encoder(object): + def __init__(self, alphabet=DEFAULT_ALPHABET, block_size=DEFAULT_BLOCK_SIZE): + self.alphabet = alphabet + self.block_size = block_size + self.mask = (1 << block_size) - 1 + self.mapping = range(block_size) + self.mapping.reverse() + def encode_url(self, n, min_length=0): + return self.enbase(self.encode(n), min_length) + def decode_url(self, n): + return self.decode(self.debase(n)) + def encode(self, n): + return (n & ~self.mask) | self._encode(n & self.mask) + def _encode(self, n): + result = 0 + for i, b in enumerate(self.mapping): + if n & (1 << i): + result |= (1 << b) + return result + def decode(self, n): + return (n & ~self.mask) | self._decode(n & self.mask) + def _decode(self, n): + result = 0 + for i, b in enumerate(self.mapping): + if n & (1 << b): + result |= (1 << i) + return result + def enbase(self, x, min_length=0): + result = self._enbase(x) + padding = self.alphabet[0] * (min_length - len(result)) + return '%s%s' % (padding, result) + def _enbase(self, x): + n = len(self.alphabet) + if x < n: + return self.alphabet[x] + return self.enbase(x/n) + self.alphabet[x%n] + def debase(self, x): + n = len(self.alphabet) + result = 0 + for i, c in enumerate(reversed(x)): + result += self.alphabet.index(c) * (n**i) + return result + +DEFAULT_ENCODER = Encoder() + +def encode(n): + return DEFAULT_ENCODER.encode(n) + +def decode(n): + return DEFAULT_ENCODER.decode(n) + +def enbase(n, min_length=0): + return DEFAULT_ENCODER.enbase(n, min_length) + +def debase(n): + return DEFAULT_ENCODER.debase(n) + +def encode_url(n, min_length=0): + return DEFAULT_ENCODER.encode_url(n, min_length) + +def decode_url(n): + return DEFAULT_ENCODER.decode_url(n) + +def to_key(n): + return enbase(encode(n)) + +def from_key(n): + return decode(debase(n)) diff --git a/storefront/lib/error_logging.py b/storefront/lib/error_logging.py new file mode 100644 index 0000000..dbf927b --- /dev/null +++ b/storefront/lib/error_logging.py @@ -0,0 +1,13 @@ +import traceback +from lib import flaptor_logging +from django.http import HttpResponse + +logger = flaptor_logging.get_logger('error_logging') + +class ViewErrorLoggingMiddleware: + + def process_view(self, request, view_func, view_args, view_kwargs): + self.view_name = view_func.__name__ + def process_exception(self, request, exception): + logger.error('UNEXPECTED EXCEPTION in view "%s". Exception is: %s', self.view_name, repr(traceback.print_exc())) + return HttpResponse('{"status":"ERROR", "message":"Unexpected error."}') diff --git a/storefront/lib/exceptions.py b/storefront/lib/exceptions.py new file mode 100644 index 0000000..c6ff0a7 --- /dev/null +++ b/storefront/lib/exceptions.py @@ -0,0 +1,8 @@ + + +class CloudException(Exception): + pass + +class NoIndexerException(CloudException): + pass + diff --git a/storefront/lib/flaptor_logging.py b/storefront/lib/flaptor_logging.py new file mode 100644 index 0000000..1af893c --- /dev/null +++ b/storefront/lib/flaptor_logging.py @@ -0,0 +1,100 @@ +import logging as pylogging +from logging import config +import os + +usingNativeLogger = True + +__loggers = {} + + + +def get_logger(name, force_new=False): + '''Get the Logger instance for a given name''' + global __loggers + if __loggers is None: + __loggers = {} + if force_new: + return pylogging.getLogger(name) + if not __loggers.has_key(name): + __loggers[name] = pylogging.getLogger(name) + return __loggers[name] + +class SpecialFormatter(pylogging.Formatter): + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) + RESET_SEQ = "\033[0m" + COLOR_SEQ = "\033[37;4%dm" + PIDCOLOR_SEQ = "\033[1;3%dm" + BOLD_SEQ = "\033[1m" + COLORS = { + 'WARN': YELLOW, + 'INFO': GREEN, + 'DEBU': BLUE, + 'CRIT': RED, + 'ERRO': RED + } + + def __init__(self, *args, **kwargs): + pylogging.Formatter.__init__(self, *args, **kwargs) + def format(self, record): + if not hasattr(record, 'prefix'): record.prefix = '' + if not hasattr(record, 'suffix'): record.suffix = '' + if not hasattr(record, 'compname'): record.compname = '' + record.pid = os.getpid() + + record.levelname = record.levelname[:4] + + r = pylogging.Formatter.format(self, record) + if record.levelname in SpecialFormatter.COLORS: + levelcolor = SpecialFormatter.COLOR_SEQ % (SpecialFormatter.COLORS[record.levelname]) + r = r.replace('$LEVELCOLOR', levelcolor) + r = r.replace('$RESET', SpecialFormatter.RESET_SEQ) + else: + r = r.replace('$COLOR', '') + r = r.replace('$RESET', '') + pidcolor = SpecialFormatter.COLOR_SEQ % (1 + (record.pid % 5)) + r = r.replace('$PIDCOLOR', pidcolor) + r = r.replace('$BOLD', SpecialFormatter.BOLD_SEQ) + return r + +pylogging.SpecialFormatter = SpecialFormatter + +if usingNativeLogger: + try: + config.fileConfig('logging.conf') + except Exception, e: + print e + +#class NativePythonLogger: +# def __init__(self, name): +# '''Creates a new Logger for the given name. +# Do not call this method directly, instead use +# get_logger(name) to get the appropriate instance''' +# self.name = name +# self.__logger = pylogging.getLogger(name) +# #self.updateLevel(5) +# +# def updateLevel(self, level): +# self.__level = level +# if level == 1: +# self.__logger.setLevel(pylogging.CRITICAL) +# elif level == 2: +# self.__logger.setLevel(pylogging.INFO) +# elif level == 3: +# self.__logger.setLevel(pylogging.WARNING) +# elif level == 4: +# self.__logger.setLevel(pylogging.INFO) +# elif level == 5: +# self.__logger.setLevel(pylogging.DEBUG) +# +# def debug(self, format_str, *values): +# self.__logger.debug(format_str, *values) +# def info(self, format_str, *values): +# self.__logger.info(format_str, *values) +# def warn(self, format_str, *values): +# self.__logger.warn(format_str, *values) +# def error(self, format_str, *values): +# self.__logger.error(format_str, *values) +# def exception(self, format_str, *values): +# self.__logger.exception(format_str, *values) +# def fatal(self, format_str, *values): +# self.__logger.critical(format_str, *values) diff --git a/storefront/lib/mail.py b/storefront/lib/mail.py new file mode 100644 index 0000000..b1d3a08 --- /dev/null +++ b/storefront/lib/mail.py @@ -0,0 +1,101 @@ +from django.core.mail import send_mail + +try: + ENV=open('/data/env.name').readline().strip() +except Exception: + ENV='PROD+EXC' + +def _no_fail(method, *args, **kwargs): + def decorated(*args, **kwargs): + try: + return method(*args, **kwargs) + except Exception, e: + print e + return + return decorated + + + +@_no_fail +def report_payment_data(account): + activity_report = 'An Account has entered payment data\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + account.package.name + '\n' + activity_report += 'Email: ' + account.user.email + '\n' + activity_report += 'API KEY: ' + account.apikey + '' + + report_activity('Payment Data for ' + account.package.name + ' Account (' + account.user.email + ')', activity_report) + +@_no_fail +def report_new_account(account): + activity_report = 'A new Account was created\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + account.package.name + '\n' + activity_report += 'Email: ' + account.user.email + '\n' + activity_report += 'API KEY: ' + account.apikey + '' + + report_activity('New ' + account.package.name + ' Account (' + account.user.email + ')', activity_report) + +@_no_fail +def report_new_index(index): + activity_report = 'A new Index was created\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + index.account.package.name + '\n' + activity_report += 'User Email: ' + index.account.user.email + '\n' + activity_report += 'Index Name: ' + index.name + '' + + report_activity('Index activity (' + index.code + ')', activity_report, 'l') + +@_no_fail +def report_new_deploy(deploy): + activity_report = 'A new Deploy is now controllable\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + deploy.index.account.package.name + '\n' + activity_report += 'User Email: ' + deploy.index.account.user.email + '\n' + activity_report += 'Index Name: ' + deploy.index.name + '\n' + activity_report += 'Worker: #' + str(deploy.worker.id) + '\n' + activity_report += ('Deploy: %r' % deploy) + '\n' + activity_report += ('Container Index: %r' % deploy.index) + '\n' + + report_activity('Index activity (' + deploy.index.code + ')', activity_report, 'l') + +@_no_fail +def report_delete_index(index): + activity_report = 'An Index has been deleted\n' + activity_report += '---------------------------\n' + activity_report += 'Plan: ' + index.account.package.name + '\n' + activity_report += 'User Email: ' + index.account.user.email + '\n' + activity_report += 'Index Name: ' + index.name + '\n' + + report_activity('Index activity (' + index.code + ')', activity_report, 'l') + +@_no_fail +def report_new_worker(worker): + activity_report = 'A new Worker was created\n' + activity_report += '---------------------------\n' + activity_report += repr(worker) + + report_activity('New Worker (%d)' % (worker.pk), activity_report, 't') + +@_no_fail +def report_automatic_redeploy(deploy, initial_xmx, new_xmx): + activity_report = 'Automatic redeploy.\n' + activity_report += '---------------------------\n' + activity_report += 'initial xmx value: %d\n' % (initial_xmx) + activity_report += 'new xmx value: %d\n' % (new_xmx) + activity_report += repr(deploy) + + report_activity('Automatic redeploy', activity_report, 't') + +@_no_fail +def report_activity(subject, body, type='b'): + if type == 'b': + mail_to = 'activity@indextank.com' + elif type == 't': + mail_to = 'activitytech@indextank.com' + elif type == 'l': + mail_to = 'lowactivity@indextank.com' + else: + raise Exception('Wrong report type') + + send_mail(ENV + ' - ' + subject, body, 'IndexTank Activity ', [mail_to], fail_silently=False) diff --git a/storefront/lib/monitor.py b/storefront/lib/monitor.py new file mode 100644 index 0000000..3fe9818 --- /dev/null +++ b/storefront/lib/monitor.py @@ -0,0 +1,148 @@ + +from threading import Thread +from traceback import format_tb +import time, datetime +import sys +import shelve + +from django.core.mail import send_mail + +from lib import flaptor_logging + +try: + ENV=open('/data/env.name').readline().strip() +except Exception: + ENV='PROD+EXC' + +#helper functions +def is_prod(): + return ENV == 'PROD' or ENV == 'QoS_Monitor' + +def env_name(): + if ENV == 'PROD': + return 'PRODUCTION' + elif ENV == 'QoS_Monitor': + return 'QoS_Monitor' + else: + return ENV + +class Monitor(Thread): + def __init__(self, pagerduty_email='api-monitor@flaptor.pagerduty.com'): + super(Monitor, self).__init__() + self.name = self.__class__.__name__ + self.statuses = shelve.open('/data/monitor-%s.shelf' % self.name) + self.logger = flaptor_logging.get_logger(self.name) + self.failure_threshold = 1 + self.fatal_failure_threshold = 0 + self.severity = 'WARNING' + self.title_template = '%s::%s: [%s] %s' + self.pagerduty_email = pagerduty_email + + def iterable(self): + return [None] + + def run(self): + self.step = 1 + while True: + starttime = int(time.time()) + try: + self.logger.info("running cycle %d", self.step) + for object in self.iterable(): + self._monitor(object) + self.report_ok("unexpected error in monitor cycle") + self.clean() + except Exception: + self.logger.exception("Unexpected error while executing cycle") + self.report_bad("unexpected error in monitor cycle", 1, 0, 'UNEXPECTED ERROR IN THE CYCLE OF %s\n\n%s' % (self.name, self.describe_error())) + self.step += 1 + self.statuses.sync() + time.sleep(max(0, self.period - (int(time.time()) - starttime))) + + def clean(self): + for title, status in self.statuses.items(): + if not status['working']: + if status['last_update'] != self.step: + self.report_ok(title) + else: + del self.statuses[title] + + + def _monitor(self, object): + try: + if self.monitor(object): + self.report_ok(str(self.alert_title(object))) + else: + self.report_bad(str(self.alert_title(object)), self.failure_threshold, self.fatal_failure_threshold, self.alert_msg(object)) + self.report_ok("unexpected error in monitor") + except Exception, e: + self.logger.exception("Unexpected error while executing monitor. Exception is: %s" % (e)) + message = 'UNEXPECTED ERROR IN THE MONITORING OF %s FOR TITLE: %s\n\n%s' % (self.name, self.alert_title(object), self.describe_error()) + self.report_bad("unexpected error in monitor", 1, 'WARNING', message) + + def describe_error(self): + exc_type, exc_value, exc_traceback = sys.exc_info() + return 'EXCEPTION: %s : %s\ntraceback:\n%s' % (exc_type, exc_value, ''.join(format_tb(exc_traceback))) + + def update_status(self, key, **kwargs): + self.statuses[key] = kwargs + + def send_alert(self, title, message, severity): + try: + if is_prod(): + if severity == 'FATAL': + name = 'FATAL ALERT (%s)' % env_name() + else: + name = 'ALERT (%s)' % env_name() + else: + name = '%s test alert' % ENV + + title = self.title_template % (ENV, self.name, severity, title) + message += '\n\n--------SENT AT ' + str(datetime.datetime.now()) + to = ['alerts@indextank.com'] + if severity == 'FATAL' and is_prod(): + to.append('alerts+fatal@indextank.com') + to.append(self.pagerduty_email) + send_mail(title, message, '"%s" ' % name, to, fail_silently=False) + self.logger.info('Sending alert for title: %s\n============\n%s', title, message) + except Exception, e: + self.logger.exception("Unexpected error while sending alerts. Exception is: %s" % (e)) + + def report_ok(self, title): + if title in self.statuses and not self.statuses[title]['working'] and (self.statuses[title]['alerted'] or self.statuses[title]['alerted_fatal']): + # it has just been resolved + self.send_alert(title, 'The problem is no longer reported. The last message was:\n %s' % (self.statuses[title]['message']), self.severity) + if title in self.statuses: + del self.statuses[title] + + def report_bad(self, title, threshold, fatal_threshold, message): + if title in self.statuses and not self.statuses[title]['working']: + # this object had already failed, let's grab the first step in which it failed + first_failure = self.statuses[title]['first_failure'] + has_alerted = self.statuses[title]['alerted'] + has_alerted_fatal = self.statuses[title]['alerted_fatal'] + else: + # this object was fine, first failure is now + first_failure = self.step + has_alerted = False + has_alerted_fatal = False + + + should_alert = self.step - first_failure + 1 >= threshold + should_alert_fatal = fatal_threshold > 0 and self.step - first_failure + 1 >= fatal_threshold + + if should_alert_fatal: + if not has_alerted_fatal: + has_alerted_fatal = True + if is_prod(): + self.send_alert(title, 'A new problem on IndexTank has been detected:\n %s' % (message), 'FATAL') + else: + self.logger.info('Fatal error was found but alert has already been sent') + elif should_alert: + if not has_alerted: + has_alerted = True + self.send_alert(title, 'A new problem on IndexTank has been detected:\n %s' % (message), self.severity) + else: + self.logger.info('Error was found but alert has already been sent') + + # save current state of the object (is_failed, message, first_failure, last_update) + self.update_status(title, working=False, last_update=self.step, message=message, first_failure=first_failure, alerted=has_alerted, alerted_fatal=has_alerted_fatal) diff --git a/storefront/logging.conf b/storefront/logging.conf new file mode 100644 index 0000000..c09a3b7 --- /dev/null +++ b/storefront/logging.conf @@ -0,0 +1,35 @@ +[loggers] +keys=root,rpc,boto + +[handlers] +keys=consoleHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler + +[logger_rpc] +level=INFO +handlers=consoleHandler +propagate=0 +qualname=RPC + +[logger_boto] +level=INFO +handlers=consoleHandler +propagate=0 +qualname=boto + +[handler_consoleHandler] +class=StreamHandler +formatter=simpleFormatter +args=(sys.stdout,) + +[formatter_simpleFormatter] +format=%(pid)+5s %(asctime)s %(name)+8.8s:%(levelname)s%(prefix)s %(message)-90s %(suffix)s@%(filename)s:%(lineno)s +datefmt=%d/%m-%H.%M.%S +class=logging.SpecialFormatter + diff --git a/storefront/manage.py b/storefront/manage.py new file mode 100644 index 0000000..db37c60 --- /dev/null +++ b/storefront/manage.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +from django.core.management import execute_manager + +from os import environ +from sys import argv + +environ['DJANGO_LOCAL'] = '' + +if argv[1] == 'runserver': + environ['DJANGO_LOCAL'] = '1' +if argv[1].startswith('local'): + environ['DJANGO_LOCAL'] = '1' + argv[1] = argv[1][5:] + +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) + sys.exit(1) + +if __name__ == "__main__": + execute_manager(settings) diff --git a/storefront/management.py b/storefront/management.py new file mode 100644 index 0000000..08316a5 --- /dev/null +++ b/storefront/management.py @@ -0,0 +1,97 @@ +from updates import updates +from django.db.models import signals +from models import create_package, create_analyzer, create_provisioner, AnalyzerComponent, DataSet + +def update_database(app, created_models, verbosity, **kwargs): + updates.perform_updates() + +signals.post_syncdb.connect(update_database) + +def populate_packages(app, created_models, verbosity, **kwargs): + +# snippets=index.allows_snippets, +# facets=index.allows_faceting, +# facets_bits=index.facets_bits, +# autocomplete=index.allows_autocomplete, +# autocomplete_type=index.autocomplete_type, +# variables=index.max_variables, +# rti_size=index.rti_documents_number, +# xmx=index.max_memory_mb) +# + + default_config = {'allows_snippets': True, + 'allows_facets': True, + 'facets_bits': 5, + 'autocomplete': True, + 'autocomplete_type': 'documents', + 'max_variables': 3, + 'rti_size': 500, + 'xmx': 600, + } + def config(dct): + cfg = default_config.copy() + cfg.update(dct) + return cfg + + ########################### CUSTOM PLANS ############################# + create_package(name='Formspring custom', code="FORMSPRING", base_price=0, index_max_size=5000000, searches_per_day=2000, max_indexes=1, + configuration_map=config({'allows_facets': False, 'rti_size': 10000, 'xmx': 13000})) + create_package(name='Spoke custom', code="SPOKE", base_price=0, index_max_size=12000000, searches_per_day=2000, max_indexes=1, + configuration_map=config({'allows_facets': False, 'rti_size': 10000, 'xmx': 26000})) + create_package(name='Reddit custom', code="REDDIT", base_price=0, index_max_size=5000000, searches_per_day=400000, max_indexes=4, + configuration_map=config({ 'allow_snippets': False, 'allows_facets': False, 'autocomplete_type': 'queries', 'xmx': 13000, 'rti_size': 2500, 'vmargs': ['-DlimitTermBasedQueryMatcher=50000,1500'] })) + + ########################## FREE ########################## + create_package(name='Free', code="FREE", base_price=0, index_max_size=100000, searches_per_day=500, max_indexes=5, + configuration_map=config({'allows_facets': True, 'xmx': 700})) + + ########################## PAID PLANS ########################## + create_package(name='Plus', code="PLUS_TANK", base_price=49, index_max_size=500000, searches_per_day=1000, max_indexes=5, + configuration_map=config({'allows_facets': True, 'xmx': 750, 'storage': 'bdb', 'bdb_cache': 50})) + create_package(name='Premium', code="PREMIUM_TANK", base_price=98, index_max_size=1000000, searches_per_day=1000, max_indexes=5, + configuration_map=config({'allows_facets': True, 'xmx': 1200, 'storage': 'bdb', 'bdb_cache': 150})) + create_package(name='Pro', code="PRO_TANK", base_price=175, index_max_size=2000000, searches_per_day=1000, max_indexes=5, + configuration_map=config({'allows_facets': True, 'xmx': 2048, 'storage': 'bdb', 'bdb_cache': 300, 'rti_size': 10000})) + + ########################## INCUBATOR PLANS ########################## + create_package(name='YCombinator 90 days free', code="YCOMBINATOR_90DAYS", base_price=0, index_max_size=2000000, searches_per_day=1000, max_indexes=5, + configuration_map=config({'allows_facets': True, 'xmx': 2048, 'storage': 'bdb', 'bdb_cache': 300, 'rti_size': 10000})) + + ########################## HEROKU PLANS ########################## + create_package(name='Heroku Starter', code="HEROKU_STARTER", base_price=0, index_max_size=100000, searches_per_day=500, max_indexes=5, + configuration_map=config({'allows_facets': True, 'xmx': 700})) + create_package(name='Heroku Plus', code="HEROKU_PLUS", base_price=49, index_max_size=500000, searches_per_day=1000, max_indexes=5, + configuration_map=config({'allows_facets': True, 'xmx': 750, 'storage': 'bdb', 'bdb_cache': 50})) + create_package(name='Heroku Pro', code="HEROKU_PRO", base_price=175, index_max_size=2000000, searches_per_day=1000, max_indexes=5, + configuration_map=config({'allows_facets': True, 'xmx': 2048, 'storage': 'bdb', 'bdb_cache': 300, 'rti_size': 10000})) + + ########################## CONTEST PLANS ########################## + create_package(name='Heroku Contest', code="HEROKU_CONTEST_30DAY", base_price=0, index_max_size=1000000, searches_per_day=20000, max_indexes=5, + configuration_map=config({'max_variables': 5, 'xmx': 1500})) + create_package(name='Factual Contest', code="FACTUAL_CONTEST_30DAY", base_price=0, index_max_size=1000000, searches_per_day=20000, max_indexes=5, + configuration_map=config({'max_variables': 5, 'xmx': 1500})) + create_package(name='Crawlathon', code="80LEGS_CONTEST_30DAY", base_price=0, index_max_size=1000000, searches_per_day=20000, max_indexes=5, + configuration_map=config({'max_variables': 5, 'allows_facets': True, 'xmx': 1200, 'storage': 'bdb', 'bdb_cache': 150})) + +def populate_datasets(app, created_models, verbosity, **kwargs): + datasets = DataSet.objects.filter(code='DEMO').all() + + if len(datasets) == 0: + dataset = DataSet() + dataset.name = 'DEMO' + dataset.code = 'DEMO' + dataset.filename = 'inst.json' + dataset.size = 440 + + dataset.save() + +def populate_analyzers(app, created_models, verbosity, **kwargs): + create_analyzer(code='DEFAULT', name='Default IndexTank analyzer', config='{}', factory='com.flaptor.indextank.query.IndexEngineAnalyzer', type=AnalyzerComponent.Types.tokenizer, enabled=True) + create_analyzer(code='ENG_STOPWORDS', name='Default IndexTank analyzer with english stopwords', config='{"stopwords":["a","about","above","after","again","against","all","am","an","and","any","are","aren\'t","as","at","be","because","been","before","being","below","between","both","but","by","can\'t","cannot","could","couldn\'t","did","didn\'t","do","does","doesn\'t","doing","don\'t","down","during","each","few","for","from","further","had","hadn\'t","has","hasn\'t","have","haven\'t","having","he","he\'d","he\'ll","he\'s","her","here","here\'s","hers","herself","him","himself","his","how","how\'s","i","i\'d","i\'ll","i\'m","i\'ve","if","in","into","is","isn\'t","it","it\'s","its","itself","let\'s","me","more","most","mustn\'t","my","myself","no","nor","not","of","off","on","once","only","or","other","ought","our","ours"," ourselves","out","over","own","same","shan\'t","she","she\'d","she\'ll","she\'s","should","shouldn\'t","so","some","such","than","that","that\'s","the","their","theirs","them","themselves","then","there","there\'s","these","they","they\'d","they\'ll","they\'re","they\'ve","this","those","through","to","too","under","until","up","very","was","wasn\'t","we","we\'d","we\'ll","we\'re","we\'ve","were","weren\'t","what","what\'s","when","when\'s","where","where\'s","which","while","who","who\'s","whom","why","why\'s","with","won\'t","would","wouldn\'t","you","you\'d","you\'ll","you\'re","you\'ve","your","yours","yourself","yourselves"]}', factory='com.flaptor.indextank.query.IndexEngineAnalyzer', type=AnalyzerComponent.Types.tokenizer, enabled=True) + create_analyzer(code='ENGLISH_STEMMER', name='English Stemmer filter', config='{"stemmerName":"English"}', factory='com.flaptor.indextank.query.analyzers.StemmerFilter', type=AnalyzerComponent.Types.filter, enabled=True) + create_analyzer(code='CJK', name='CJK Analyzer', config='{"match_version":"29"}', factory='com.flaptor.indextank.query.IndexEngineCJKAnalyzer', type=AnalyzerComponent.Types.tokenizer, enabled=True) + + +signals.post_syncdb.connect(populate_packages) +signals.post_syncdb.connect(populate_analyzers) +signals.post_syncdb.connect(populate_datasets) diff --git a/storefront/models.py b/storefront/models.py new file mode 100644 index 0000000..cb70b14 --- /dev/null +++ b/storefront/models.py @@ -0,0 +1,2 @@ +from api_linked_models import * + diff --git a/storefront/settings.py b/storefront/settings.py new file mode 100644 index 0000000..7265a00 --- /dev/null +++ b/storefront/settings.py @@ -0,0 +1,117 @@ +# Django settings for RTS project. + +from os import environ, path + +LOCAL = (environ.get('DJANGO_LOCAL') == '1') + +DEBUG = LOCAL +TEMPLATE_DEBUG = DEBUG + +ADMINS = ( + # ('Your Name', 'your_email@domain.com'), +) + +MANAGERS = ADMINS + +DATABASE_ENGINE = 'mysql' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'ado_mssql'. +DATABASE_NAME = 'indextank' # Or path to database file if using sqlite3. +DATABASE_USER = '****' # Not used with sqlite3. +DATABASE_PASSWORD = '****' # Not used with sqlite3. +DATABASE_HOST = 'database' # Set to empty string for localhost. Not used with sqlite3. +DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'Etc/GMT+0' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.load_template_source', + 'django.template.loaders.app_directories.load_template_source', +# 'django.template.loaders.eggs.load_template_source', +) + +MIDDLEWARE_CLASSES = ( + 'django_cookies.CookiePreHandlerMiddleware', + 'storefront.error_logging.ViewErrorLoggingMiddleware', + 'django.middleware.gzip.GZipMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.locale.LocaleMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django_cookies.CookiePostHandlerMiddleware', +) + +ROOT_URLCONF = 'storefront.urls' + +TEMPLATE_CONTEXT_PROCESSORS = ( + 'django.contrib.messages.context_processors.messages', + 'django.contrib.auth.context_processors.auth', + 'django.core.context_processors.request', +) + +MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage' + +TEMPLATE_DIRS = ( + 'templates' + # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.admin', + 'django.contrib.humanize', + 'django.contrib.messages', + 'storefront', +) + +AUTH_PROFILE_MODULE = 'storefront.PFUser' + +AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'storefront.auth.ApiUrlBackend', # if they fail the normal test +) + +STATIC_URLS = [ '/_static' ] +ALLOWED_INCLUDE_ROOTS = ('static') +LOGIN_URL = '/login' + +#PROCS_DIR = '../indexengine/' + +USER_COOKIE_NAME = "pf_user" +COMMON_DOMAIN = 'localhost' +#SESSION_COOKIE_DOMAIN = COMMON_DOMAIN +FORCE_SCRIPT_NAME = '' +USE_MULTITHREADED_SERVER = True +LOGGER_CONFIG_FILE='logging.conf' + +# 0.001 MB per doc +INDEX_SIZE_RATIO = 0.002 + +STORAGE_ENV = 'LOCALTEST' if LOCAL else open('/data/env.name').readline().rstrip("\n") if path.exists('/data/env.name') else 'PROD' + +EMAIL_HOST='localhost' +EMAIL_PORT=252 +EMAIL_HOST_USER='user%localhost' +EMAIL_HOST_PASSWORD='****' + +#Tracking and analytics configuration +CLICKY_SITE_ID = '*********' +ANALYTICAL_INTERNAL_IPS = ['127.0.0.1'] +GOOGLE_ANALYTICS_PROPERTY_ID = 'UA-*******-**' +MIXPANEL_API_TOKEN = '*******************' diff --git a/storefront/start_post_feed.sh b/storefront/start_post_feed.sh new file mode 100755 index 0000000..da11150 --- /dev/null +++ b/storefront/start_post_feed.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +nohup python get_blog_posts.py & diff --git a/storefront/start_webapp.sh b/storefront/start_webapp.sh new file mode 100755 index 0000000..44e6326 --- /dev/null +++ b/storefront/start_webapp.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +LOGFILE="/data/logs/storefront.log" +python manage.py runfcgi method=prefork maxchildren=30 host=127.0.0.1 port=4300 pidfile=pid workdir="$PWD" outlog="$LOGFILE" errlog="$LOGFILE" >>$LOGFILE 2>&1 diff --git a/storefront/static/clippy.swf b/storefront/static/clippy.swf new file mode 100644 index 0000000..e46886c Binary files /dev/null and b/storefront/static/clippy.swf differ diff --git a/storefront/static/common/css/av-hack.css b/storefront/static/common/css/av-hack.css new file mode 100644 index 0000000..e885218 --- /dev/null +++ b/storefront/static/common/css/av-hack.css @@ -0,0 +1,6 @@ +/* CSS Document created by -Pixelcrayons */ +html,body { margin:0;padding:0; height:100%;} +#outer_layout{min-height:100%; height:100%; height:auto; position:relative;} +.bottom_box{height:100%; padding-bottom:160px!important /* Height of the footer */} +#outer_footer {position:absolute; bottom:0; left:0px; float:left; width:100%; /* Height of the footer */} +/* CSS Document created by -Pixelcrayons */ diff --git a/storefront/static/common/css/ie.css b/storefront/static/common/css/ie.css new file mode 100644 index 0000000..99a8091 --- /dev/null +++ b/storefront/static/common/css/ie.css @@ -0,0 +1,2 @@ + + diff --git a/storefront/static/common/css/jquery-ui.css b/storefront/static/common/css/jquery-ui.css new file mode 100644 index 0000000..8f2ab28 --- /dev/null +++ b/storefront/static/common/css/jquery-ui.css @@ -0,0 +1,572 @@ +/* + * jQuery UI CSS Framework 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Theming/API + */ + +/* Layout helpers +----------------------------------*/ +.ui-helper-hidden { display: none; } +.ui-helper-hidden-accessible { position: absolute !important; clip: rect(1px 1px 1px 1px); clip: rect(1px,1px,1px,1px); } +.ui-helper-reset { margin: 0; padding: 0; border: 0; outline: 0; line-height: 1.3; text-decoration: none; font-size: 100%; list-style: none; } +.ui-helper-clearfix:after { content: "."; display: block; height: 0; clear: both; visibility: hidden; } +.ui-helper-clearfix { display: inline-block; } +/* required comment for clearfix to work in Opera \*/ +* html .ui-helper-clearfix { height:1%; } +.ui-helper-clearfix { display:block; } +/* end clearfix */ +.ui-helper-zfix { width: 100%; height: 100%; top: 0; left: 0; position: absolute; opacity: 0; filter:Alpha(Opacity=0); } + + +/* Interaction Cues +----------------------------------*/ +.ui-state-disabled { cursor: default !important; } + + +/* Icons +----------------------------------*/ + +/* states and images */ +.ui-icon { display: block; text-indent: -99999px; overflow: hidden; background-repeat: no-repeat; } + + +/* Misc visuals +----------------------------------*/ + +/* Overlays */ +.ui-widget-overlay { position: absolute; top: 0; left: 0; width: 100%; height: 100%; } + + +/* + * jQuery UI CSS Framework 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Theming/API + * + * To view and modify this theme, visit http://jqueryui.com/themeroller/?ffDefault=Helvetica,%20Arial,%20sans-serif&fwDefault=bold&fsDefault=1.1em&cornerRadius=2px&bgColorHeader=dddddd&bgTextureHeader=03_highlight_soft.png&bgImgOpacityHeader=50&borderColorHeader=dddddd&fcHeader=444444&iconColorHeader=0073ea&bgColorContent=ffffff&bgTextureContent=01_flat.png&bgImgOpacityContent=75&borderColorContent=dddddd&fcContent=444444&iconColorContent=ff0084&bgColorDefault=f6f6f6&bgTextureDefault=03_highlight_soft.png&bgImgOpacityDefault=100&borderColorDefault=dddddd&fcDefault=0073ea&iconColorDefault=666666&bgColorHover=0073ea&bgTextureHover=03_highlight_soft.png&bgImgOpacityHover=25&borderColorHover=0073ea&fcHover=ffffff&iconColorHover=ffffff&bgColorActive=ffffff&bgTextureActive=02_glass.png&bgImgOpacityActive=65&borderColorActive=dddddd&fcActive=ff0084&iconColorActive=454545&bgColorHighlight=ffffff&bgTextureHighlight=01_flat.png&bgImgOpacityHighlight=55&borderColorHighlight=cccccc&fcHighlight=444444&iconColorHighlight=0073ea&bgColorError=ffffff&bgTextureError=01_flat.png&bgImgOpacityError=55&borderColorError=ff0084&fcError=222222&iconColorError=ff0084&bgColorOverlay=eeeeee&bgTextureOverlay=01_flat.png&bgImgOpacityOverlay=0&opacityOverlay=80&bgColorShadow=aaaaaa&bgTextureShadow=01_flat.png&bgImgOpacityShadow=0&opacityShadow=60&thicknessShadow=4px&offsetTopShadow=-4px&offsetLeftShadow=-4px&cornerRadiusShadow=0px + */ + + +/* Component containers +----------------------------------*/ +.ui-widget { font-family: Helvetica, Arial, sans-serif; font-size: 1.1em; } +.ui-widget .ui-widget { font-size: 1em; } +.ui-widget input, .ui-widget select, .ui-widget textarea, .ui-widget button { font-family: Helvetica, Arial, sans-serif; font-size: 1em; } +.ui-widget-content { border: 1px solid #dddddd; background: #ffffff; color: #444444; } +.ui-widget-content a { color: #444444; } +.ui-widget-header { border: 1px solid #dddddd; background: #dddddd url(../images/ui-bg_highlight-soft_50_dddddd_1x100.png) 50% 50% repeat-x; color: #444444; font-weight: bold; } +.ui-widget-header a { color: #444444; } + +/* Interaction states +----------------------------------*/ +.ui-state-default, .ui-widget-content .ui-state-default, .ui-widget-header .ui-state-default { border: 1px solid #dddddd; background: #f6f6f6 url(images/ui-bg_highlight-soft_100_f6f6f6_1x100.png) 50% 50% repeat-x; font-weight: bold; color: #0073ea; } +.ui-state-default a, .ui-state-default a:link, .ui-state-default a:visited { color: #0073ea; text-decoration: none; } +.ui-state-hover, .ui-widget-content .ui-state-hover, .ui-widget-header .ui-state-hover, .ui-state-focus, .ui-widget-content .ui-state-focus, .ui-widget-header .ui-state-focus { border: 1px solid #0073ea; background: #0073ea url(images/ui-bg_highlight-soft_25_0073ea_1x100.png) 50% 50% repeat-x; font-weight: bold; color: #ffffff; } +.ui-state-hover a, .ui-state-hover a:hover { color: #ffffff; text-decoration: none; } +.ui-state-active, .ui-widget-content .ui-state-active, .ui-widget-header .ui-state-active { border: 1px solid #dddddd; background: #ffffff url(images/ui-bg_glass_65_ffffff_1x400.png) 50% 50% repeat-x; font-weight: bold; color: #ff0084; } +.ui-state-active a, .ui-state-active a:link, .ui-state-active a:visited { color: #ff0084; text-decoration: none; } +.ui-widget :active { outline: none; } + +/* Interaction Cues +----------------------------------*/ +.ui-state-highlight, .ui-widget-content .ui-state-highlight, .ui-widget-header .ui-state-highlight {border: 1px solid #cccccc; background: #ffffff url(images/ui-bg_flat_55_ffffff_40x100.png) 50% 50% repeat-x; color: #444444; } +.ui-state-highlight a, .ui-widget-content .ui-state-highlight a,.ui-widget-header .ui-state-highlight a { color: #444444; } +.ui-state-error, .ui-widget-content .ui-state-error, .ui-widget-header .ui-state-error {border: 1px solid #ff0084; background: #ffffff url(images/ui-bg_flat_55_ffffff_40x100.png) 50% 50% repeat-x; color: #222222; } +.ui-state-error a, .ui-widget-content .ui-state-error a, .ui-widget-header .ui-state-error a { color: #222222; } +.ui-state-error-text, .ui-widget-content .ui-state-error-text, .ui-widget-header .ui-state-error-text { color: #222222; } +.ui-priority-primary, .ui-widget-content .ui-priority-primary, .ui-widget-header .ui-priority-primary { font-weight: bold; } +.ui-priority-secondary, .ui-widget-content .ui-priority-secondary, .ui-widget-header .ui-priority-secondary { opacity: .7; filter:Alpha(Opacity=70); font-weight: normal; } +.ui-state-disabled, .ui-widget-content .ui-state-disabled, .ui-widget-header .ui-state-disabled { opacity: .35; filter:Alpha(Opacity=35); background-image: none; } + +/* Icons +----------------------------------*/ + +/* states and images */ +.ui-icon { width: 16px; height: 16px; background-image: url(../images/ui-icons_ff0084_256x240.png); } +.ui-widget-content .ui-icon {background-image: url(../images/ui-icons_ff0084_256x240.png); } +.ui-widget-header .ui-icon {background-image: url(../images/ui-icons_0073ea_256x240.png); } +.ui-state-default .ui-icon { background-image: url(../images/ui-icons_666666_256x240.png); } +.ui-state-hover .ui-icon, .ui-state-focus .ui-icon {background-image: url(../images/ui-icons_ffffff_256x240.png); } +.ui-state-active .ui-icon {background-image: url(../images/ui-icons_454545_256x240.png); } +.ui-state-highlight .ui-icon {background-image: url(../images/ui-icons_0073ea_256x240.png); } +.ui-state-error .ui-icon, .ui-state-error-text .ui-icon {background-image: url(../images/ui-icons_ff0084_256x240.png); } + +/* positioning */ +.ui-icon-carat-1-n { background-position: 0 0; } +.ui-icon-carat-1-ne { background-position: -16px 0; } +.ui-icon-carat-1-e { background-position: -32px 0; } +.ui-icon-carat-1-se { background-position: -48px 0; } +.ui-icon-carat-1-s { background-position: -64px 0; } +.ui-icon-carat-1-sw { background-position: -80px 0; } +.ui-icon-carat-1-w { background-position: -96px 0; } +.ui-icon-carat-1-nw { background-position: -112px 0; } +.ui-icon-carat-2-n-s { background-position: -128px 0; } +.ui-icon-carat-2-e-w { background-position: -144px 0; } +.ui-icon-triangle-1-n { background-position: 0 -16px; } +.ui-icon-triangle-1-ne { background-position: -16px -16px; } +.ui-icon-triangle-1-e { background-position: -32px -16px; } +.ui-icon-triangle-1-se { background-position: -48px -16px; } +.ui-icon-triangle-1-s { background-position: -64px -16px; } +.ui-icon-triangle-1-sw { background-position: -80px -16px; } +.ui-icon-triangle-1-w { background-position: -96px -16px; } +.ui-icon-triangle-1-nw { background-position: -112px -16px; } +.ui-icon-triangle-2-n-s { background-position: -128px -16px; } +.ui-icon-triangle-2-e-w { background-position: -144px -16px; } +.ui-icon-arrow-1-n { background-position: 0 -32px; } +.ui-icon-arrow-1-ne { background-position: -16px -32px; } +.ui-icon-arrow-1-e { background-position: -32px -32px; } +.ui-icon-arrow-1-se { background-position: -48px -32px; } +.ui-icon-arrow-1-s { background-position: -64px -32px; } +.ui-icon-arrow-1-sw { background-position: -80px -32px; } +.ui-icon-arrow-1-w { background-position: -96px -32px; } +.ui-icon-arrow-1-nw { background-position: -112px -32px; } +.ui-icon-arrow-2-n-s { background-position: -128px -32px; } +.ui-icon-arrow-2-ne-sw { background-position: -144px -32px; } +.ui-icon-arrow-2-e-w { background-position: -160px -32px; } +.ui-icon-arrow-2-se-nw { background-position: -176px -32px; } +.ui-icon-arrowstop-1-n { background-position: -192px -32px; } +.ui-icon-arrowstop-1-e { background-position: -208px -32px; } +.ui-icon-arrowstop-1-s { background-position: -224px -32px; } +.ui-icon-arrowstop-1-w { background-position: -240px -32px; } +.ui-icon-arrowthick-1-n { background-position: 0 -48px; } +.ui-icon-arrowthick-1-ne { background-position: -16px -48px; } +.ui-icon-arrowthick-1-e { background-position: -32px -48px; } +.ui-icon-arrowthick-1-se { background-position: -48px -48px; } +.ui-icon-arrowthick-1-s { background-position: -64px -48px; } +.ui-icon-arrowthick-1-sw { background-position: -80px -48px; } +.ui-icon-arrowthick-1-w { background-position: -96px -48px; } +.ui-icon-arrowthick-1-nw { background-position: -112px -48px; } +.ui-icon-arrowthick-2-n-s { background-position: -128px -48px; } +.ui-icon-arrowthick-2-ne-sw { background-position: -144px -48px; } +.ui-icon-arrowthick-2-e-w { background-position: -160px -48px; } +.ui-icon-arrowthick-2-se-nw { background-position: -176px -48px; } +.ui-icon-arrowthickstop-1-n { background-position: -192px -48px; } +.ui-icon-arrowthickstop-1-e { background-position: -208px -48px; } +.ui-icon-arrowthickstop-1-s { background-position: -224px -48px; } +.ui-icon-arrowthickstop-1-w { background-position: -240px -48px; } +.ui-icon-arrowreturnthick-1-w { background-position: 0 -64px; } +.ui-icon-arrowreturnthick-1-n { background-position: -16px -64px; } +.ui-icon-arrowreturnthick-1-e { background-position: -32px -64px; } +.ui-icon-arrowreturnthick-1-s { background-position: -48px -64px; } +.ui-icon-arrowreturn-1-w { background-position: -64px -64px; } +.ui-icon-arrowreturn-1-n { background-position: -80px -64px; } +.ui-icon-arrowreturn-1-e { background-position: -96px -64px; } +.ui-icon-arrowreturn-1-s { background-position: -112px -64px; } +.ui-icon-arrowrefresh-1-w { background-position: -128px -64px; } +.ui-icon-arrowrefresh-1-n { background-position: -144px -64px; } +.ui-icon-arrowrefresh-1-e { background-position: -160px -64px; } +.ui-icon-arrowrefresh-1-s { background-position: -176px -64px; } +.ui-icon-arrow-4 { background-position: 0 -80px; } +.ui-icon-arrow-4-diag { background-position: -16px -80px; } +.ui-icon-extlink { background-position: -32px -80px; } +.ui-icon-newwin { background-position: -48px -80px; } +.ui-icon-refresh { background-position: -64px -80px; } +.ui-icon-shuffle { background-position: -80px -80px; } +.ui-icon-transfer-e-w { background-position: -96px -80px; } +.ui-icon-transferthick-e-w { background-position: -112px -80px; } +.ui-icon-folder-collapsed { background-position: 0 -96px; } +.ui-icon-folder-open { background-position: -16px -96px; } +.ui-icon-document { background-position: -32px -96px; } +.ui-icon-document-b { background-position: -48px -96px; } +.ui-icon-note { background-position: -64px -96px; } +.ui-icon-mail-closed { background-position: -80px -96px; } +.ui-icon-mail-open { background-position: -96px -96px; } +.ui-icon-suitcase { background-position: -112px -96px; } +.ui-icon-comment { background-position: -128px -96px; } +.ui-icon-person { background-position: -144px -96px; } +.ui-icon-print { background-position: -160px -96px; } +.ui-icon-trash { background-position: -176px -96px; } +.ui-icon-locked { background-position: -192px -96px; } +.ui-icon-unlocked { background-position: -208px -96px; } +.ui-icon-bookmark { background-position: -224px -96px; } +.ui-icon-tag { background-position: -240px -96px; } +.ui-icon-home { background-position: 0 -112px; } +.ui-icon-flag { background-position: -16px -112px; } +.ui-icon-calendar { background-position: -32px -112px; } +.ui-icon-cart { background-position: -48px -112px; } +.ui-icon-pencil { background-position: -64px -112px; } +.ui-icon-clock { background-position: -80px -112px; } +.ui-icon-disk { background-position: -96px -112px; } +.ui-icon-calculator { background-position: -112px -112px; } +.ui-icon-zoomin { background-position: -128px -112px; } +.ui-icon-zoomout { background-position: -144px -112px; } +.ui-icon-search { background-position: -160px -112px; } +.ui-icon-wrench { background-position: -176px -112px; } +.ui-icon-gear { background-position: -192px -112px; } +.ui-icon-heart { background-position: -208px -112px; } +.ui-icon-star { background-position: -224px -112px; } +.ui-icon-link { background-position: -240px -112px; } +.ui-icon-cancel { background-position: 0 -128px; } +.ui-icon-plus { background-position: -16px -128px; } +.ui-icon-plusthick { background-position: -32px -128px; } +.ui-icon-minus { background-position: -48px -128px; } +.ui-icon-minusthick { background-position: -64px -128px; } +.ui-icon-close { background-position: -80px -128px; } +.ui-icon-closethick { background-position: -96px -128px; } +.ui-icon-key { background-position: -112px -128px; } +.ui-icon-lightbulb { background-position: -128px -128px; } +.ui-icon-scissors { background-position: -144px -128px; } +.ui-icon-clipboard { background-position: -160px -128px; } +.ui-icon-copy { background-position: -176px -128px; } +.ui-icon-contact { background-position: -192px -128px; } +.ui-icon-image { background-position: -208px -128px; } +.ui-icon-video { background-position: -224px -128px; } +.ui-icon-script { background-position: -240px -128px; } +.ui-icon-alert { background-position: 0 -144px; } +.ui-icon-info { background-position: -16px -144px; } +.ui-icon-notice { background-position: -32px -144px; } +.ui-icon-help { background-position: -48px -144px; } +.ui-icon-check { background-position: -64px -144px; } +.ui-icon-bullet { background-position: -80px -144px; } +.ui-icon-radio-off { background-position: -96px -144px; } +.ui-icon-radio-on { background-position: -112px -144px; } +.ui-icon-pin-w { background-position: -128px -144px; } +.ui-icon-pin-s { background-position: -144px -144px; } +.ui-icon-play { background-position: 0 -160px; } +.ui-icon-pause { background-position: -16px -160px; } +.ui-icon-seek-next { background-position: -32px -160px; } +.ui-icon-seek-prev { background-position: -48px -160px; } +.ui-icon-seek-end { background-position: -64px -160px; } +.ui-icon-seek-start { background-position: -80px -160px; } +/* ui-icon-seek-first is deprecated, use ui-icon-seek-start instead */ +.ui-icon-seek-first { background-position: -80px -160px; } +.ui-icon-stop { background-position: -96px -160px; } +.ui-icon-eject { background-position: -112px -160px; } +.ui-icon-volume-off { background-position: -128px -160px; } +.ui-icon-volume-on { background-position: -144px -160px; } +.ui-icon-power { background-position: 0 -176px; } +.ui-icon-signal-diag { background-position: -16px -176px; } +.ui-icon-signal { background-position: -32px -176px; } +.ui-icon-battery-0 { background-position: -48px -176px; } +.ui-icon-battery-1 { background-position: -64px -176px; } +.ui-icon-battery-2 { background-position: -80px -176px; } +.ui-icon-battery-3 { background-position: -96px -176px; } +.ui-icon-circle-plus { background-position: 0 -192px; } +.ui-icon-circle-minus { background-position: -16px -192px; } +.ui-icon-circle-close { background-position: -32px -192px; } +.ui-icon-circle-triangle-e { background-position: -48px -192px; } +.ui-icon-circle-triangle-s { background-position: -64px -192px; } +.ui-icon-circle-triangle-w { background-position: -80px -192px; } +.ui-icon-circle-triangle-n { background-position: -96px -192px; } +.ui-icon-circle-arrow-e { background-position: -112px -192px; } +.ui-icon-circle-arrow-s { background-position: -128px -192px; } +.ui-icon-circle-arrow-w { background-position: -144px -192px; } +.ui-icon-circle-arrow-n { background-position: -160px -192px; } +.ui-icon-circle-zoomin { background-position: -176px -192px; } +.ui-icon-circle-zoomout { background-position: -192px -192px; } +.ui-icon-circle-check { background-position: -208px -192px; } +.ui-icon-circlesmall-plus { background-position: 0 -208px; } +.ui-icon-circlesmall-minus { background-position: -16px -208px; } +.ui-icon-circlesmall-close { background-position: -32px -208px; } +.ui-icon-squaresmall-plus { background-position: -48px -208px; } +.ui-icon-squaresmall-minus { background-position: -64px -208px; } +.ui-icon-squaresmall-close { background-position: -80px -208px; } +.ui-icon-grip-dotted-vertical { background-position: 0 -224px; } +.ui-icon-grip-dotted-horizontal { background-position: -16px -224px; } +.ui-icon-grip-solid-vertical { background-position: -32px -224px; } +.ui-icon-grip-solid-horizontal { background-position: -48px -224px; } +.ui-icon-gripsmall-diagonal-se { background-position: -64px -224px; } +.ui-icon-grip-diagonal-se { background-position: -80px -224px; } + + +/* Misc visuals +----------------------------------*/ + +/* Corner radius */ +.ui-corner-tl { -moz-border-radius-topleft: 2px; -webkit-border-top-left-radius: 2px; border-top-left-radius: 2px; } +.ui-corner-tr { -moz-border-radius-topright: 2px; -webkit-border-top-right-radius: 2px; border-top-right-radius: 2px; } +.ui-corner-bl { -moz-border-radius-bottomleft: 2px; -webkit-border-bottom-left-radius: 2px; border-bottom-left-radius: 2px; } +.ui-corner-br { -moz-border-radius-bottomright: 2px; -webkit-border-bottom-right-radius: 2px; border-bottom-right-radius: 2px; } +.ui-corner-top { -moz-border-radius-topleft: 2px; -webkit-border-top-left-radius: 2px; border-top-left-radius: 2px; -moz-border-radius-topright: 2px; -webkit-border-top-right-radius: 2px; border-top-right-radius: 2px; } +.ui-corner-bottom { -moz-border-radius-bottomleft: 2px; -webkit-border-bottom-left-radius: 2px; border-bottom-left-radius: 2px; -moz-border-radius-bottomright: 2px; -webkit-border-bottom-right-radius: 2px; border-bottom-right-radius: 2px; } +.ui-corner-right { -moz-border-radius-topright: 2px; -webkit-border-top-right-radius: 2px; border-top-right-radius: 2px; -moz-border-radius-bottomright: 2px; -webkit-border-bottom-right-radius: 2px; border-bottom-right-radius: 2px; } +.ui-corner-left { -moz-border-radius-topleft: 2px; -webkit-border-top-left-radius: 2px; border-top-left-radius: 2px; -moz-border-radius-bottomleft: 2px; -webkit-border-bottom-left-radius: 2px; border-bottom-left-radius: 2px; } +.ui-corner-all { -moz-border-radius: 2px; -webkit-border-radius: 2px; border-radius: 2px; } + +/* Overlays */ +.ui-widget-overlay { background: #eeeeee url(../images/ui-bg_flat_0_eeeeee_40x100.png) 50% 50% repeat-x; opacity: .80;filter:Alpha(Opacity=80); } +.ui-widget-shadow { margin: -4px 0 0 -4px; padding: 4px; background: #aaaaaa url(../images/ui-bg_flat_0_aaaaaa_40x100.png) 50% 50% repeat-x; opacity: .60;filter:Alpha(Opacity=60); -moz-border-radius: 0px; -webkit-border-radius: 0px; border-radius: 0px; }/* + * jQuery UI Resizable 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Resizable#theming + */ +.ui-resizable { position: relative;} +.ui-resizable-handle { position: absolute;font-size: 0.1px;z-index: 99999; display: block;} +.ui-resizable-disabled .ui-resizable-handle, .ui-resizable-autohide .ui-resizable-handle { display: none; } +.ui-resizable-n { cursor: n-resize; height: 7px; width: 100%; top: -5px; left: 0; } +.ui-resizable-s { cursor: s-resize; height: 7px; width: 100%; bottom: -5px; left: 0; } +.ui-resizable-e { cursor: e-resize; width: 7px; right: -5px; top: 0; height: 100%; } +.ui-resizable-w { cursor: w-resize; width: 7px; left: -5px; top: 0; height: 100%; } +.ui-resizable-se { cursor: se-resize; width: 12px; height: 12px; right: 1px; bottom: 1px; } +.ui-resizable-sw { cursor: sw-resize; width: 9px; height: 9px; left: -5px; bottom: -5px; } +.ui-resizable-nw { cursor: nw-resize; width: 9px; height: 9px; left: -5px; top: -5px; } +.ui-resizable-ne { cursor: ne-resize; width: 9px; height: 9px; right: -5px; top: -5px;}/* + * jQuery UI Selectable 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Selectable#theming + */ +.ui-selectable-helper { position: absolute; z-index: 100; border:1px dotted black; } +/* + * jQuery UI Accordion 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Accordion#theming + */ +/* IE/Win - Fix animation bug - #4615 */ +.ui-accordion { width: 100%; } +.ui-accordion .ui-accordion-header { cursor: pointer; position: relative; margin-top: 1px; zoom: 1; } +.ui-accordion .ui-accordion-li-fix { display: inline; } +.ui-accordion .ui-accordion-header-active { border-bottom: 0 !important; } +.ui-accordion .ui-accordion-header a { display: block; font-size: 1em; padding: .5em .5em .5em .7em; } +.ui-accordion-icons .ui-accordion-header a { padding-left: 2.2em; } +.ui-accordion .ui-accordion-header .ui-icon { position: absolute; left: .5em; top: 50%; margin-top: -8px; } +.ui-accordion .ui-accordion-content { padding: 1em 2.2em; border-top: 0; margin-top: -2px; position: relative; top: 1px; margin-bottom: 2px; overflow: auto; display: none; zoom: 1; } +.ui-accordion .ui-accordion-content-active { display: block; }/* + * jQuery UI Autocomplete 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Autocomplete#theming + */ +.ui-autocomplete { position: absolute; cursor: default; } + +/* workarounds */ +* html .ui-autocomplete { width:1px; } /* without this, the menu expands to 100% in IE6 */ + +/* + * jQuery UI Menu 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Menu#theming + */ +.ui-menu { + list-style:none; + padding: 2px; + margin: 0; + display:block; + float: left; +} +.ui-menu .ui-menu { + margin-top: -3px; +} +.ui-menu .ui-menu-item { + margin:0; + padding: 0; + zoom: 1; + float: left; + clear: left; + width: 100%; +} +.ui-menu .ui-menu-item a { + text-decoration:none; + display:block; + padding:.2em .4em; + line-height:1.5; + zoom:1; +} +.ui-menu .ui-menu-item a.ui-state-hover, +.ui-menu .ui-menu-item a.ui-state-active { + font-weight: normal; + margin: -1px; +} +/* + * jQuery UI Button 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Button#theming + */ +.ui-button { display: inline-block; position: relative; padding: 0; margin-right: .1em; text-decoration: none !important; cursor: pointer; text-align: center; zoom: 1; overflow: visible; } /* the overflow property removes extra width in IE */ +.ui-button-icon-only { width: 2.2em; } /* to make room for the icon, a width needs to be set here */ +button.ui-button-icon-only { width: 2.4em; } /* button elements seem to need a little more width */ +.ui-button-icons-only { width: 3.4em; } +button.ui-button-icons-only { width: 3.7em; } + +/*button text element */ +.ui-button .ui-button-text { display: block; line-height: 1.4; } +.ui-button-text-only .ui-button-text { padding: .4em 1em; } +.ui-button-icon-only .ui-button-text, .ui-button-icons-only .ui-button-text { padding: .4em; text-indent: -9999999px; } +.ui-button-text-icon-primary .ui-button-text, .ui-button-text-icons .ui-button-text { padding: .4em 1em .4em 2.1em; } +.ui-button-text-icon-secondary .ui-button-text, .ui-button-text-icons .ui-button-text { padding: .4em 2.1em .4em 1em; } +.ui-button-text-icons .ui-button-text { padding-left: 2.1em; padding-right: 2.1em; } +/* no icon support for input elements, provide padding by default */ +input.ui-button { padding: .4em 1em; } + +/*button icon element(s) */ +.ui-button-icon-only .ui-icon, .ui-button-text-icon-primary .ui-icon, .ui-button-text-icon-secondary .ui-icon, .ui-button-text-icons .ui-icon, .ui-button-icons-only .ui-icon { position: absolute; top: 50%; margin-top: -8px; } +.ui-button-icon-only .ui-icon { left: 50%; margin-left: -8px; } +.ui-button-text-icon-primary .ui-button-icon-primary, .ui-button-text-icons .ui-button-icon-primary, .ui-button-icons-only .ui-button-icon-primary { left: .5em; } +.ui-button-text-icon-secondary .ui-button-icon-secondary, .ui-button-text-icons .ui-button-icon-secondary, .ui-button-icons-only .ui-button-icon-secondary { right: .5em; } +.ui-button-text-icons .ui-button-icon-secondary, .ui-button-icons-only .ui-button-icon-secondary { right: .5em; } + +/*button sets*/ +.ui-buttonset { margin-right: 7px; } +.ui-buttonset .ui-button { margin-left: 0; margin-right: -.3em; } + +/* workarounds */ +button.ui-button::-moz-focus-inner { border: 0; padding: 0; } /* reset extra padding in Firefox */ +/* + * jQuery UI Dialog 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Dialog#theming + */ +.ui-dialog { position: absolute; padding: .2em; width: 300px; overflow: hidden; } +.ui-dialog .ui-dialog-titlebar { padding: .5em 1em .3em; position: relative; } +.ui-dialog .ui-dialog-title { float: left; margin: .1em 16px .2em 0; } +.ui-dialog .ui-dialog-titlebar-close { position: absolute; right: .3em; top: 50%; width: 19px; margin: -10px 0 0 0; padding: 1px; height: 18px; } +.ui-dialog .ui-dialog-titlebar-close span { display: block; margin: 1px; } +.ui-dialog .ui-dialog-titlebar-close:hover, .ui-dialog .ui-dialog-titlebar-close:focus { padding: 0; } +.ui-dialog .ui-dialog-content { position: relative; border: 0; padding: .5em 1em; background: none; overflow: auto; zoom: 1; } +.ui-dialog .ui-dialog-buttonpane { text-align: left; border-width: 1px 0 0 0; background-image: none; margin: .5em 0 0 0; padding: .3em 1em .5em .4em; } +.ui-dialog .ui-dialog-buttonpane .ui-dialog-buttonset { float: right; } +.ui-dialog .ui-dialog-buttonpane button { margin: .5em .4em .5em 0; cursor: pointer; } +.ui-dialog .ui-resizable-se { width: 14px; height: 14px; right: 3px; bottom: 3px; } +.ui-draggable .ui-dialog-titlebar { cursor: move; } +/* + * jQuery UI Slider 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Slider#theming + */ +.ui-slider { position: relative; text-align: left; } +.ui-slider .ui-slider-handle { position: absolute; z-index: 2; width: 1.2em; height: 1.2em; cursor: default; } +.ui-slider .ui-slider-range { position: absolute; z-index: 1; font-size: .7em; display: block; border: 0; background-position: 0 0; } + +.ui-slider-horizontal { height: .8em; } +.ui-slider-horizontal .ui-slider-handle { top: -.3em; margin-left: -.6em; } +.ui-slider-horizontal .ui-slider-range { top: 0; height: 100%; } +.ui-slider-horizontal .ui-slider-range-min { left: 0; } +.ui-slider-horizontal .ui-slider-range-max { right: 0; } + +.ui-slider-vertical { width: .8em; height: 100px; } +.ui-slider-vertical .ui-slider-handle { left: -.3em; margin-left: 0; margin-bottom: -.6em; } +.ui-slider-vertical .ui-slider-range { left: 0; width: 100%; } +.ui-slider-vertical .ui-slider-range-min { bottom: 0; } +.ui-slider-vertical .ui-slider-range-max { top: 0; }/* + * jQuery UI Tabs 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Tabs#theming + */ +.ui-tabs { position: relative; padding: .2em; zoom: 1; } /* position: relative prevents IE scroll bug (element with position: relative inside container with overflow: auto appear as "fixed") */ +.ui-tabs .ui-tabs-nav { margin: 0; padding: .2em .2em 0; } +.ui-tabs .ui-tabs-nav li { list-style: none; float: left; position: relative; top: 1px; margin: 0 .2em 1px 0; border-bottom: 0 !important; padding: 0; white-space: nowrap; } +.ui-tabs .ui-tabs-nav li a { float: left; padding: .5em 1em; text-decoration: none; } +.ui-tabs .ui-tabs-nav li.ui-tabs-selected { margin-bottom: 0; padding-bottom: 1px; } +.ui-tabs .ui-tabs-nav li.ui-tabs-selected a, .ui-tabs .ui-tabs-nav li.ui-state-disabled a, .ui-tabs .ui-tabs-nav li.ui-state-processing a { cursor: text; } +.ui-tabs .ui-tabs-nav li a, .ui-tabs.ui-tabs-collapsible .ui-tabs-nav li.ui-tabs-selected a { cursor: pointer; } /* first selector in group seems obsolete, but required to overcome bug in Opera applying cursor: text overall if defined elsewhere... */ +.ui-tabs .ui-tabs-panel { display: block; border-width: 0; padding: 1em 1.4em; background: none; } +.ui-tabs .ui-tabs-hide { display: none !important; } +/* + * jQuery UI Datepicker 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Datepicker#theming + */ +.ui-datepicker { width: 17em; padding: .2em .2em 0; display: none; } +.ui-datepicker .ui-datepicker-header { position:relative; padding:.2em 0; } +.ui-datepicker .ui-datepicker-prev, .ui-datepicker .ui-datepicker-next { position:absolute; top: 2px; width: 1.8em; height: 1.8em; } +.ui-datepicker .ui-datepicker-prev-hover, .ui-datepicker .ui-datepicker-next-hover { top: 1px; } +.ui-datepicker .ui-datepicker-prev { left:2px; } +.ui-datepicker .ui-datepicker-next { right:2px; } +.ui-datepicker .ui-datepicker-prev-hover { left:1px; } +.ui-datepicker .ui-datepicker-next-hover { right:1px; } +.ui-datepicker .ui-datepicker-prev span, .ui-datepicker .ui-datepicker-next span { display: block; position: absolute; left: 50%; margin-left: -8px; top: 50%; margin-top: -8px; } +.ui-datepicker .ui-datepicker-title { margin: 0 2.3em; line-height: 1.8em; text-align: center; } +.ui-datepicker .ui-datepicker-title select { font-size:1em; margin:1px 0; } +.ui-datepicker select.ui-datepicker-month-year {width: 100%;} +.ui-datepicker select.ui-datepicker-month, +.ui-datepicker select.ui-datepicker-year { width: 49%;} +.ui-datepicker table {width: 100%; font-size: .9em; border-collapse: collapse; margin:0 0 .4em; } +.ui-datepicker th { padding: .7em .3em; text-align: center; font-weight: bold; border: 0; } +.ui-datepicker td { border: 0; padding: 1px; } +.ui-datepicker td span, .ui-datepicker td a { display: block; padding: .2em; text-align: right; text-decoration: none; } +.ui-datepicker .ui-datepicker-buttonpane { background-image: none; margin: .7em 0 0 0; padding:0 .2em; border-left: 0; border-right: 0; border-bottom: 0; } +.ui-datepicker .ui-datepicker-buttonpane button { float: right; margin: .5em .2em .4em; cursor: pointer; padding: .2em .6em .3em .6em; width:auto; overflow:visible; } +.ui-datepicker .ui-datepicker-buttonpane button.ui-datepicker-current { float:left; } + +/* with multiple calendars */ +.ui-datepicker.ui-datepicker-multi { width:auto; } +.ui-datepicker-multi .ui-datepicker-group { float:left; } +.ui-datepicker-multi .ui-datepicker-group table { width:95%; margin:0 auto .4em; } +.ui-datepicker-multi-2 .ui-datepicker-group { width:50%; } +.ui-datepicker-multi-3 .ui-datepicker-group { width:33.3%; } +.ui-datepicker-multi-4 .ui-datepicker-group { width:25%; } +.ui-datepicker-multi .ui-datepicker-group-last .ui-datepicker-header { border-left-width:0; } +.ui-datepicker-multi .ui-datepicker-group-middle .ui-datepicker-header { border-left-width:0; } +.ui-datepicker-multi .ui-datepicker-buttonpane { clear:left; } +.ui-datepicker-row-break { clear:both; width:100%; } + +/* RTL support */ +.ui-datepicker-rtl { direction: rtl; } +.ui-datepicker-rtl .ui-datepicker-prev { right: 2px; left: auto; } +.ui-datepicker-rtl .ui-datepicker-next { left: 2px; right: auto; } +.ui-datepicker-rtl .ui-datepicker-prev:hover { right: 1px; left: auto; } +.ui-datepicker-rtl .ui-datepicker-next:hover { left: 1px; right: auto; } +.ui-datepicker-rtl .ui-datepicker-buttonpane { clear:right; } +.ui-datepicker-rtl .ui-datepicker-buttonpane button { float: left; } +.ui-datepicker-rtl .ui-datepicker-buttonpane button.ui-datepicker-current { float:right; } +.ui-datepicker-rtl .ui-datepicker-group { float:right; } +.ui-datepicker-rtl .ui-datepicker-group-last .ui-datepicker-header { border-right-width:0; border-left-width:1px; } +.ui-datepicker-rtl .ui-datepicker-group-middle .ui-datepicker-header { border-right-width:0; border-left-width:1px; } + +/* IE6 IFRAME FIX (taken from datepicker 1.5.3 */ +.ui-datepicker-cover { + display: none; /*sorry for IE5*/ + display/**/: block; /*sorry for IE5*/ + position: absolute; /*must have*/ + z-index: -1; /*must have*/ + filter: mask(); /*must have*/ + top: -4px; /*must have*/ + left: -4px; /*must have*/ + width: 200px; /*must have*/ + height: 200px; /*must have*/ +}/* + * jQuery UI Progressbar 1.8.7 + * + * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Progressbar#theming + */ +.ui-progressbar { height:2em; text-align: left; } +.ui-progressbar .ui-progressbar-value {margin: -1px; height:100%; } \ No newline at end of file diff --git a/storefront/static/common/css/layout.css b/storefront/static/common/css/layout.css new file mode 100644 index 0000000..6a5cdf9 --- /dev/null +++ b/storefront/static/common/css/layout.css @@ -0,0 +1,2658 @@ + + .messages { + font-size: 16px; + line-height: 20px; + } + + .messages li { + list-style: none; + } + + .info, + .success, + .warning, + .error, + .validation { + border: 2px solid; + margin: 10px 0px 30px; + padding:10px 20px 10px 48px; + background-repeat: no-repeat; + background-position: 5px center; + } + + .info { + color: #00529B; + border-color: #789FCC; + -webkit-box-shadow: 0px 0px 5px #789FCC; + -moz-box-shadow: 0px 0px 5px #789FCC; + background-color: #CDEAF7; + background-image: url('/_static/img/icons/info.png'); + } + + .success { + color: #264409; + border-color: #C6D880; + -webkit-box-shadow: 0px 0px 5px #C6D880; + -moz-box-shadow: 0px 0px 5px #C6D880; + background-color: #E6EFC2; + background-image:url('/_static/img/icons/success.png'); + } + + .warning { + color: #514721; + border-color: #FFD324; + -webkit-box-shadow: 0px 0px 5px #FFD324; + -moz-box-shadow: 0px 0px 5px #FFD324; + background-color: #FFF6BF; + background-image: url('/_static/img/icons/warning.png'); + } + + .error { + color: #8A1F11; + border-color: #FBC2C4; + -webkit-box-shadow: 0px 0px 5px #FBC2C4; + -moz-box-shadow: 0px 0px 5px #FBC2C4; + background-color: #FBE3E4; + background-image: url('/_static/img/icons/error.png'); + } + + + + +div.works_left .code_main pre { font-size: 12px; } + +.screenshot { + padding: 8px; + border: solid 1px silver; + -webkit-box-shadow: 0px 0px 5px silver; + -moz-box-shadow: 0px 0px 5px silver; + margin: 10px; + background: white; +} + +div.works_left .dashboard-index ul {padding: 5px 0; list-style:none;} +div.works_left .dashboard-index ul li {margin-left: 10px; padding: 0; color:#aaa;font-size: 14px;line-height: 18px;} +div.works_left .dashboard-index p {font-size: 14px;line-height: 18px;} +div.works_left .dashboard-index h2 {text-transform:none;} +div.works_left .dashboard-index h2 strong {color:#aaa; font-weight:normal;margin:25px 0; font-size:smaller} +div.works_left .dashboard-index hr {margin:20px 0 0 0; border-collapse: collapse; border:none; border-top:solid 1px #e6e6e6;} +div.works_left .dashboard-index h3 {color:#444;text-transform:uppercase;} +div.works_left .dashboard-index .functions {font-size: 14px; line-height: 18px;} +div.works_left .dashboard-index .functions input {font-size: 14px; line-height: 18px; width: 400px; border: solid 1px #999; color:gray; padding: 4px; } +div.works_left .dashboard-index .functions .button { + background:url(../images/graybutton_whitebg_107.png) no-repeat 0 0; + width:107px; + height:32px; + border:none; + font: bold 14px / 30px "Helvetica", Arial, Helvetica, sans-serif; + color: #2c84b3; + padding:0; +} +div.works_left .dashboard-index .functions .button.cancel { + color: #999; +} +div.works_left .dashboard-index .functions .button:hover { + background-position:0 bottom; +} + +div.works_left .dashboard-index .functions input.fnum {width: 35px;} +div.works_left .dashboard-index .functions td {padding: 3px 6px;} +div.works_left .dashboard-index .functions .function.modified {color: blue;} +div.works_left .dashboard-index a.activesection {color: #9f1e1b; font-weight: bold;} +div.works_left .dashboard-index a.activesection:AFTER {content:' (collapse)'; font-size: 90%; color: #2C84B3; font-weight: normal;} + +div.works_left .dashboard-index .insights .leftcolumn {float:left; width: 160px} +div.works_left .dashboard-index .insights p {margin: 1.4em 0 0 0;} +div.works_left .dashboard-index .insights .rightcolumn {float:right; width: 650px} +div.works_left .dashboard-index .insights .rightcolumn .chart {width: 650px; height: 240px;} + + +div.works_left .search_results {border-collapse: collapse; margin-top: 15px;} +div.works_left .search_results ul {list-style:none;} +div.works_left .search_results ul li {font-size: 12px;line-height: 15px; color: white;} +div.works_left .search_results p {font-size: 12px;line-height: 15px;} +div.works_left .search_results p b {color:yellow;} +div.works_left .search_results td {font-size: 12px;line-height: 15px; padding: 3px 20px; vertical-align: top;} +div.works_left .search_results th {font-size: 12px;line-height: 15px; text-align: left; padding: 6px 2px; color:#9f1e1b; vertical-align: top;} + +div.works_left .search_results td.result {cursor: pointer; overflow-x: hidden; white-space: nowrap; max-width: 350px; width: 350px;} +div.works_left .search_results td.result.active {background: #2C84B3; color:white;} +div.works_left .search_results td.result:hover {background: #9f1e1b; color: white;} +div.works_left .search_results td.fullresult {background: #2C84B3; color:white;max-width: 480px; width: 480px; overflow-x: auto; padding: 10px; -moz-border-radius: 10px; -webkit-border-radius: 10px;} + +div.tos li {list-style: circle; font-size: 15px; line-height: 1.4em; margin: 0px 30px 0.6em; } + +a.twitter { + text-decoration: none; + background: url(../images/icon03.gif) no-repeat 0 0; + padding: 0 0 0 30px; + height: 25px; + float: left; + line-height: 22px; + margin-left: 3px; + display: inline; +} + +.documentation ul, .documentation ol { + padding-left: 0px; +} +.works.documentation ul { + list-style: square; +} + +.documentation ul li { + font-size: 15px; + margin-left: 15px; + padding-left: 5px; + line-height: 1.5em; +} + +.works.dashboard ul { + list-style: none; +} + +.works.dashboard ul li:before { content: '\2013'; padding-right: 8px; } +.works.dashboard ul li { + font-size: 15px; + padding-left: 5px; + line-height: 1.5em; + padding-top: 0; +} + +.works.dashboard a.apiurl { + color:#3B8DB8; + font-size:16px; + letter-spacing: 1px; + vertical-align: bottom; +} + +.dashboard .bottom h2 { + margin: 18px 0 9px; +} + +.documentation ol li { + list-style: decimal; + margin-left: 25px; + margin-bottom: 10px; + color: #9f2318; + font-size: 14px; + font-weight: 600; + font-style: italic; +} +.documentation ol li p, .documentation ol li div, .documentation ol li table { + font-size: 15px; + color: #666; + font-weight: normal; + font-style: normal; +} + +.documentation .formula_title { + font-weight: bold; +} + +a {text-decoration: none;color: #3b8db8;} +a:hover{text-decoration:underline;} +.toplink{float:right;font-size:15px;font-weight:normal;} +.module{clear:both;} +.gist-meta{display: none;} +.gist-file{margin-bottom: 0 !important;border: none !important;} +.gist-data{border: none !important;background: transparent !important;} +input.empty {color: #BBB !important;} + +/*****outer layout starts here*****/ +div#outer_layout {width: 100%;float: left;background: #f5f5f5;} +div.layout {width: 950px;margin: 0 auto;} + +/*****header starts here*****/ +div#outer_header {width:100%;float:left;background: url(../images/header.gif) repeat-x 0 0;} +div#header {width: 950px;float: left;} +div#header h1 {float: left;width: auto;padding: 12px 0 14px 1px;} + +/*****header ends here*****/ +/*****menu starts here*****/ +div#menu {width: auto;float: left;padding: 16px 0 0 45px;} + +div#menu ul {list-style: none;width: auto;float:left;} + +div#menu ul li { + width: auto; + float: left; + background: url(../images/menu_sep.gif) no-repeat 0 4px; + padding: 5px 19px 5px 18px; +} + +div#menu ul li.first { + padding-left: 0; + background: none; +} + +div#menu ul li a { + text-decoration: none; + color: #666; + font: bold 13px / 19px "Whitney HTF", Helvetica, Arial, sans-serif; +} + +div#menu ul li a.current { + color: #3b8db8; +} + +div#menu ul li a.active { + color: #3b8db8; +} + +div#menu ul li a:hover { + color: #9f1e1b; +} + +/*****menu ends here*****/ +/*****search area starts here*****/ +div#search_area { + width: 203px; + float: right; + padding-top: 19px; +} + +div#search_area form { + width: auto; + float: left; +} + +div#search_area fieldset { + width: auto; + float: left; +} + +div#search_area label { + float: left; + width: auto; +} + +div#search_area input { + border: none; + background: none; +} + +div#search_area .search_bg { + background: url(../images/search_bg.gif) no-repeat 0 0; + width: 175px; + float: left; + height: 23px; +} + +div#search_area .search_bg input { + float: left; + width: 168px; + padding: 4px 7px 4px 0; + font: 11px / 16px Arial, Helvetica, sans-serif; + color: #444; + height: 15px; +} + +div#search_area input.search_btn { + background: url(../images/search_btn.gif) no-repeat 0 0; + width: 28px; + height: 23px; + float: left; + cursor: pointer; +} + +.none { + display: none !important; +} + +/*****search area ends here*****/ +/*****head starts here*****/ +div.head_bg { + background: #ffffff; +} + +div.head { + width: 950px; + float: left; + padding: 0 0 20px; +} + +div.head h2 { + font: 52px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #2c84b3; + float: left; + width: auto; +} + +div.head h2 strong { + font-size: 36px; + display: block; + font-weight: normal; + margin-top: 5px; +} + +/*****head ends here*****/ +/*****login area starts here*****/ +div#login_area { + width: auto; + float: right; + padding-top: 2px; +} + +div#login_area ul { + list-style: none; + width: auto; + float: right; + margin-bottom: 9px; +} + +div#login_area ul li { + width: auto; + float: left; + background: url(../images/log_sep.gif) no-repeat 0 4px; + padding: 0 6px 0 9px; + font: 10px / 15px "Helvetica", Arial, Helvetica, sans-serif; + color: #666; +} + +div#login_area ul li strong { + font-weight: normal; + color: #3b8db8; +} + +div#login_area ul li.first { + padding-left: 0; + background: none; +} + +div#login_area ul li.last { + padding-right: 0; +} + +div#login_area ul li a { + text-decoration: none; + color: #3b8db8; + font-weight: bold; +} + +div#login_area ul li a:hover { + text-decoration: underline; +} + +/*****login area ends here*****/ +/*****content section starts here*****/ +div.outer_contaent_sec { + width: 100%; + float: left; + background: #f5f5f5; + padding: 0 0 36px; +} + +div.content_sec { + width: 950px; + float: left; +} + +div.left_content_sec { + width: 402px; + float: left; +} + +div.left_content_sec ul { + list-style: none; + width: 402px; + float: left; + padding-top: 9px; +} + +div.left_content_sec ul li { + width: 100%; + float: left; + padding: 9px 0 0; + font: 14px / 16px "Helvetica", Arial, Helvetica, sans-serif; + color: #737373; +} + +div.left_content_sec ul li h2 { + font: bold 18px / 21px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #9d1813; + display: block; +} + +div.left_content_sec ul li a { + text-decoration: underline; + color: #2c84b3; +} + +div.left_content_sec ul li a:hover { + text-decoration: none; +} + +div.right_content_sec { + width: 464px; + float: right; + position: relative; +} + +.blue_box { + position: absolute; + top: -29px; + right: 8px; + width: 464px; +} + +.blue_box_bg { + background: url(../images/blue_box_bg.gif) repeat-y 0 0; + width: 464px; + float: left; +} + +.blue_box_top { + background: url(../images/blue_box_top.gif) no-repeat 0 0; + width: 464px; + float: left; +} + +.blue_box_bottom { + background: url(../images/blue_box_bottom.gif) no-repeat 0 bottom; + width: 464px; + float: left; +} + +.blue_box_main { + width: 446px; + float: left; + padding: 25px 0 34px 18px; +} + +.blue_box_main h2 { + display: block; + padding: 0 0 0 10px; + margin-bottom: 5px; + font: 36px / 40px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #fff; +} + +.blue_box_main h3 { + display: block; + padding: 0 0 0 14px; + font: 18px / 23px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #fff; +} + +.blue_box_main .red_btn { + width: 100%; + float: left; + margin: 31px 0 9px 17px; + display: inline; + clear: left; +} + +.blue_box_main .red_btn a { + text-decoration: none; + float: left; + color: #fff; + font: bold 22px / 40px "Helvetica", Arial, Helvetica, sans-serif; + text-align: center; + background: url(../images/try_bg01.png) no-repeat 0 0; + width: 198px; + height: 40px; + padding: 0 11px; +} + +.blue_box_main .red_btn a:hover { + background-position: 0 bottom; +} + +.blue_box01 .red_btn { + width: 100%; + float: left; + margin: 31px 0 9px 17px; + display: inline; + clear: left; +} + +.blue_box01 .red_btn a { + text-decoration: none; + float: left; + color: #fff; + font: bold 22px / 40px "Helvetica", Arial, Helvetica, sans-serif; + text-align: center; + background: url(../images/try_bg01.png) no-repeat 0 0; + width: 198px; + height: 40px; + padding: 0 11px; +} + +.blue_box01 .red_btn a:hover { + background-position: 0 bottom; +} + +.blue_box_main ul.icon { + width: 100%; + float: left; + list-style: none; + padding: 23px 0 0 10px; +} + +.blue_box_main ul.icon li { + width: auto; + float: left; + padding-left: 12px; +} + +.blue_box_main ul.icon li a { + text-decoration: none; + background: url(../images/icon01.gif) no-repeat 0 0; + float: left; +} + +.blue_box_main ul.icon li.last { + padding-left: 5px; +} + +.blue_box_main ul.icon li a.icon01 { + background-position: 0px 0px; + width: 27px; + height: 27px; + float: left; +} + +.blue_box_main ul.icon li a.icon02 { + background-position: -40px 0; + width: 28px; + height: 27px; + float: left; +} + +.blue_box_main ul.icon li a.icon03 { + background-position: -77px 0px; + width: 24px; + height: 27px; + float: left; +} + +.blue_box_main ul.icon li a.icon04 { + background-position: -108px 0px; + padding: 0 7px 8px 5px; + width: 35px; + height: 18px; + float: left; +} + +.blue_box_main ul.icon li a span { + display: none; +} + +div.right_content_sec p { + background: url(../images/quote.gif) no-repeat left 247px; + float: left; + width: 425px; + font: italic 24px / 28px "Times New Roman", Times, serif; + text-shadow: 1px 1px 0 #fff; + padding: 253px 0 0 22px; +} + +/*****content section ends here*****/ +/*****tank box starts here*****/ +div.outer_tank_box { + width: 100%; + float: left; + background: #e6e6e6; + border-top: 1px solid #ccc; + border-bottom: 1px solid #ccc; + margin-top: 1px; + padding: 30px 0 32px; +} +div.outer_tank_box.white { + background: white; + border: none; +} + +div.tank_box { + width: 950px; + float: left; + padding: 0 0 0 0; +} + +div.tank_box h2 { + display: block; + font: 18px / 21px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #4c4c4c; +} + +div.white_box { + width: 463px; + float: left; + padding: 12px 0 0; +} + +div.white_box .white_box_bg { + background: url(../images/white_box_bg.gif) repeat-y 0 0; + width: 463px; + float: left; +} + +div.white_box .white_box_top { + background: url(../images/white_box_top.gif) no-repeat 0 0; + width: 463px; + float: left; +} + +div.white_box .white_box_bottom { + background: url(../images/white_box_bottom.gif) no-repeat 0 bottom; + width: 463px; + float: left; +} + +div.white_box .white_box_main { + width: 463px; + float: left; +} + +div.white_box .box_left { + width: 228px; + float: left; +} + +div.white_box .box_left ul { + list-style: none; + width: 228px; + float: left; +} + +div.white_box .box_left ul li { + width: auto; + float: left; +} + +div.white_box .box_left ul li a { + text-decoration: none; + background: url(../images/img01.gif) no-repeat 0 0; + float: left; + font: bold 12px / 18px Arial, Helvetica, sans-serif; + color: #999; +} + +div.white_box .box_left ul li a.news { + background-position: 41px 22px; + width: 150px; + height: 50px; + padding: 5px 67px 35px 11px; + float: left; + border-right: 1px solid #ccc; + border-bottom: 1px solid #ccc; +} + +div.white_box .box_left ul li a.web { + background-position: 42px -70px; + width: 150px; + height: 50px; + padding: 3px 68px 36px 10px; + float: left; + border-right: 1px solid #ccc; + border-bottom: 1px solid #ccc; +} + +div.white_box .box_left ul li a.commerce { + background-position: 42px -159px; + width: 150px; + height: 50px; + padding: 3px 68px 39px 10px; + float: left; + border-right: 1px solid #ccc; +} + +div.white_box .box_left ul li a:hover { + text-decoration: none; +} + +div.white_box .box_right { + width: auto; + float: left; + padding: 0 0 0 0; +} + +div.white_box .box_right ul { + list-style: none; + width: 220px; + float: left; +} + +div.white_box .box_right ul li { + width: auto; + float: left; +} + +div.white_box .box_right ul li a { + text-decoration: none; + background: url(../images/img02.gif) no-repeat 0 0; + float: left; + font: bold 12px / 18px Arial, Helvetica, sans-serif; + color: #999; +} + +div.white_box .box_right ul li a.news { + background-position: 23px 38px; + width: 120px; + height: 50px; + padding: 5px 102px 35px 11px; + float: left; + border-bottom: 1px solid #ccc; +} + +div.white_box .box_right ul li a.web { + background-position: 23px -53px; + width: 120px; + height: 50px; + padding: 3px 102px 36px 10px; + float: left; + border-bottom: 1px solid #ccc; +} + +div.white_box .box_right ul li a.commerce { + background-position: 23px -142px; + width: 120px; + height: 50px; + padding: 3px 68px 39px 10px; + float: left; +} + +div.white_box .box_right ul li a:hover { + text-decoration: none; +} + +.pad_left { + padding-left: 16px !important; +} + +div.white_box .white_box_main01 { + width: 414px; + float: left; + padding: 18px 32px 22px 23px; +} + +div.white_box .white_box_main01 h2 { + font: 24px / 28px Arial, Helvetica, sans-serif; + color: #7f7f7f; + float: left; + width: auto; +} + +div.white_box .white_box_main01 a.tv { + background: url(../images/tv_icon.gif) no-repeat 0 0; + width: 71px; + height: 33px; + float: right; +} + +div.white_box .white_box_main01 .search_tv { + width: 358px; + height: 164px; + float: left; + background: url(../images/right_search_bg.gif) no-repeat 0 0; + margin: 6px 0 0 30px; + display: inline; +} + +div.white_box .white_box_main01 .search_tv form { + width: auto; + float: left; +} + +div.white_box .white_box_main01 .search_tv fieldset { + width: auto; + float: left; +} + +div.white_box .white_box_main01 .search_tv input { + border: none; + background: none; +} + +div.white_box .white_box_main01 .search_tv label { + float: left; + width: auto; +} + +div.white_box .white_box_main01 .search_tv_main { + width: 300px; + float: left; + padding: 66px 0 0 29px; +} + +div.white_box .white_box_main01 .serch_tv_bg { + background: url(../images/tv_search_bg.gif) no-repeat 0 0; + width: 195px; + height: 30px; + float: left; +} + +div.white_box .white_box_main01 .serch_tv_bg input { + font: 14px / 16px Arial, Helvetica, sans-serif; + color: #444; + padding: 6px 5px 7px 20px; + width: 170px; + float: left; +} + +div.white_box .white_box_main01 input.tv_btn { + background: url(../images/tv_search_btn.gif) no-repeat 0 0; + width: 105px; + float: left; + cursor: pointer; + height: 30px; +} + +div.white_box .white_box_main01 p { + font: 14px / 18px Arial, Helvetica, sans-serif; + float: left; + width: auto; + padding: 11px 0 0 9px; +} + +div.white_box .white_box_main01 p a { + text-decoration: none; + color: #727272; +} + +div.white_box .white_box_main01 p a:hover { + text-decoration: underline; +} + +/*****tank box ends here*****/ +/*****bottom box starts here*****/ +div.bottom_box { + width: 950px; + float: left; + padding: 0 0 0 0; +} + +div.home_bottom ul { + list-style: none; + width: 950px; + float: left; +} + +div.home_bottom ul li { + width: 293px; + float: left; + padding-left: 28px; +} + +div.home_bottom ul li.first { + padding-left: 0; +} + +div.home_bottom ul li p { + font: 14px / 20px "Helvetica", Arial, Helvetica, sans-serif; + color: #727272; + padding-top: 8px; +} + +div.home_bottom ul li p strong { + font-weight: bold; + color: #2c84b4; + margin: 6px 0 3px; + display: block; +} + +div.home_bottom ul li p a { + text-decoration: none; + color: #2c84b4; +} +div.home_bottom ul li p strong a { + color: #2c84b4; +} + +div.home_bottom ul li p a:hover { + text-decoration: underline; +} + +div.home_bottom ul li h2 { + font: bold 18px / 21px "Helvetica", Arial, Helvetica, sans-serif; + display: block; + color: #9d1815; +} + +.pad_bot { + padding-bottom: 45px !important; +} + +/*****bottom box ends here*****/ +/*****footer starts here*****/ +div#outer_footer { + width: 100%; + float: left; + background: #2d84b4 url(../images/footer_bg.gif) repeat-x 0 0; + padding: 32px 0 22px; +} + +div#footer { + width: 950px; + float: left; +} + +div#footer ul { + width: 160px; + float: left; + list-style: none; +} + +div#footer ul li { + width: 100%; + float: left; +} + +div#footer ul li a { + text-decoration: none; + font: 12px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #fff; +} + +div#footer ul li p { + float: left; + width: auto; + font: 10px / 15px "Helvetica", Arial, Helvetica, sans-serif; + color: #fff; + padding-top: 10px; +} + +div#footer ul.space { + width: 260px; +} + +div#footer ul li.last { + padding-top: 9px; +} + +div#footer ul li a.blog { + text-decoration: none; + background: url(../images/icon02.gif) no-repeat 0 0; + font-weight: bold; + padding: 0 0 0 33px; + height: 25px; + float: left; + font-size: 12px; + line-height: 26px; +} + +div#footer ul li a.twitter { + font-weight: bold; + font-size: 12px; +} + +div#footer ul li a:hover { + text-decoration: underline; +} + +/*****footer ends here*****/ +/*****why us page starts here*****/ +div.layout01 { + width: 964px; + margin: 0 auto; +} + +div.try_btn { + width: 160px; + float: right; + padding: 8px 12px 0 0; +} + +div.try_btn a { + text-decoration: none; + color: #fff; + font: bold 22px / 40px "Helvetica", Arial, Helvetica, sans-serif; + text-align: center; + background: url(../images/try_bg02.gif) no-repeat 0 0; + width: 132px; + height: 40px; + float: left; + padding: 0 14px; +} + +div.try_btn a:hover { + background-position: 0 bottom; + text-decoration: none; +} + +div#email_error { + float: left; + clear: left; + padding: 3px 10px 10px; + color: red; +} + +div.tab_sec { + width: 964px; + float: left; + padding: 22px 0 0 0; +} + +div.tab_sec h2 { + display: block; + font: 18px / 21px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #9d1815; + font-weight: bold; + padding-left: 14px; +} + +div.tab_sec .tab { + float: left; + width: auto; + padding: 13px 0 0; + position: relative; + z-index: 999; +} + +div.tab_sec .tab ul { + list-style: none; + width: auto; + float: left; +} + +div.tab_sec .tab ul li { + width: auto; + float: left; +} + +div.tab_sec .tab ul li a { + text-decoration: none; + background: url(../images/tab_left_01.gif) no-repeat left 0; + width: auto; + float: left; + padding: 0 0 0 11px; + height: 51px; +} + +div.tab_sec .tab ul li a span { + background: url(../images/tab_right_01.gif) no-repeat right 0; + width: auto; + float: left; + padding: 9px 20px 0 14px; + height: 42px; +} + +div.tab_sec .tab ul li a.first { + text-decoration: none; + background: url(../images/tab_left.gif) no-repeat left 0; + width: auto; + float: left; + padding: 0 0 0 11px; + height: 51px; +} + +div.tab_sec .tab ul li a span.last { + text-decoration: none; + background: url(../images/tab_left01.gif) no-repeat right 0; + width: auto; + float: left; + padding: 9px 20px 0 14px; + height: 42px; +} + +div.tab_sec .tab ul li a:hover { + background-position: 0 bottom; +} + +div.tab_sec .tab ul li a:hover span { + background-position: right bottom; +} + +div.tab_sec .tab ul li a.active { + background-position: 0 bottom; +} + +div.tab_sec .tab ul li a.active span { + background-position: right bottom; +} + +div.tab_sec .tab_content { + width: 964px; + float: left; + position: relative; + z-index: 9; + top: -1px; +} + +div.tab_sec .tab_bg { + background: url(../images/tab_curve_bg.gif) repeat-y 0 0; + width: 964px; + float: left; +} + +div.tab_sec .tab_top { + background: url(../images/tab_top_curve.gif) no-repeat 0 0; + width: 964px; + float: left; +} + +div.tab_sec .tab_bottom { + background: url(../images/tab_bottom_curve.gif) no-repeat 0 bottom; + width: 964px; + float: left; +} + +div.tab_sec .content_main { + width: 920px; + float: left; + padding: 3px 16px 38px 20px; +} + +div.tab_sec .content_main img { + float: right; + margin: 22px 0 0 0; +} + +div.tab_sec .content_main ul { + list-style: none; + width: 280px; + float: left; +} + +div.tab_sec .content_main p { + width: 280px; + float: left; + padding: 0 0 0 5px; + margin-top: 22px; + font: 14px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #727272; +} + +div.tab_sec .content_main blockquote { + width: 280px; + float: left; + padding: 0 0 0 5px; + margin: 10px 0 0 0; + font: italic 14px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #727272; +} + +div.tab_sec .content_main ul li { + width: 242px; + float: left; + background: url(../images/icon001.gif) no-repeat 0 0; + padding: 0 0 0 38px; + margin-top: 22px; + font: 14px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #727272; +} + +div.tab_sec .content_main ul li.second { + background: url(../images/icon002.gif) no-repeat 0 0; +} + +div.tab_sec .content_main ul li.third { + background: url(../images/icon003.gif) no-repeat 0 0; +} + +div.tab_sec .content_main ul li.four { + background: url(../images/icon004.gif) no-repeat 0 0; +} + +div.tab_sec .content_main ul li.five { + background: url(../images/icon005.gif) no-repeat 0 0; +} + +div.tab_sec .content_main ul li.six { + background: url(../images/icon006.gif) no-repeat 0 0; +} + +div.feature_section { + width: 950px; + float: left; +} + +div.feature_section .feature_left { + width: 600px; + float: left; +} + +div.feature_section .feature_box { + width: 600px; + float: left; + padding: 26px 0 0; +} + +div.feature_section .feature_box h2 { + font: bold 18px / 21px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #9d1814; + display: block; +} + +div.feature_section .feature_box ul { + list-style: none; + width: 600px; + float: left; +} + +div.feature_section .feature_box ul li { + width: 100%; + float: left; + padding: 0; +} + +div.feature_section .feature_box ul li h3 { + font: bold 14px / 21px "Helvetica", Arial, Helvetica, sans-serif; + color: #4c4c4c; + display: block; + padding: 16px 0 0; +} + +div.feature_section .feature_box ul li p { + font: 14px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #737373; + display: block; + padding: 9px 0 0; +} + +div.feature_section .feature_right { + width: 304px; + float: right; + padding: 26px 7px 0 0; +} + + +div.feature_section .feature_right .common_box { + width: 304px; + float: left; + background: url(../images/sep.gif) repeat-x 0 bottom; + padding: 22px 0 23px 4px; +} + +div.feature_section .feature_right .common_box h2 { + font: bold 18px / 21px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #9d1815; + display: block; +} + +div.feature_section .feature_right .common_box ul { + list-style: none; + width: 284px; + float: left; + padding: 0 20px 0 0; +} + +div.feature_section .feature_right .common_box ul li { + width: 100%; + float: left; + padding: 15px 0 0; +} + +div.feature_section .feature_right .common_box ul li p { + font: 14px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #737373; + display: block; + clear: left; + padding: 0; +} + +div.feature_section .feature_right .common_box ul li p em { + font-style: normal; + display: block; +} + +div.feature_section .feature_right .common_box ul li p a { + text-decoration: none; + color: #2c84b4; +} + +div.feature_section .feature_right .common_box ul li p a:hover { + text-decoration: underline; +} + +div.feature_section .feature_right .common_box ul li em { + font: italic 14px / 21px "Helvetica", Arial, Helvetica, sans-serif; + color: #737373; + display: block; + clear: left; +} + +div.feature_section .feature_right .common_box ul li strong { + font: bold 14px / 21px "Helvetica", Arial, Helvetica, sans-serif; + color: #666; + display: block; + clear: left; +} + +.none_bg { + background: none !important; +} + +/*****why us page ends here*****/ +/*****works page starts here*****/ +div.works { + width: 950px; + float: left; +} + +div.wide div.works_left { + width: 900px; + float: left; + padding: 18px 0 0; +} +div.works_left { + width: 580px; + float: left; + padding: 18px 0 0; +} + +div.works_left h2 { + font: bold 18px / 21px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #9d1813; + display: block; +} + +div.works_left ul { + list-style: none; + width: 100%; + float: left; +} +div.works_left ol { + width: 100%; + float: left; +} + +div.works_left ul li { + width: 100%; + float: left; + padding: 14px 0 0 0; +} + +div.works_left ol li { + width: 100%; + float: left; +} + +div.works_left ul li h3 { + font: bold 32px / 38px "Helvetica", Arial, Helvetica, sans-serif; + color: #2c84b2; + display: block; + font-weight: normal; +} +/*.documentation div.works_left h2 { + font-size: 26px; + font-family: "Helvetica", Arial, Helvetica, sans-serif + font-weight: normal; + margin: 1.2em 0 .8em 0; +} +.documentation div.works_left h3 { + font-size: 22px; + font-family: "Helvetica", Arial, Helvetica, sans-serif; + font-weight: normal; + margin: .6em 0 .2em 0; +}*/ + +div.works_left h2.first { + margin-top: 0px; +} +div.works_left h2 { + text-transform: uppercase; + font: bold 18px / 21px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #9d1813; + display: block; + font-weight: bold; + margin: 24px 0px 12px; + float: left; + width: 100%; +} + +div.works_left h3 { + /*text-transform: uppercase;*/ + font: bold 16px / 21px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #9d1813; + display: block; + font-weight: bold; + margin: 20px 0px 10px; + float: left; + width: 100%; +} + + +.documentation p { + font-size: 15px; + line-height: 1.4em; + margin: 0 0 .6em 0; +} + +div.works_left p { + font-size: 15px; + line-height: 1.4em; + margin: 0 0 .6em 0; +} + +div.works_left ul li p { + font: 14px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #737373; + display: block; + padding: 14px 0 0 40px; +} + +div.works_left ul li .form_sec { + padding: 17px 0 0 40px; +} + +div.works_left .form_sec { + width: 464px; + float: left; + padding: 5px 0 0 0px; +} + +div.works_left .form_sec form { + width: auto; + float: left; +} + +div.works_left .form_sec fieldset { + width: auto; + float: left; +} + +div.works_left .form_sec input { + border: none; + background: none; +} + +div.works_left .form_sec .select_field .input_bg { + background: none; + width: 282px; + float: left; + height: auto; +} + +div.works_left .form_sec .input_bg { + background: url(../images/input_bg.gif) no-repeat 0 0; + width: 282px; + height: 42px; + float: left; +} + +div.works_left .form_sec .input_bg select { + font: 18px / 20px "Helvetica", Arial, Helvetica, sans-serif; + color: #444; + padding: 10px 4px 8px 0px; + width: 282px; + float: left; + border: solid 1px gray; + height: 42px; + text-indent: 4px; +} + +div.works_left .form_sec .input_bg input { + font: 18px / 20px "Helvetica", Arial, Helvetica, sans-serif; + color: #444; + padding: 10px 4px 8px 14px; + width: 264px; + float: left; +} + +div.works_left .form_sec input.signup { + background: url(../images/sign_bg.gif) no-repeat 0 0; + width: 120px; + height: 40px; + margin: 0 0 0 10px; + display: inline; + cursor: pointer; + float: left; + font: bold 20px / 24px "Helvetica", Arial, Helvetica, sans-serif; + color: #fff; +} +div.works_left .form_sec .buttons input.signup { + margin: 0 10px 0 0; +} +div.works_left .form_sec .buttons { + margin-top: 10px; + display:block; +} +div.works_left .form_sec .buttons a { + display: block; + float: left; + font-size: 15px; + line-height: 20px; + padding: 10px 0; + vertical-align: middle; +} + +div.works_left .form_sec input.signup:hover { + background: url(../images/sign_bg_over.gif) no-repeat 0 0; + width: 120px; + height: 40px; + display: inline; + cursor: pointer; + float: left; + font: bold 20px / 24px "Helvetica", Arial, Helvetica, sans-serif; + color: #fff; +} + +div.works_left ul li .work_tab { + width: 464px; + float: left; + padding: 12px 0 0 40px; + position: relative; + z-index: 999 +} + +div.works_left ul li .work_tab ul { + list-style: none; + width: 504px; + float: left; +} + +div.works_left ul li .work_tab ul li { + width: auto; + float: left; + padding: 0; +} + + +div.works_left ul li .work_tab ul li a { + text-decoration: none; + float: left; + width: auto; + padding: 0 0 0 5px; + height: 61px; +} + +div.works_left ul li .work_tab ul li a span { + width: auto; + float: left; + height: 50px; + padding: 11px 18px 0 13px; +} + +div.works_left ul li .work_tab ul li a:hover { + background: url(../images/work_tab_left.gif) no-repeat left 0; +} + +div.works_left ul li .work_tab ul li a:hover span { + background: url(../images/work_tab_right.gif) no-repeat right 0; +} + +div.works_left ul li .work_tab ul li a.active { + background: url(../images/work_tab_left.gif) no-repeat left 0; +} + +div.works_left ul li .work_tab ul li a.active span { + background: url(../images/work_tab_right.gif) no-repeat right 0; +} + +div.works_left ul li .work { + width: 504px; + float: left; + position: relative; + z-index: 9; + top: -2px; + padding: 0 0 0 40px; +} + +div.works_left .work_top { + background: url(../images/work504_top01.gif) no-repeat 0 0; + width: 504px; + float: left; +} + +div.works_left .work_bottom { + background: url(../images/work504_bottom.gif) no-repeat 0 bottom; + width: 504px; + float: left; +} + +div.works_left .work_bg { + background: url(../images/work504_bg.gif) repeat-y 0 0; + width: 504px; + float: left; +} + +div.works_left .work_main { + float: left; + width: auto; + padding: 16px 24px 20px 20px; +} + +div.works_left ul li .work_main p { + font: 12px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #737373; + display: block; + padding: 0; +} + +div.works_left p.text { + font: 11px / 15px "Helvetica", Arial, Helvetica, sans-serif; + color: #737373; + display: block; + padding: 0; +} + +div.works_left p.text em { + font: normal 10px / 12px Arial, Helvetica, sans-serif; + color: #388bb7; +} + +.pad { + padding-top: 18px !important; +} + +div.works_left ul li .commands { + padding-left: 40px; +} + +.commands { + width: 504px; + float: left; + padding: 12px 25px 8px 0; +} + +.commands_top { + background: url(../images/work504_top01.gif) no-repeat 0 0 !important; + width: 504px; + float: left; +} + +.commands_bottom { + background: url(../images/work504_bottom.gif) no-repeat 0 bottom; + width: 504px; + float: left; +} + +.commands_bg { + background: url(../images/work504_bg.gif) repeat-y 0 0; + width: 504px; + float: left; +} + +.commands_main { + float: left; + width: 494px; + margin: 0 5px 5px 5px; + overflow-x: auto; +} + +.commands pre { + margin: 15px 0px 15px 15px; +} + +.form { + width: 504px; + float: left; + padding: 12px 0 8px 0; +} + +.form_top { + background: url(../images/work504_top01.gif) no-repeat 0 0 !important; + width: 504px; + float: left; +} + +.form_bottom { + background: url(../images/work504_bottom.gif) no-repeat 0 bottom; + width: 504px; + float: left; +} + +.form_bg { + background: url(../images/work504_bg.gif) repeat-y 0 0; + width: 504px; + float: left; +} + +.form_main { + float: left; + width: 444px; + padding: 25px 30px; +} + +.largebox { + width: 900px; + float: left; + margin: 10px 0; +} + +.largebox .largebox_top { + background: url(../images/largebox_top.gif) no-repeat 0 0 !important; + width: 900px; + float: left; +} + +.largebox .largebox_bottom { + background: url(../images/largebox_bottom.gif) no-repeat 0 bottom; + width: 900px; + float: left; +} + +.largebox .largebox_bg { + background: url(../images/largebox_bg.gif) repeat-y 0 0; + width: 900px; + float: left; +} + +.largebox .largebox_main { + float: left; + width: 850px; + margin: 0 5px 5px; + padding: 20px; + overflow-x: auto; +} + +.largebox .largebox_main h2.first { + margin-top: 0; +} + +.documentation .largebox .largebox_main { + text-align: center; +} + +.mediumbox { + width: 780px; + float: left; + margin: 10px 0; +} + +.mediumbox .mediumbox_top { + background: url(../images/mediumbox_top.gif) no-repeat 0 0 !important; + width: 780px; + float: left; +} + +.mediumbox .mediumbox_bottom { + background: url(../images/mediumbox_bottom.gif) no-repeat 0 bottom; + width: 780px; + float: left; +} + +.mediumbox .mediumbox_bg { + background: url(../images/mediumbox_bg.gif) repeat-y 0 0; + width: 780px; + float: left; +} + +.mediumbox .mediumbox_main { + float: left; + width: 730px; + margin: 5px 5px 5px 5px; + padding: 20px; + overflow-x: auto; +} + +.documentation .mediumbox .mediumbox_main { + text-align: center; +} + +.documentation .sideNote { + margin: 10px -270px 10px 0px; + float: right; + display: block; + width: 190px; + padding: 8px 15px; + border: solid 1px #909090; + -webkit-box-shadow: 0px 0px 5px #444444; + -moz-box-shadow: 0px 0px 5px #444444; + background-color: #d8eeff; + background: -webkit-gradient(linear, left top, left bottom, from(#d8eeff), to(#c8deff)); /* for safari/chrome */ + background: -moz-linear-gradient(top, #efefef, #e5e5e5); /* for firefox 3.6+ */ + font: 12px/16px "Helvetica",Arial,Helvetica,sans-serif; +} + +.documentation .sideNote b { + color: #DB0000; +} + +.documentation table { + float: left; + border-collapse: collapse; + margin: 10px 50px 10px 10px; + font-size: 14px; + overflow: auto; + background-color: #fafafa; + padding: 8px; + border: solid 1px silver; + -webkit-box-shadow: 0px 0px 5px silver; + -moz-box-shadow: 0px 0px 5px silver; +} + +.documentation td, .documentation th { + padding: 4px 8px; + border: solid 1px silver; + text-align: left; +} + +.documentation th { + background: #dcdcdc; + background: -webkit-gradient(linear, left top, left bottom, from(#f3f3f3), to(#e0e0e0)); /* for safari/chrome */ + background: -moz-linear-gradient(top, #efefef, #e5e5e5); /* for firefox 3.6+ */ + text-transform: uppercase; + font-weight: bold; + color: #434343; + text-shadow: 0px 0px 3px white; + border: none; +} + + +.documentation code { + font-size: 12px; +} + +.documentation .indextable { + margin-top: 25px; + font-size: 13px; +} + +.documentation .indextable td { + padding: 10px 20px 10px 10px; +} + +.documentation .indextable td .first { + width: 400px; +} + +.urlcode { + color: #9D1813; + font-weight: bold; + font-size: 15px; + margin-bottom: 6px; + margin-top: 2px; +} + +.apidesc { + font-size: 12px; + padding: 5px; +} + +.documentation .apidesc ul li { + font-size: 12px; + padding: 5px 0 5px; +} + +.documentation .facets table { + width: 350px; +} + +.documentation .facets th { + padding-top: 9px; +} + +.documentation .facets ul li { + list-style: square; + padding: 2px; + font-size: 12px; + color: #3333DD; + text-decoration: underline; +} + +.documentation th.first, .documentation th.last { + padding: 7px; +} + +.documentation td.first, .documentation td.last { + vertical-align: top; +} + +.works div.tips .table ul { + width: 240px; +} + +.works div.tips .table ul li h2 { + margin-top: 4px; + color: #9D1813; +} + +.works div.tips .table ul li.first { + list-style: none; + padding-bottom: 0px; + margin-left: 10px; +} + +.works div.tips .table ul li { + font-size: 12px; + list-style: square; + padding-bottom: 10px; + margin-left: 35px; +} +.code { + width: 580px; + float: left; + margin: 10px 0; +} + +.code .code_top { + background: url(../images/work_top01.gif) no-repeat 0 0 !important; + width: 574px; + float: left; +} + +.code .code_bottom { + background: url(../images/work_bottom.gif) no-repeat 0 bottom; + width: 574px; + float: left; +} + +.code .code_bg { + background: url(../images/work_bg.gif) repeat-y 0 0; + width: 574px; + float: left; +} + +.code .code_main { + float: left; + width: 564px; + margin: 0 5px 5px; + overflow-x: auto; +} + +.code pre { + margin: 15px 0px 15px 15px; +} + +/*.code:hover, .code:hover .code_top, .code:hover .code_bg, .code:hover .code_bottom{width:804px;} + .code:hover .code_main{width:794px;}*/ +div.works_right { + width: 304px; + float: right; + padding: 26px 7px 0 0; +} + +div.blue_box01 { + width: 300px; + float: left; + padding: 0 0 18px; +} + +div.blue_box01_top { + background: url(../images/blue_box_top01.gif) no-repeat 0 0; + width: 304px; + float: left; +} + +div.blue_box01_bottom { + background: url(../images/blue_box_bottom01.gif) no-repeat 0 bottom; + width: 304px; + float: left; +} + +div.blue_box01_bg { + background: url(../images/blue_box_bg01.gif) repeat-y 0 0; + width: 304px; + float: left; +} + +div.blue_box01_main { + width: auto; + float: left; + padding: 34px 15px 40px 23px; +} + +div.blue_box01_main p { + font: 23px / 30px "Helvetica", Arial, Helvetica, sans-serif; + color: #fff; + display: block; + padding: 0; +} + +div.blue_box01_main p a { + text-decoration: underline; + color: #fff; +} + +div.blue_box01_main p a:hover { + text-decoration: none; +} + +div.blue_box01_main p.size { + font-size: 18px; +} + +div.works_left div.apiurl { + padding: 10px 15px; +} + +div.works_left div.apiurl h4 { + font-size: 16px; + margin-bottom: 4px; +} + +div#account_created span.user_email { + color: #9d1813; + font-weight: bold; +} + +/*****works page ends here*****/ +/*****details page starts here*****/ +div.details { + width: 504px; + float: left; +} + +div.details h3 { + font: bold 14px / 21px "Helvetica", Arial, Helvetica, sans-serif; + color: #4c4c4c; + padding-top: 15px; + float: left; + width: 100%; +} + +div.details p { + font: 14px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #727272; + padding-top: 15px; + float: left; + width: 100%; +} + +div.details p em { + font-style: normal; + color: #2c84b2; +} + +div.details p em a { + font-style: normal; + color: #2c84b2; + text-decoration: none; +} + +div.details p em a:hover { + text-decoration: underline; +} + +div.details p.install { + font: 12px / 15px "Helvetica", Arial, Helvetica, sans-serif; + color: #727272; + padding: 0; + float: left; + width: 100%; +} + +.pad01 { + padding: 16px 20px !important; +} + +.pad02 { + padding-top: 18px !important; + padding-left: 0 !important; +} + +div.table { + width: 304px; + float: left; + padding: 17px 0 25px; +} +div.works_left div.table { + padding: 0; +} + +div.table_bg { + background: url(../images/table_bg.gif) repeat-y 0 0; + width: 304px; + float: left; +} + +div.table_top { + background: url(../images/table_top.gif) no-repeat 0 0; + width: 304px; + float: left; +} + +div.table_bottom { + background: url(../images/table_bottom.gif) no-repeat 0 bottom; + width: 304px; + float: left; +} + +.works div.table ul { + list-style: none; + width: 280px; + padding: 16px 0 22px 10px; + float: left; +} + +.works div.table ul li { + width: 100%; + float: left; + padding: 0 0 0 0; +} + +.works div.table ul li h2 { + font: bold 14px / 26px "Helvetica", Arial, Helvetica, sans-serif; + color: #4c4c4c; + display: block; + padding: 0 0 .3em 0; +} + +.works div.table ul li a { + font: 14px / 22px "Helvetica", Arial, Helvetica, sans-serif; +} + +.works div.table ul li a:hover { + text-decoration: none; +} + +/*****details page ends here*****/ +/*****documentation page starts here*****/ +div.document_left { + width: 620px; + float: left; + padding: 22px 0 0; +} + +div.document_left h2 { + font: 18px / 21px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #9d1815; + display: block; + font-weight: bold; +} + +/*div.document_left */.doc_row { + width: 620px; + float: left; + background: url(../images/sep.gif) repeat-x 0 bottom; + padding: 0 0 38px; +} + +div.document_left .doc_col01 { + width: 320px; + float: left; +} + +div.document_left .doc_col02 { + width: 275px; + float: left; +} + +div.document_left .doc_row h3 { + font: bold 14px / 21px "Helvetica", Arial, Helvetica, sans-serif; + color: #4c4c4c; + float: left; + width: 100%; + padding-top: 16px; +} + +div.document_left .doc_row ul { + list-style: none; + width: 100%; + float: left; + padding-top: 2px; +} + +div.document_left .doc_row ul li { + width: auto; + float: left; + font: 14px / 21px "Helvetica", Arial, Helvetica, sans-serif; + color: #4c4c4c; + background: url(../images/doc_sep.gif) no-repeat 0 5px; + padding: 0 5px 0 7px; +} + +div.document_left .doc_row ul li.first { + padding-left: 0; + background: none; +} + +div.document_left .doc_row ul li.last { + background: none; +} + +div.document_left .doc_row ul li a { + text-decoration: underline; + color: #2c84b2; +} + +div.document_left .doc_row ul li a:hover { + text-decoration: none; +} + +div.document_left .latest { + width: 620px; + float: left; + padding-top: 20px; +} + +div.document_left .latest ul { + list-style: none; + width: 620px; + float: left; +} + +div.document_left .latest ul li { + width: auto; + float: left; + padding: 14px 0 0 0; +} + +div.document_left .latest ul li strong { + font: bold 14px / 21px "Helvetica", Arial, Helvetica, sans-serif; + color: #2c84b2; + display: block; +} + +div.document_left .latest ul li strong em { + color: #727272; + font-weight: normal; +} + +div.document_left .latest ul li p { + font: 14px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #727272; + display: block; + padding-top: 6px; +} + +div.photo_sec { + width: auto; + float: left; + padding: 2px 0 0 0; +} + +div.photo_sec ul.pict { + width: 265px; + float: left; + list-style: none; + padding-top: 0px; +} + +div.photo_sec ul.pict li { + width: auto; + float: left; + padding-left: 9px; + display: list-item; + clear:none; +} + +div.photo_sec ul.pict li.first { + padding-left: 0; +} + +div.photo_sec ul.pict li img { + float: left; + width: 80px; + height: 80px; +} + +div.follow { + width: 200px; + float: left; +} + +div.follow ul { + list-style: none; + width: 200px; + float: left; +} + +div.follow ul li { + width: 200px; + float: left; + padding: 9px 0 0 0; + font: 12px / 18px Arial, Helvetica, sans-serif; + color: #fff; +} + +div.follow ul li a { + text-decoration: none; + color: #fff; +} + +div.follow ul li a.blog { + text-decoration: none; + background: url(../images/icon02.gif) no-repeat 0 0; + padding: 0 0 0 33px; + height: 25px; + float: left; + line-height: 26px; +} + + +div.photo_sec h3 { + font: bold 14px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #fff; + float: left; + width: 100%; + padding: 8px 0 5px; +} + +.pad03 { + padding-bottom: 20px !important; +} + +.pad04 { + padding-top: 10px !important; +} + +.pad05 { + padding-bottom: 0px !important; +} + +/*****documentation page ends here*****/ +/*****popup starts here*****/ +div.popup { + position: absolute; + top: 0; + left: 0; + width: 782px; +} + +div.popup_main { + width: 782px; + float: left; +} + +div.popup_main .popup_repeat { + background: url(../images/popup_repeat.gif) repeat-y 0 0; + width: 782px; + float: left; +} + +div.popup_main .popup_top { + background: url(../images/popup_top.gif) no-repeat 0 0; + width: 782px; + float: left; +} + +div.popup_main .popup_bottom { + background: url(../images/popup_bottom.gif) no-repeat 0 bottom; + width: 782px; + float: left; +} + +div.popup_main .poup_inner { + width: 731px; + float: left; + padding: 14px 15px 34px 36px; +} + +div.popup_main .poup_inner a.close { + background: url(../images/close_btn.gif) no-repeat 0 0; + width: 22px; + height: 22px; + float: right; +} + +div.popup_main .poup_inner a.close span { + display: none; +} + +div.popup_main .poup_inner h2 { + font: 52px / 54px Arial, Helvetica, sans-serif; + color: #fff; + display: block; + padding-top: 20px; +} + +div.popup_main .poup_inner ul { + list-style: none; + width: 731px; + float: left; + padding-top: 40px; +} + +div.popup_main .poup_inner ul li { + width: 214px; + float: left; + padding-left: 26px; +} + +div.popup_main .poup_inner ul li.first { + padding-left: 0; +} + +div.popup_main .poup_inner ul li h3 { + font: 18px / 21px Arial, Helvetica, sans-serif; + color: #fff; + display: block; + padding-bottom: 14px; +} + +div.popup_main .poup_inner ul li p { + font: 14px / 18px Arial, Helvetica, sans-serif; + color: #333; + display: block; +} + +.pad_right { + padding: 25px 0 0 0 !important; +} + +.pad_none { + padding-top: 0 !important; +} + +.left_btn01 { + background: url(../images/pop_btn_left.gif) no-repeat left 0 !important; +} + +.right_btn01 { + background: url(../images/pop_btn_right.gif) no-repeat right 0 !important; +} + +.btn01_bg { + background: url(../images/pop_btn_bg.gif) repeat-x 0 0 !important; +} + +/*****popup ends here*****/ +/*****outer layout ends here*****/ + +div.chat_box, div.chatsimple_box { + width: 304px; + float: left; +} + +div.chat_top { + background: url(../images/chat_top.gif) no-repeat 0 0; + width: 304px; + float: left; +} + +div.chatsimple_top { + background: url(../images/chat_top01.gif) no-repeat 0 0; + width: 304px; + float: left; +} + +div.chat_bottom, div.chatsimple_bottom { + background: url(../images/chat_bottom.gif) no-repeat 0 bottom; + width: 304px; + float: left; +} + +div.chat_bg, div.chatsimple_bg { + background: url(../images/chat_bg.gif) repeat-y 0 0; + width: 304px; + float: left; +} + +div.chat_main { + width: auto; + float: left; + padding: 19px 22px 132px 22px; +} + +div.chatsimple_main { + width: auto; + float: left; + padding: 19px 22px 20px 22px; +} + +div.chat_main h2, div.chatsimple_main h2 { + font: bold 18px / 21px "Whitney HTF", Arial, Helvetica, sans-serif; + color: #fff; + display: block; + margin: 0; +} + +div.chat_main p, div.chatsimple_main p { + font: 14px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #fff; + display: block; + clear: left; +} + +div.chat_main ul, div.chatsimple_main ul { + padding-left: 0; + margin-top: 9px; +} +div.chat_main ul li, div.chatsimple_main ul li { + font: 14px / 18px "Helvetica", Arial, Helvetica, sans-serif; + color: #fff; + display: block; + clear: left; + padding-left: 0; + margin-left: 0; +} + +div.chat_main a, div.chatsimple_main a { + text-decoration: underline; + color: #fff; + font-weight: bold; +} +div.chat_main a:hover, div.chatsimple_main a:hover { + text-decoration: none; +} + +div.chat_btn { + width: 160px; + float: left; + padding: 13px 0 14px; +} + +div.chat_btn a { + text-decoration: none; + float: left; + text-align: center; + font: bold 14px / 30px "Helvetica", Arial, Helvetica, sans-serif; + color: #2c84b3; + margin-top: 7px; + background: url(../images/chat_bg01.gif) no-repeat 0 0; + width: 160px; + height: 30px; +} + +div.chat_btn a:hover { + background-position: 0 bottom; +} + + +/** Staff page starts here */ +.staff-member blockquote { + width: 270px; + float: right; + background: #ffffff url(../img/structure/ballon.gif) no-repeat top left; + padding: 10px 10px 10px 30px; + font-family: Georgia, Times, serif; + font-style: italic; + margin-bottom: 5px; +} + +div.staff-member-contact a.twitter { + margin-top: 14px; +} + +div.staff-member { + padding-bottom: 10px; +} +/** Staff page ends here */ + +table.pricing { + border-collapse: collapse; + color: gray; + width: 100%; +} +table.pricing th, table.pricing td { + border: solid 1px #ddd; + padding: 5px 12px; + text-align: center; + width: 15%; + font-size: 16px; + line-height: 24px; +} + +table.pricing th { + font-size: 20px; + line-height: 35px; + border-top: none; + padding-bottom: 15px; +} +table.pricing th strong { + display: block; + font-weight: normal; + font-size: 14px; + line-height: 12px; +} + +table.pricing tr.last td { + border-bottom: none; +} + +table.pricing th.starter { + color: #77b45b; +} + +table.pricing th.plus { + color: #DCA632; +} + +table.pricing th.custom { + color: #4DA0C2; +} + +table.pricing th.pro { + color: #666666; +} + +table.pricing tr.highlight td { + font-weight: bold; +} + +table.pricing td.first, table.pricing th.first { + border-left: none; + text-align: left; + width: 25%; +} + +table.pricing td.last, table.pricing th.last { + border-right: none; +} + +table.pricing .signup { + background: url(../images/sign_bg.gif) no-repeat 0 0; + width: 120px; + margin: 6px 0 6px 6px; + display: inline; + cursor: pointer; + float: left; + padding: 8px 0; + font: bold 20px / 24px "Helvetica", Arial, Helvetica, sans-serif; + color: #fff; +} + +table.pricing .signup:hover { + text-decoration: none; + background: url(../images/sign_bg_over.gif) no-repeat 0 0; +} + +.itsfree { + background: url(../images/itsfree.png) no-repeat 0 0; + display: block; + width: 903px; + height: 97px; + margin: 24px -5px; +} + +div.pricing_userlist h2 { + color: #333 !important; + margin-top: 0 !important; +} + +div.pricing_userlist .userlist { + display: inline-block; + width: 150px; + height: 50px; + background: url(../images/img01.gif) no-repeat; + padding: 0 10px; +} + +div.pricing_userlist .userlist.user1 { + background-position: 0 -90px; +} +div.pricing_userlist .userlist.user2 { + background-position: 0 -180px; +} +div.pricing_userlist .userlist.user3 { + background: url(../images/img02.gif) no-repeat -10px 10px; + width: 130px; +} +div.pricing_userlist .userlist.user4 { + background: url(../images/img02.gif) no-repeat; + background-position: 0 -80px; +} diff --git a/storefront/static/common/css/reset.css b/storefront/static/common/css/reset.css new file mode 100644 index 0000000..076f350 --- /dev/null +++ b/storefront/static/common/css/reset.css @@ -0,0 +1,13 @@ +body, p, form, fieldset, h1, h2, h3, h4, h5, h6,ul,li,ols{margin:0; padding:0} +form, fieldset, img{border:none;} +*{outline:none} +body{font:12px/16px "Helvetica",Arial, Helvetica, sans-serif; color:#666;} +.left{float:left !important;} +.right{float:right !important;} +.clear{clear:both} +a{text-decoration:none; cursor:pointer;} +a:hover{text-decoration:underline;} + + + + diff --git a/storefront/static/common/css/shadow.css b/storefront/static/common/css/shadow.css new file mode 100644 index 0000000..d36a34b --- /dev/null +++ b/storefront/static/common/css/shadow.css @@ -0,0 +1 @@ +div.popup_main .poup_inner h2{text-shadow: 0.1em 0.1em 0.2em black} diff --git a/storefront/static/common/images/blue_add.png b/storefront/static/common/images/blue_add.png new file mode 100644 index 0000000..069e3fd Binary files /dev/null and b/storefront/static/common/images/blue_add.png differ diff --git a/storefront/static/common/images/blue_box_bg.gif b/storefront/static/common/images/blue_box_bg.gif new file mode 100644 index 0000000..8b9cbae Binary files /dev/null and b/storefront/static/common/images/blue_box_bg.gif differ diff --git a/storefront/static/common/images/blue_box_bg01.gif b/storefront/static/common/images/blue_box_bg01.gif new file mode 100644 index 0000000..09bf16e Binary files /dev/null and b/storefront/static/common/images/blue_box_bg01.gif differ diff --git a/storefront/static/common/images/blue_box_bottom.gif b/storefront/static/common/images/blue_box_bottom.gif new file mode 100644 index 0000000..5d46b1d Binary files /dev/null and b/storefront/static/common/images/blue_box_bottom.gif differ diff --git a/storefront/static/common/images/blue_box_bottom01.gif b/storefront/static/common/images/blue_box_bottom01.gif new file mode 100644 index 0000000..13bcb87 Binary files /dev/null and b/storefront/static/common/images/blue_box_bottom01.gif differ diff --git a/storefront/static/common/images/blue_box_top.gif b/storefront/static/common/images/blue_box_top.gif new file mode 100644 index 0000000..70e8782 Binary files /dev/null and b/storefront/static/common/images/blue_box_top.gif differ diff --git a/storefront/static/common/images/blue_box_top01.gif b/storefront/static/common/images/blue_box_top01.gif new file mode 100644 index 0000000..2eea2cd Binary files /dev/null and b/storefront/static/common/images/blue_box_top01.gif differ diff --git a/storefront/static/common/images/chat_bg.gif b/storefront/static/common/images/chat_bg.gif new file mode 100644 index 0000000..d02ad4e Binary files /dev/null and b/storefront/static/common/images/chat_bg.gif differ diff --git a/storefront/static/common/images/chat_bg01.gif b/storefront/static/common/images/chat_bg01.gif new file mode 100644 index 0000000..1e5b18f Binary files /dev/null and b/storefront/static/common/images/chat_bg01.gif differ diff --git a/storefront/static/common/images/chat_bottom.gif b/storefront/static/common/images/chat_bottom.gif new file mode 100644 index 0000000..4cf0c78 Binary files /dev/null and b/storefront/static/common/images/chat_bottom.gif differ diff --git a/storefront/static/common/images/chat_top.gif b/storefront/static/common/images/chat_top.gif new file mode 100644 index 0000000..f413220 Binary files /dev/null and b/storefront/static/common/images/chat_top.gif differ diff --git a/storefront/static/common/images/chat_top01.gif b/storefront/static/common/images/chat_top01.gif new file mode 100644 index 0000000..f55176e Binary files /dev/null and b/storefront/static/common/images/chat_top01.gif differ diff --git a/storefront/static/common/images/chat_top_short.gif b/storefront/static/common/images/chat_top_short.gif new file mode 100644 index 0000000..305988f Binary files /dev/null and b/storefront/static/common/images/chat_top_short.gif differ diff --git a/storefront/static/common/images/close_btn.gif b/storefront/static/common/images/close_btn.gif new file mode 100644 index 0000000..91c6406 Binary files /dev/null and b/storefront/static/common/images/close_btn.gif differ diff --git a/storefront/static/common/images/doc_sep.gif b/storefront/static/common/images/doc_sep.gif new file mode 100644 index 0000000..1fa809c Binary files /dev/null and b/storefront/static/common/images/doc_sep.gif differ diff --git a/storefront/static/common/images/dsafsdf.jpg b/storefront/static/common/images/dsafsdf.jpg new file mode 100644 index 0000000..eda27fb Binary files /dev/null and b/storefront/static/common/images/dsafsdf.jpg differ diff --git a/storefront/static/common/images/footer_bg.gif b/storefront/static/common/images/footer_bg.gif new file mode 100644 index 0000000..7ef95ee Binary files /dev/null and b/storefront/static/common/images/footer_bg.gif differ diff --git a/storefront/static/common/images/gazaro_features.png b/storefront/static/common/images/gazaro_features.png new file mode 100644 index 0000000..0049a14 Binary files /dev/null and b/storefront/static/common/images/gazaro_features.png differ diff --git a/storefront/static/common/images/graybutton_whitebg_107.png b/storefront/static/common/images/graybutton_whitebg_107.png new file mode 100644 index 0000000..a69c233 Binary files /dev/null and b/storefront/static/common/images/graybutton_whitebg_107.png differ diff --git a/storefront/static/common/images/green_check.png b/storefront/static/common/images/green_check.png new file mode 100644 index 0000000..4d30ba6 Binary files /dev/null and b/storefront/static/common/images/green_check.png differ diff --git a/storefront/static/common/images/header.gif b/storefront/static/common/images/header.gif new file mode 100644 index 0000000..6e1f072 Binary files /dev/null and b/storefront/static/common/images/header.gif differ diff --git a/storefront/static/common/images/icon001.gif b/storefront/static/common/images/icon001.gif new file mode 100644 index 0000000..93e98ab Binary files /dev/null and b/storefront/static/common/images/icon001.gif differ diff --git a/storefront/static/common/images/icon002.gif b/storefront/static/common/images/icon002.gif new file mode 100644 index 0000000..69464af Binary files /dev/null and b/storefront/static/common/images/icon002.gif differ diff --git a/storefront/static/common/images/icon003.gif b/storefront/static/common/images/icon003.gif new file mode 100644 index 0000000..68f0bdb Binary files /dev/null and b/storefront/static/common/images/icon003.gif differ diff --git a/storefront/static/common/images/icon004.gif b/storefront/static/common/images/icon004.gif new file mode 100644 index 0000000..ced5bf5 Binary files /dev/null and b/storefront/static/common/images/icon004.gif differ diff --git a/storefront/static/common/images/icon005.gif b/storefront/static/common/images/icon005.gif new file mode 100644 index 0000000..4267723 Binary files /dev/null and b/storefront/static/common/images/icon005.gif differ diff --git a/storefront/static/common/images/icon006.gif b/storefront/static/common/images/icon006.gif new file mode 100644 index 0000000..aa6654f Binary files /dev/null and b/storefront/static/common/images/icon006.gif differ diff --git a/storefront/static/common/images/icon01.gif b/storefront/static/common/images/icon01.gif new file mode 100644 index 0000000..0cbf5c7 Binary files /dev/null and b/storefront/static/common/images/icon01.gif differ diff --git a/storefront/static/common/images/icon02.gif b/storefront/static/common/images/icon02.gif new file mode 100644 index 0000000..6d0382b Binary files /dev/null and b/storefront/static/common/images/icon02.gif differ diff --git a/storefront/static/common/images/icon03.gif b/storefront/static/common/images/icon03.gif new file mode 100644 index 0000000..fb2f5e2 Binary files /dev/null and b/storefront/static/common/images/icon03.gif differ diff --git a/storefront/static/common/images/img01.gif b/storefront/static/common/images/img01.gif new file mode 100644 index 0000000..e311ed5 Binary files /dev/null and b/storefront/static/common/images/img01.gif differ diff --git a/storefront/static/common/images/img02.gif b/storefront/static/common/images/img02.gif new file mode 100644 index 0000000..1b31822 Binary files /dev/null and b/storefront/static/common/images/img02.gif differ diff --git a/storefront/static/common/images/input_bg.gif b/storefront/static/common/images/input_bg.gif new file mode 100644 index 0000000..e4ca526 Binary files /dev/null and b/storefront/static/common/images/input_bg.gif differ diff --git a/storefront/static/common/images/itsfree.png b/storefront/static/common/images/itsfree.png new file mode 100644 index 0000000..2db789c Binary files /dev/null and b/storefront/static/common/images/itsfree.png differ diff --git a/storefront/static/common/images/largebox_bg.gif b/storefront/static/common/images/largebox_bg.gif new file mode 100644 index 0000000..fc02ead Binary files /dev/null and b/storefront/static/common/images/largebox_bg.gif differ diff --git a/storefront/static/common/images/largebox_bottom.gif b/storefront/static/common/images/largebox_bottom.gif new file mode 100644 index 0000000..a961292 Binary files /dev/null and b/storefront/static/common/images/largebox_bottom.gif differ diff --git a/storefront/static/common/images/largebox_top.gif b/storefront/static/common/images/largebox_top.gif new file mode 100644 index 0000000..78ef816 Binary files /dev/null and b/storefront/static/common/images/largebox_top.gif differ diff --git a/storefront/static/common/images/log_sep.gif b/storefront/static/common/images/log_sep.gif new file mode 100644 index 0000000..eeed4f3 Binary files /dev/null and b/storefront/static/common/images/log_sep.gif differ diff --git a/storefront/static/common/images/logo.gif b/storefront/static/common/images/logo.gif new file mode 100644 index 0000000..618fc5d Binary files /dev/null and b/storefront/static/common/images/logo.gif differ diff --git a/storefront/static/common/images/mediumbox_bg.gif b/storefront/static/common/images/mediumbox_bg.gif new file mode 100644 index 0000000..6e937a0 Binary files /dev/null and b/storefront/static/common/images/mediumbox_bg.gif differ diff --git a/storefront/static/common/images/mediumbox_bottom.gif b/storefront/static/common/images/mediumbox_bottom.gif new file mode 100644 index 0000000..10909d5 Binary files /dev/null and b/storefront/static/common/images/mediumbox_bottom.gif differ diff --git a/storefront/static/common/images/mediumbox_top.gif b/storefront/static/common/images/mediumbox_top.gif new file mode 100644 index 0000000..cdb4242 Binary files /dev/null and b/storefront/static/common/images/mediumbox_top.gif differ diff --git a/storefront/static/common/images/menu_sep.gif b/storefront/static/common/images/menu_sep.gif new file mode 100644 index 0000000..69e7f4b Binary files /dev/null and b/storefront/static/common/images/menu_sep.gif differ diff --git a/storefront/static/common/images/modal.png b/storefront/static/common/images/modal.png new file mode 100644 index 0000000..d95c625 Binary files /dev/null and b/storefront/static/common/images/modal.png differ diff --git a/storefront/static/common/images/photo.gif b/storefront/static/common/images/photo.gif new file mode 100644 index 0000000..18f0726 Binary files /dev/null and b/storefront/static/common/images/photo.gif differ diff --git a/storefront/static/common/images/pict01.jpg b/storefront/static/common/images/pict01.jpg new file mode 100644 index 0000000..7364a8d Binary files /dev/null and b/storefront/static/common/images/pict01.jpg differ diff --git a/storefront/static/common/images/pict02.jpg b/storefront/static/common/images/pict02.jpg new file mode 100644 index 0000000..38fe6dd Binary files /dev/null and b/storefront/static/common/images/pict02.jpg differ diff --git a/storefront/static/common/images/pict03.jpg b/storefront/static/common/images/pict03.jpg new file mode 100644 index 0000000..95e475f Binary files /dev/null and b/storefront/static/common/images/pict03.jpg differ diff --git a/storefront/static/common/images/placeholder01.gif b/storefront/static/common/images/placeholder01.gif new file mode 100644 index 0000000..7f3e0a4 Binary files /dev/null and b/storefront/static/common/images/placeholder01.gif differ diff --git a/storefront/static/common/images/pop_btn_bg.gif b/storefront/static/common/images/pop_btn_bg.gif new file mode 100644 index 0000000..ed90bca Binary files /dev/null and b/storefront/static/common/images/pop_btn_bg.gif differ diff --git a/storefront/static/common/images/pop_btn_left.gif b/storefront/static/common/images/pop_btn_left.gif new file mode 100644 index 0000000..8bd5cce Binary files /dev/null and b/storefront/static/common/images/pop_btn_left.gif differ diff --git a/storefront/static/common/images/pop_btn_right.gif b/storefront/static/common/images/pop_btn_right.gif new file mode 100644 index 0000000..84eb655 Binary files /dev/null and b/storefront/static/common/images/pop_btn_right.gif differ diff --git a/storefront/static/common/images/popup_bottom.gif b/storefront/static/common/images/popup_bottom.gif new file mode 100644 index 0000000..d8be80d Binary files /dev/null and b/storefront/static/common/images/popup_bottom.gif differ diff --git a/storefront/static/common/images/popup_repeat.gif b/storefront/static/common/images/popup_repeat.gif new file mode 100644 index 0000000..e019987 Binary files /dev/null and b/storefront/static/common/images/popup_repeat.gif differ diff --git a/storefront/static/common/images/popup_top.gif b/storefront/static/common/images/popup_top.gif new file mode 100644 index 0000000..8c90914 Binary files /dev/null and b/storefront/static/common/images/popup_top.gif differ diff --git a/storefront/static/common/images/quote.gif b/storefront/static/common/images/quote.gif new file mode 100644 index 0000000..821a226 Binary files /dev/null and b/storefront/static/common/images/quote.gif differ diff --git a/storefront/static/common/images/red_delete.png b/storefront/static/common/images/red_delete.png new file mode 100644 index 0000000..b13d1ed Binary files /dev/null and b/storefront/static/common/images/red_delete.png differ diff --git a/storefront/static/common/images/reddit_features.gif b/storefront/static/common/images/reddit_features.gif new file mode 100644 index 0000000..85dfa9d Binary files /dev/null and b/storefront/static/common/images/reddit_features.gif differ diff --git a/storefront/static/common/images/right_search_bg.gif b/storefront/static/common/images/right_search_bg.gif new file mode 100644 index 0000000..c0a4eba Binary files /dev/null and b/storefront/static/common/images/right_search_bg.gif differ diff --git a/storefront/static/common/images/search_bg.gif b/storefront/static/common/images/search_bg.gif new file mode 100644 index 0000000..d208764 Binary files /dev/null and b/storefront/static/common/images/search_bg.gif differ diff --git a/storefront/static/common/images/search_btn.gif b/storefront/static/common/images/search_btn.gif new file mode 100644 index 0000000..59bb05c Binary files /dev/null and b/storefront/static/common/images/search_btn.gif differ diff --git a/storefront/static/common/images/sign_bg.gif b/storefront/static/common/images/sign_bg.gif new file mode 100644 index 0000000..a034648 Binary files /dev/null and b/storefront/static/common/images/sign_bg.gif differ diff --git a/storefront/static/common/images/sign_bg_over.gif b/storefront/static/common/images/sign_bg_over.gif new file mode 100644 index 0000000..69a88e7 Binary files /dev/null and b/storefront/static/common/images/sign_bg_over.gif differ diff --git a/storefront/static/common/images/tab_bottom_curve.gif b/storefront/static/common/images/tab_bottom_curve.gif new file mode 100644 index 0000000..608b4f7 Binary files /dev/null and b/storefront/static/common/images/tab_bottom_curve.gif differ diff --git a/storefront/static/common/images/tab_curve_bg.gif b/storefront/static/common/images/tab_curve_bg.gif new file mode 100644 index 0000000..6acc69a Binary files /dev/null and b/storefront/static/common/images/tab_curve_bg.gif differ diff --git a/storefront/static/common/images/tab_img01.gif b/storefront/static/common/images/tab_img01.gif new file mode 100644 index 0000000..55e1646 Binary files /dev/null and b/storefront/static/common/images/tab_img01.gif differ diff --git a/storefront/static/common/images/tab_img02.gif b/storefront/static/common/images/tab_img02.gif new file mode 100644 index 0000000..5868ac6 Binary files /dev/null and b/storefront/static/common/images/tab_img02.gif differ diff --git a/storefront/static/common/images/tab_img03.gif b/storefront/static/common/images/tab_img03.gif new file mode 100644 index 0000000..0ba2a50 Binary files /dev/null and b/storefront/static/common/images/tab_img03.gif differ diff --git a/storefront/static/common/images/tab_left.gif b/storefront/static/common/images/tab_left.gif new file mode 100644 index 0000000..37fb9c0 Binary files /dev/null and b/storefront/static/common/images/tab_left.gif differ diff --git a/storefront/static/common/images/tab_left01.gif b/storefront/static/common/images/tab_left01.gif new file mode 100644 index 0000000..5f69fca Binary files /dev/null and b/storefront/static/common/images/tab_left01.gif differ diff --git a/storefront/static/common/images/tab_left_01.gif b/storefront/static/common/images/tab_left_01.gif new file mode 100644 index 0000000..39c0283 Binary files /dev/null and b/storefront/static/common/images/tab_left_01.gif differ diff --git a/storefront/static/common/images/tab_right_01.gif b/storefront/static/common/images/tab_right_01.gif new file mode 100644 index 0000000..5b6934f Binary files /dev/null and b/storefront/static/common/images/tab_right_01.gif differ diff --git a/storefront/static/common/images/tab_top_curve.gif b/storefront/static/common/images/tab_top_curve.gif new file mode 100644 index 0000000..82aceb2 Binary files /dev/null and b/storefront/static/common/images/tab_top_curve.gif differ diff --git a/storefront/static/common/images/table_bg.gif b/storefront/static/common/images/table_bg.gif new file mode 100644 index 0000000..05b88fd Binary files /dev/null and b/storefront/static/common/images/table_bg.gif differ diff --git a/storefront/static/common/images/table_bottom.gif b/storefront/static/common/images/table_bottom.gif new file mode 100644 index 0000000..fcea555 Binary files /dev/null and b/storefront/static/common/images/table_bottom.gif differ diff --git a/storefront/static/common/images/table_top.gif b/storefront/static/common/images/table_top.gif new file mode 100644 index 0000000..095fbd5 Binary files /dev/null and b/storefront/static/common/images/table_top.gif differ diff --git a/storefront/static/common/images/try_bg01.gif b/storefront/static/common/images/try_bg01.gif new file mode 100644 index 0000000..b5934b4 Binary files /dev/null and b/storefront/static/common/images/try_bg01.gif differ diff --git a/storefront/static/common/images/try_bg01.png b/storefront/static/common/images/try_bg01.png new file mode 100644 index 0000000..cd5ab3e Binary files /dev/null and b/storefront/static/common/images/try_bg01.png differ diff --git a/storefront/static/common/images/try_bg02.gif b/storefront/static/common/images/try_bg02.gif new file mode 100644 index 0000000..06fb0e9 Binary files /dev/null and b/storefront/static/common/images/try_bg02.gif differ diff --git a/storefront/static/common/images/tv_icon.gif b/storefront/static/common/images/tv_icon.gif new file mode 100644 index 0000000..1e4354a Binary files /dev/null and b/storefront/static/common/images/tv_icon.gif differ diff --git a/storefront/static/common/images/tv_search_bg.gif b/storefront/static/common/images/tv_search_bg.gif new file mode 100644 index 0000000..a7312f3 Binary files /dev/null and b/storefront/static/common/images/tv_search_bg.gif differ diff --git a/storefront/static/common/images/tv_search_btn.gif b/storefront/static/common/images/tv_search_btn.gif new file mode 100644 index 0000000..9b7314f Binary files /dev/null and b/storefront/static/common/images/tv_search_btn.gif differ diff --git a/storefront/static/common/images/white_box_bg.gif b/storefront/static/common/images/white_box_bg.gif new file mode 100644 index 0000000..c568c70 Binary files /dev/null and b/storefront/static/common/images/white_box_bg.gif differ diff --git a/storefront/static/common/images/white_box_bottom.gif b/storefront/static/common/images/white_box_bottom.gif new file mode 100644 index 0000000..f530ca4 Binary files /dev/null and b/storefront/static/common/images/white_box_bottom.gif differ diff --git a/storefront/static/common/images/white_box_top.gif b/storefront/static/common/images/white_box_top.gif new file mode 100644 index 0000000..d50281d Binary files /dev/null and b/storefront/static/common/images/white_box_top.gif differ diff --git a/storefront/static/common/images/work504_bg.gif b/storefront/static/common/images/work504_bg.gif new file mode 100644 index 0000000..9c6d20a Binary files /dev/null and b/storefront/static/common/images/work504_bg.gif differ diff --git a/storefront/static/common/images/work504_bottom.gif b/storefront/static/common/images/work504_bottom.gif new file mode 100644 index 0000000..cb8f1b5 Binary files /dev/null and b/storefront/static/common/images/work504_bottom.gif differ diff --git a/storefront/static/common/images/work504_top01.gif b/storefront/static/common/images/work504_top01.gif new file mode 100644 index 0000000..55590d1 Binary files /dev/null and b/storefront/static/common/images/work504_top01.gif differ diff --git a/storefront/static/common/images/work_bg.gif b/storefront/static/common/images/work_bg.gif new file mode 100644 index 0000000..bf5fe96 Binary files /dev/null and b/storefront/static/common/images/work_bg.gif differ diff --git a/storefront/static/common/images/work_bottom.gif b/storefront/static/common/images/work_bottom.gif new file mode 100644 index 0000000..60657e7 Binary files /dev/null and b/storefront/static/common/images/work_bottom.gif differ diff --git a/storefront/static/common/images/work_icon01.gif b/storefront/static/common/images/work_icon01.gif new file mode 100644 index 0000000..941a629 Binary files /dev/null and b/storefront/static/common/images/work_icon01.gif differ diff --git a/storefront/static/common/images/work_icon02.gif b/storefront/static/common/images/work_icon02.gif new file mode 100644 index 0000000..04e5695 Binary files /dev/null and b/storefront/static/common/images/work_icon02.gif differ diff --git a/storefront/static/common/images/work_icon03.gif b/storefront/static/common/images/work_icon03.gif new file mode 100644 index 0000000..0f0ca47 Binary files /dev/null and b/storefront/static/common/images/work_icon03.gif differ diff --git a/storefront/static/common/images/work_icon04.gif b/storefront/static/common/images/work_icon04.gif new file mode 100644 index 0000000..427ef52 Binary files /dev/null and b/storefront/static/common/images/work_icon04.gif differ diff --git a/storefront/static/common/images/work_icon05.png b/storefront/static/common/images/work_icon05.png new file mode 100644 index 0000000..28ddcc5 Binary files /dev/null and b/storefront/static/common/images/work_icon05.png differ diff --git a/storefront/static/common/images/work_tab_left.gif b/storefront/static/common/images/work_tab_left.gif new file mode 100644 index 0000000..1987b92 Binary files /dev/null and b/storefront/static/common/images/work_tab_left.gif differ diff --git a/storefront/static/common/images/work_tab_right.gif b/storefront/static/common/images/work_tab_right.gif new file mode 100644 index 0000000..ab5b428 Binary files /dev/null and b/storefront/static/common/images/work_tab_right.gif differ diff --git a/storefront/static/common/images/work_top.gif b/storefront/static/common/images/work_top.gif new file mode 100644 index 0000000..b1981d0 Binary files /dev/null and b/storefront/static/common/images/work_top.gif differ diff --git a/storefront/static/common/images/work_top01.gif b/storefront/static/common/images/work_top01.gif new file mode 100644 index 0000000..571ce2d Binary files /dev/null and b/storefront/static/common/images/work_top01.gif differ diff --git a/storefront/static/common/js/cufon-yui.js b/storefront/static/common/js/cufon-yui.js new file mode 100644 index 0000000..6892de9 --- /dev/null +++ b/storefront/static/common/js/cufon-yui.js @@ -0,0 +1,1246 @@ +/*! + * Copyright (c) 2009 Simo Kinnunen. + * Licensed under the MIT license. + * + * @version ${Version} + */ +var Cufon = (function() { + + var api = function() { + return api.replace.apply(null, arguments); + }; + + var DOM = api.DOM = { + + ready: (function() { + + var complete = false, readyStatus = { loaded: 1, complete: 1 }; + + var queue = [], perform = function() { + if (complete) return; + complete = true; + for (var fn; fn = queue.shift(); fn()); + }; + + // Gecko, Opera, WebKit r26101+ + + if (document.addEventListener) { + document.addEventListener('DOMContentLoaded', perform, false); + window.addEventListener('pageshow', perform, false); // For cached Gecko pages + } + + // Old WebKit, Internet Explorer + + if (!window.opera && document.readyState) (function() { + readyStatus[document.readyState] ? perform() : setTimeout(arguments.callee, 10); + })(); + + // Internet Explorer + + if (document.readyState && document.createStyleSheet) (function() { + try { + document.body.doScroll('left'); + perform(); + } + catch (e) { + setTimeout(arguments.callee, 1); + } + })(); + + addEvent(window, 'load', perform); // Fallback + + return function(listener) { + if (!arguments.length) perform(); + else complete ? listener() : queue.push(listener); + }; + + })(), + + root: function() { + return document.documentElement || document.body; + } + + }; + + var CSS = api.CSS = { + + Size: function(value, base) { + + this.value = parseFloat(value); + this.unit = String(value).match(/[a-z%]*$/)[0] || 'px'; + + this.convert = function(value) { + return value / base * this.value; + }; + + this.convertFrom = function(value) { + return value / this.value * base; + }; + + this.toString = function() { + return this.value + this.unit; + }; + + }, + + addClass: function(el, className) { + var current = el.className; + el.className = current + (current && ' ') + className; + return el; + }, + + color: cached(function(value) { + var parsed = {}; + parsed.color = value.replace(/^rgba\((.*?),\s*([\d.]+)\)/, function($0, $1, $2) { + parsed.opacity = parseFloat($2); + return 'rgb(' + $1 + ')'; + }); + return parsed; + }), + + // has no direct CSS equivalent. + // @see http://msdn.microsoft.com/en-us/library/system.windows.fontstretches.aspx + fontStretch: cached(function(value) { + if (typeof value == 'number') return value; + if (/%$/.test(value)) return parseFloat(value) / 100; + return { + 'ultra-condensed': 0.5, + 'extra-condensed': 0.625, + condensed: 0.75, + 'semi-condensed': 0.875, + 'semi-expanded': 1.125, + expanded: 1.25, + 'extra-expanded': 1.5, + 'ultra-expanded': 2 + }[value] || 1; + }), + + getStyle: function(el) { + var view = document.defaultView; + if (view && view.getComputedStyle) return new Style(view.getComputedStyle(el, null)); + if (el.currentStyle) return new Style(el.currentStyle); + return new Style(el.style); + }, + + gradient: cached(function(value) { + var gradient = { + id: value, + type: value.match(/^-([a-z]+)-gradient\(/)[1], + stops: [] + }, colors = value.substr(value.indexOf('(')).match(/([\d.]+=)?(#[a-f0-9]+|[a-z]+\(.*?\)|[a-z]+)/ig); + for (var i = 0, l = colors.length, stop; i < l; ++i) { + stop = colors[i].split('=', 2).reverse(); + gradient.stops.push([ stop[1] || i / (l - 1), stop[0] ]); + } + return gradient; + }), + + quotedList: cached(function(value) { + // doesn't work properly with empty quoted strings (""), but + // it's not worth the extra code. + var list = [], re = /\s*((["'])([\s\S]*?[^\\])\2|[^,]+)\s*/g, match; + while (match = re.exec(value)) list.push(match[3] || match[1]); + return list; + }), + + recognizesMedia: cached(function(media) { + var el = document.createElement('style'), sheet, container, supported; + el.type = 'text/css'; + el.media = media; + try { // this is cached anyway + el.appendChild(document.createTextNode('/**/')); + } catch (e) {} + container = elementsByTagName('head')[0]; + container.insertBefore(el, container.firstChild); + sheet = (el.sheet || el.styleSheet); + supported = sheet && !sheet.disabled; + container.removeChild(el); + return supported; + }), + + removeClass: function(el, className) { + var re = RegExp('(?:^|\\s+)' + className + '(?=\\s|$)', 'g'); + el.className = el.className.replace(re, ''); + return el; + }, + + supports: function(property, value) { + var checker = document.createElement('span').style; + if (checker[property] === undefined) return false; + checker[property] = value; + return checker[property] === value; + }, + + textAlign: function(word, style, position, wordCount) { + if (style.get('textAlign') == 'right') { + if (position > 0) word = ' ' + word; + } + else if (position < wordCount - 1) word += ' '; + return word; + }, + + textShadow: cached(function(value) { + if (value == 'none') return null; + var shadows = [], currentShadow = {}, result, offCount = 0; + var re = /(#[a-f0-9]+|[a-z]+\(.*?\)|[a-z]+)|(-?[\d.]+[a-z%]*)|,/ig; + while (result = re.exec(value)) { + if (result[0] == ',') { + shadows.push(currentShadow); + currentShadow = {}; + offCount = 0; + } + else if (result[1]) { + currentShadow.color = result[1]; + } + else { + currentShadow[[ 'offX', 'offY', 'blur' ][offCount++]] = result[2]; + } + } + shadows.push(currentShadow); + return shadows; + }), + + textTransform: (function() { + var map = { + uppercase: function(s) { + return s.toUpperCase(); + }, + lowercase: function(s) { + return s.toLowerCase(); + }, + capitalize: function(s) { + return s.replace(/\b./g, function($0) { + return $0.toUpperCase(); + }); + } + }; + return function(text, style) { + var transform = map[style.get('textTransform')]; + return transform ? transform(text) : text; + }; + })(), + + whiteSpace: (function() { + var ignore = { + inline: 1, + 'inline-block': 1, + 'run-in': 1 + }; + var wsStart = /^\s+/, wsEnd = /\s+$/; + return function(text, style, node, previousElement) { + if (previousElement) { + if (previousElement.nodeName.toLowerCase() == 'br') { + text = text.replace(wsStart, ''); + } + } + if (ignore[style.get('display')]) return text; + if (!node.previousSibling) text = text.replace(wsStart, ''); + if (!node.nextSibling) text = text.replace(wsEnd, ''); + return text; + }; + })() + + }; + + CSS.ready = (function() { + + // don't do anything in Safari 2 (it doesn't recognize any media type) + var complete = !CSS.recognizesMedia('all'), hasLayout = false; + + var queue = [], perform = function() { + complete = true; + for (var fn; fn = queue.shift(); fn()); + }; + + var links = elementsByTagName('link'), styles = elementsByTagName('style'); + + function isContainerReady(el) { + return el.disabled || isSheetReady(el.sheet, el.media || 'screen'); + } + + function isSheetReady(sheet, media) { + // in Opera sheet.disabled is true when it's still loading, + // even though link.disabled is false. they stay in sync if + // set manually. + if (!CSS.recognizesMedia(media || 'all')) return true; + if (!sheet || sheet.disabled) return false; + try { + var rules = sheet.cssRules, rule; + if (rules) { + // needed for Safari 3 and Chrome 1.0. + // in standards-conforming browsers cssRules contains @-rules. + // Chrome 1.0 weirdness: rules[] + // returns the last rule, so a for loop is the only option. + search: for (var i = 0, l = rules.length; rule = rules[i], i < l; ++i) { + switch (rule.type) { + case 2: // @charset + break; + case 3: // @import + if (!isSheetReady(rule.styleSheet, rule.media.mediaText)) return false; + break; + default: + // only @charset can precede @import + break search; + } + } + } + } + catch (e) {} // probably a style sheet from another domain + return true; + } + + function allStylesLoaded() { + // Internet Explorer's style sheet model, there's no need to do anything + if (document.createStyleSheet) return true; + // standards-compliant browsers + var el, i; + for (i = 0; el = links[i]; ++i) { + if (el.rel.toLowerCase() == 'stylesheet' && !isContainerReady(el)) return false; + } + for (i = 0; el = styles[i]; ++i) { + if (!isContainerReady(el)) return false; + } + return true; + } + + DOM.ready(function() { + // getComputedStyle returns null in Gecko if used in an iframe with display: none + if (!hasLayout) hasLayout = CSS.getStyle(document.body).isUsable(); + if (complete || (hasLayout && allStylesLoaded())) perform(); + else setTimeout(arguments.callee, 10); + }); + + return function(listener) { + if (complete) listener(); + else queue.push(listener); + }; + + })(); + + function Font(data) { + + var face = this.face = data.face, wordSeparators = { + '\u0020': 1, + '\u00a0': 1, + '\u3000': 1 + }; + + this.glyphs = data.glyphs; + this.w = data.w; + this.baseSize = parseInt(face['units-per-em'], 10); + + this.family = face['font-family'].toLowerCase(); + this.weight = face['font-weight']; + this.style = face['font-style'] || 'normal'; + + this.viewBox = (function () { + var parts = face.bbox.split(/\s+/); + var box = { + minX: parseInt(parts[0], 10), + minY: parseInt(parts[1], 10), + maxX: parseInt(parts[2], 10), + maxY: parseInt(parts[3], 10) + }; + box.width = box.maxX - box.minX; + box.height = box.maxY - box.minY; + box.toString = function() { + return [ this.minX, this.minY, this.width, this.height ].join(' '); + }; + return box; + })(); + + this.ascent = -parseInt(face.ascent, 10); + this.descent = -parseInt(face.descent, 10); + + this.height = -this.ascent + this.descent; + + this.spacing = function(chars, letterSpacing, wordSpacing) { + var glyphs = this.glyphs, glyph, kerning, k, + jumps = [], width = 0, + i = -1, j = -1, chr; + while (chr = chars[++i]) { + glyph = glyphs[chr] || this.missingGlyph; + if (!glyph) continue; + if (kerning) { + width -= k = kerning[chr] || 0; + jumps[j - 1] -= k; + } + width += jumps[++j] = ~~(glyph.w || this.w) + letterSpacing + (wordSeparators[chr] ? wordSpacing : 0); + kerning = glyph.k; + } + jumps.total = width; + return jumps; + }; + + } + + function FontFamily() { + + var styles = {}, mapping = { + oblique: 'italic', + italic: 'oblique' + }; + + this.add = function(font) { + (styles[font.style] || (styles[font.style] = {}))[font.weight] = font; + }; + + this.get = function(style, weight) { + var weights = styles[style] || styles[mapping[style]] + || styles.normal || styles.italic || styles.oblique; + if (!weights) return null; + // we don't have to worry about "bolder" and "lighter" + // because IE's currentStyle returns a numeric value for it, + // and other browsers use the computed value anyway + weight = { + normal: 400, + bold: 700 + }[weight] || parseInt(weight, 10); + if (weights[weight]) return weights[weight]; + // http://www.w3.org/TR/CSS21/fonts.html#propdef-font-weight + // Gecko uses x99/x01 for lighter/bolder + var up = { + 1: 1, + 99: 0 + }[weight % 100], alts = [], min, max; + if (up === undefined) up = weight > 400; + if (weight == 500) weight = 400; + for (var alt in weights) { + if (!hasOwnProperty(weights, alt)) continue; + alt = parseInt(alt, 10); + if (!min || alt < min) min = alt; + if (!max || alt > max) max = alt; + alts.push(alt); + } + if (weight < min) weight = min; + if (weight > max) weight = max; + alts.sort(function(a, b) { + return (up + ? (a >= weight && b >= weight) ? a < b : a > b + : (a <= weight && b <= weight) ? a > b : a < b) ? -1 : 1; + }); + return weights[alts[0]]; + }; + + } + + function HoverHandler() { + + function contains(node, anotherNode) { + if (node.contains) return node.contains(anotherNode); + return node.compareDocumentPosition(anotherNode) & 16; + } + + function onOverOut(e) { + var related = e.relatedTarget; + if (!related || contains(this, related)) return; + trigger(this, e.type == 'mouseover'); + } + + function onEnterLeave(e) { + trigger(this, e.type == 'mouseenter'); + } + + function trigger(el, hoverState) { + // A timeout is needed so that the event can actually "happen" + // before replace is triggered. This ensures that styles are up + // to date. + setTimeout(function() { + var options = sharedStorage.get(el).options; + api.replace(el, hoverState ? merge(options, options.hover) : options, true); + }, 10); + } + + this.attach = function(el) { + if (el.onmouseenter === undefined) { + addEvent(el, 'mouseover', onOverOut); + addEvent(el, 'mouseout', onOverOut); + } + else { + addEvent(el, 'mouseenter', onEnterLeave); + addEvent(el, 'mouseleave', onEnterLeave); + } + }; + + } + + function ReplaceHistory() { + + var list = [], map = {}; + + function filter(keys) { + var values = [], key; + for (var i = 0; key = keys[i]; ++i) values[i] = list[map[key]]; + return values; + } + + this.add = function(key, args) { + map[key] = list.push(args) - 1; + }; + + this.repeat = function() { + var snapshot = arguments.length ? filter(arguments) : list, args; + for (var i = 0; args = snapshot[i++];) api.replace(args[0], args[1], true); + }; + + } + + function Storage() { + + var map = {}, at = 0; + + function identify(el) { + return el.cufid || (el.cufid = ++at); + } + + this.get = function(el) { + var id = identify(el); + return map[id] || (map[id] = {}); + }; + + } + + function Style(style) { + + var custom = {}, sizes = {}; + + this.extend = function(styles) { + for (var property in styles) { + if (hasOwnProperty(styles, property)) custom[property] = styles[property]; + } + return this; + }; + + this.get = function(property) { + return custom[property] != undefined ? custom[property] : style[property]; + }; + + this.getSize = function(property, base) { + return sizes[property] || (sizes[property] = new CSS.Size(this.get(property), base)); + }; + + this.isUsable = function() { + return !!style; + }; + + } + + function addEvent(el, type, listener) { + if (el.addEventListener) { + el.addEventListener(type, listener, false); + } + else if (el.attachEvent) { + el.attachEvent('on' + type, function() { + return listener.call(el, window.event); + }); + } + } + + function attach(el, options) { + var storage = sharedStorage.get(el); + if (storage.options) return el; + if (options.hover && options.hoverables[el.nodeName.toLowerCase()]) { + hoverHandler.attach(el); + } + storage.options = options; + return el; + } + + function cached(fun) { + var cache = {}; + return function(key) { + if (!hasOwnProperty(cache, key)) cache[key] = fun.apply(null, arguments); + return cache[key]; + }; + } + + function getFont(el, style) { + var families = CSS.quotedList(style.get('fontFamily').toLowerCase()), family; + for (var i = 0; family = families[i]; ++i) { + if (fonts[family]) return fonts[family].get(style.get('fontStyle'), style.get('fontWeight')); + } + return null; + } + + function elementsByTagName(query) { + return document.getElementsByTagName(query); + } + + function hasOwnProperty(obj, property) { + return obj.hasOwnProperty(property); + } + + function merge() { + var merged = {}, arg, key; + for (var i = 0, l = arguments.length; arg = arguments[i], i < l; ++i) { + for (key in arg) { + if (hasOwnProperty(arg, key)) merged[key] = arg[key]; + } + } + return merged; + } + + function process(font, text, style, options, node, el) { + var fragment = document.createDocumentFragment(), processed; + if (text === '') return fragment; + var separate = options.separate; + var parts = text.split(separators[separate]), needsAligning = (separate == 'words'); + if (needsAligning && HAS_BROKEN_REGEXP) { + // @todo figure out a better way to do this + if (/^\s/.test(text)) parts.unshift(''); + if (/\s$/.test(text)) parts.push(''); + + } + for (var i = 0, l = parts.length; i < l; ++i) { + processed = engines[options.engine](font, + needsAligning ? CSS.textAlign(parts[i], style, i, l) : parts[i], + style, options, node, el, i < l - 1); + if (processed) fragment.appendChild(processed); + } + return fragment; + } + + function replaceElement(el, options) { + var name = el.nodeName.toLowerCase(); + if (options.ignore[name]) return; + var replace = !options.textless[name]; + var style = CSS.getStyle(attach(el, options)).extend(options); + var font = getFont(el, style), node, type, next, anchor, text, lastElement; + if (!font) return; + for (node = el.firstChild; node; node = next) { + type = node.nodeType; + next = node.nextSibling; + if (replace && type == 3) { + // Node.normalize() is broken in IE 6, 7, 8 + if (anchor) { + anchor.appendData(node.data); + el.removeChild(node); + } + else anchor = node; + if (next) continue; + } + if (anchor) { + el.replaceChild(process(font, + CSS.whiteSpace(anchor.data, style, anchor, lastElement), + style, options, node, el), anchor); + anchor = null; + } + if (type == 1) { + if (node.firstChild) { + if (node.nodeName.toLowerCase() == 'cufon') { + engines[options.engine](font, null, style, options, node, el); + } + else arguments.callee(node, options); + } + lastElement = node; + } + } + } + + var HAS_BROKEN_REGEXP = ' '.split(/\s+/).length == 0; + + var sharedStorage = new Storage(); + var hoverHandler = new HoverHandler(); + var replaceHistory = new ReplaceHistory(); + var initialized = false; + + var engines = {}, fonts = {}, defaultOptions = { + autoDetect: false, + engine: null, + //fontScale: 1, + //fontScaling: false, + forceHitArea: false, + hover: false, + hoverables: { + a: true + }, + ignore: { + applet: 1, + canvas: 1, + col: 1, + colgroup: 1, + head: 1, + iframe: 1, + map: 1, + optgroup: 1, + option: 1, + script: 1, + select: 1, + style: 1, + textarea: 1, + title: 1, + pre: 1 + }, + printable: true, + //rotation: 0, + //selectable: false, + selector: ( + window.Sizzle + || (window.jQuery && function(query) { return jQuery(query); }) // avoid noConflict issues + || (window.dojo && dojo.query) + || (window.Ext && Ext.query) + || (window.YAHOO && YAHOO.util && YAHOO.util.Selector && YAHOO.util.Selector.query) + || (window.$$ && function(query) { return $$(query); }) + || (window.$ && function(query) { return $(query); }) + || (document.querySelectorAll && function(query) { return document.querySelectorAll(query); }) + || elementsByTagName + ), + separate: 'words', // 'none' and 'characters' are also accepted + textless: { + dl: 1, + html: 1, + ol: 1, + table: 1, + tbody: 1, + thead: 1, + tfoot: 1, + tr: 1, + ul: 1 + }, + textShadow: 'none' + }; + + var separators = { + // The first pattern may cause unicode characters above + // code point 255 to be removed in Safari 3.0. Luckily enough + // Safari 3.0 does not include non-breaking spaces in \s, so + // we can just use a simple alternative pattern. + words: /\s/.test('\u00a0') ? /[^\S\u00a0]+/ : /\s+/, + characters: '', + none: /^/ + }; + + api.now = function() { + DOM.ready(); + return api; + }; + + api.refresh = function() { + replaceHistory.repeat.apply(replaceHistory, arguments); + return api; + }; + + api.registerEngine = function(id, engine) { + if (!engine) return api; + engines[id] = engine; + return api.set('engine', id); + }; + + api.registerFont = function(data) { + if (!data) return api; + var font = new Font(data), family = font.family; + if (!fonts[family]) fonts[family] = new FontFamily(); + fonts[family].add(font); + return api.set('fontFamily', '"' + family + '"'); + }; + + api.replace = function(elements, options, ignoreHistory) { + options = merge(defaultOptions, options); + if (!options.engine) return api; // there's no browser support so we'll just stop here + if (!initialized) { + CSS.addClass(DOM.root(), 'cufon-active cufon-loading'); + CSS.ready(function() { + // fires before any replace() calls, but it doesn't really matter + CSS.addClass(CSS.removeClass(DOM.root(), 'cufon-loading'), 'cufon-ready'); + }); + initialized = true; + } + if (options.hover) options.forceHitArea = true; + if (options.autoDetect) delete options.fontFamily; + if (typeof options.textShadow == 'string') { + options.textShadow = CSS.textShadow(options.textShadow); + } + if (typeof options.color == 'string' && /^-/.test(options.color)) { + options.textGradient = CSS.gradient(options.color); + } + else delete options.textGradient; + if (!ignoreHistory) replaceHistory.add(elements, arguments); + if (elements.nodeType || typeof elements == 'string') elements = [ elements ]; + CSS.ready(function() { + for (var i = 0, l = elements.length; i < l; ++i) { + var el = elements[i]; + if (typeof el == 'string') api.replace(options.selector(el), options, true); + else replaceElement(el, options); + } + }); + return api; + }; + + api.set = function(option, value) { + defaultOptions[option] = value; + return api; + }; + + return api; + +})(); + +Cufon.registerEngine('canvas', (function() { + + // Safari 2 doesn't support .apply() on native methods + + var check = document.createElement('canvas'); + if (!check || !check.getContext || !check.getContext.apply) return; + check = null; + + var HAS_INLINE_BLOCK = Cufon.CSS.supports('display', 'inline-block'); + + // Firefox 2 w/ non-strict doctype (almost standards mode) + var HAS_BROKEN_LINEHEIGHT = !HAS_INLINE_BLOCK && (document.compatMode == 'BackCompat' || /frameset|transitional/i.test(document.doctype.publicId)); + + var styleSheet = document.createElement('style'); + styleSheet.type = 'text/css'; + styleSheet.appendChild(document.createTextNode(( + 'cufon{text-indent:0;}' + + '@media screen,projection{' + + 'cufon{display:inline;display:inline-block;position:relative;vertical-align:middle;' + + (HAS_BROKEN_LINEHEIGHT + ? '' + : 'font-size:1px;line-height:1px;') + + '}cufon cufontext{display:-moz-inline-box;display:inline-block;width:0;height:0;overflow:hidden;text-indent:-10000in;}' + + (HAS_INLINE_BLOCK + ? 'cufon canvas{position:relative;}' + : 'cufon canvas{position:absolute;}') + + '}' + + '@media print{' + + 'cufon{padding:0;}' + // Firefox 2 + 'cufon canvas{display:none;}' + + '}' + ).replace(/;/g, '!important;'))); + document.getElementsByTagName('head')[0].appendChild(styleSheet); + + function generateFromVML(path, context) { + var atX = 0, atY = 0; + var code = [], re = /([mrvxe])([^a-z]*)/g, match; + generate: for (var i = 0; match = re.exec(path); ++i) { + var c = match[2].split(','); + switch (match[1]) { + case 'v': + code[i] = { m: 'bezierCurveTo', a: [ atX + ~~c[0], atY + ~~c[1], atX + ~~c[2], atY + ~~c[3], atX += ~~c[4], atY += ~~c[5] ] }; + break; + case 'r': + code[i] = { m: 'lineTo', a: [ atX += ~~c[0], atY += ~~c[1] ] }; + break; + case 'm': + code[i] = { m: 'moveTo', a: [ atX = ~~c[0], atY = ~~c[1] ] }; + break; + case 'x': + code[i] = { m: 'closePath' }; + break; + case 'e': + break generate; + } + context[code[i].m].apply(context, code[i].a); + } + return code; + } + + function interpret(code, context) { + for (var i = 0, l = code.length; i < l; ++i) { + var line = code[i]; + context[line.m].apply(context, line.a); + } + } + + return function(font, text, style, options, node, el) { + + var redraw = (text === null); + + if (redraw) text = node.getAttribute('alt'); + + var viewBox = font.viewBox; + + var size = style.getSize('fontSize', font.baseSize); + + var expandTop = 0, expandRight = 0, expandBottom = 0, expandLeft = 0; + var shadows = options.textShadow, shadowOffsets = []; + if (shadows) { + for (var i = shadows.length; i--;) { + var shadow = shadows[i]; + var x = size.convertFrom(parseFloat(shadow.offX)); + var y = size.convertFrom(parseFloat(shadow.offY)); + shadowOffsets[i] = [ x, y ]; + if (y < expandTop) expandTop = y; + if (x > expandRight) expandRight = x; + if (y > expandBottom) expandBottom = y; + if (x < expandLeft) expandLeft = x; + } + } + + var chars = Cufon.CSS.textTransform(text, style).split(''); + + var jumps = font.spacing(chars, + ~~size.convertFrom(parseFloat(style.get('letterSpacing')) || 0), + ~~size.convertFrom(parseFloat(style.get('wordSpacing')) || 0) + ); + + if (!jumps.length) return null; // there's nothing to render + + var width = jumps.total; + + expandRight += viewBox.width - jumps[jumps.length - 1]; + expandLeft += viewBox.minX; + + var wrapper, canvas; + + if (redraw) { + wrapper = node; + canvas = node.firstChild; + } + else { + wrapper = document.createElement('cufon'); + wrapper.className = 'cufon cufon-canvas'; + wrapper.setAttribute('alt', text); + + canvas = document.createElement('canvas'); + wrapper.appendChild(canvas); + + if (options.printable) { + var print = document.createElement('cufontext'); + print.appendChild(document.createTextNode(text)); + wrapper.appendChild(print); + } + } + + var wStyle = wrapper.style; + var cStyle = canvas.style; + + var height = size.convert(viewBox.height); + var roundedHeight = Math.ceil(height); + var roundingFactor = roundedHeight / height; + var stretchFactor = roundingFactor * Cufon.CSS.fontStretch(style.get('fontStretch')); + var stretchedWidth = width * stretchFactor; + + var canvasWidth = Math.ceil(size.convert(stretchedWidth + expandRight - expandLeft)); + var canvasHeight = Math.ceil(size.convert(viewBox.height - expandTop + expandBottom)); + + canvas.width = canvasWidth; + canvas.height = canvasHeight; + + // needed for WebKit and full page zoom + cStyle.width = canvasWidth + 'px'; + cStyle.height = canvasHeight + 'px'; + + // minY has no part in canvas.height + expandTop += viewBox.minY; + + cStyle.top = Math.round(size.convert(expandTop - font.ascent)) + 'px'; + cStyle.left = Math.round(size.convert(expandLeft)) + 'px'; + + var wrapperWidth = Math.max(Math.ceil(size.convert(stretchedWidth)), 0) + 'px'; + + if (HAS_INLINE_BLOCK) { + wStyle.width = wrapperWidth; + wStyle.height = size.convert(font.height) + 'px'; + } + else { + wStyle.paddingLeft = wrapperWidth; + wStyle.paddingBottom = (size.convert(font.height) - 1) + 'px'; + } + + var g = canvas.getContext('2d'), scale = height / viewBox.height; + + // proper horizontal scaling is performed later + g.scale(scale, scale * roundingFactor); + g.translate(-expandLeft, -expandTop); + g.save(); + + function renderText() { + var glyphs = font.glyphs, glyph, i = -1, j = -1, chr; + g.scale(stretchFactor, 1); + while (chr = chars[++i]) { + var glyph = glyphs[chars[i]] || font.missingGlyph; + if (!glyph) continue; + if (glyph.d) { + g.beginPath(); + if (glyph.code) interpret(glyph.code, g); + else glyph.code = generateFromVML('m' + glyph.d, g); + g.fill(); + } + g.translate(jumps[++j], 0); + } + g.restore(); + } + + if (shadows) { + for (var i = shadows.length; i--;) { + var shadow = shadows[i]; + g.save(); + g.fillStyle = shadow.color; + g.translate.apply(g, shadowOffsets[i]); + renderText(); + } + } + + var gradient = options.textGradient; + if (gradient) { + var stops = gradient.stops, fill = g.createLinearGradient(0, viewBox.minY, 0, viewBox.maxY); + for (var i = 0, l = stops.length; i < l; ++i) { + fill.addColorStop.apply(fill, stops[i]); + } + g.fillStyle = fill; + } + else g.fillStyle = style.get('color'); + + renderText(); + + return wrapper; + + }; + +})()); + +Cufon.registerEngine('vml', (function() { + + var ns = document.namespaces; + if (!ns) return; + ns.add('cvml', 'urn:schemas-microsoft-com:vml'); + ns = null; + + var check = document.createElement('cvml:shape'); + check.style.behavior = 'url(#default#VML)'; + if (!check.coordsize) return; // VML isn't supported + check = null; + + var HAS_BROKEN_LINEHEIGHT = (document.documentMode || 0) < 8; + + document.write(('').replace(/;/g, '!important;')); + + function getFontSizeInPixels(el, value) { + return getSizeInPixels(el, /(?:em|ex|%)$|^[a-z-]+$/i.test(value) ? '1em' : value); + } + + // Original by Dead Edwards. + // Combined with getFontSizeInPixels it also works with relative units. + function getSizeInPixels(el, value) { + if (value === '0') return 0; + if (/px$/i.test(value)) return parseFloat(value); + var style = el.style.left, runtimeStyle = el.runtimeStyle.left; + el.runtimeStyle.left = el.currentStyle.left; + el.style.left = value.replace('%', 'em'); + var result = el.style.pixelLeft; + el.style.left = style; + el.runtimeStyle.left = runtimeStyle; + return result; + } + + function getSpacingValue(el, style, size, property) { + var key = 'computed' + property, value = style[key]; + if (isNaN(value)) { + value = style.get(property); + style[key] = value = (value == 'normal') ? 0 : ~~size.convertFrom(getSizeInPixels(el, value)); + } + return value; + } + + var fills = {}; + + function gradientFill(gradient) { + var id = gradient.id; + if (!fills[id]) { + var stops = gradient.stops, fill = document.createElement('cvml:fill'), colors = []; + fill.type = 'gradient'; + fill.angle = 180; + fill.focus = '0'; + fill.method = 'sigma'; + fill.color = stops[0][1]; + for (var j = 1, k = stops.length - 1; j < k; ++j) { + colors.push(stops[j][0] * 100 + '% ' + stops[j][1]); + } + fill.colors = colors.join(','); + fill.color2 = stops[k][1]; + fills[id] = fill; + } + return fills[id]; + } + + return function(font, text, style, options, node, el, hasNext) { + + var redraw = (text === null); + + if (redraw) text = node.alt; + + var viewBox = font.viewBox; + + var size = style.computedFontSize || (style.computedFontSize = new Cufon.CSS.Size(getFontSizeInPixels(el, style.get('fontSize')) + 'px', font.baseSize)); + + var wrapper, canvas; + + if (redraw) { + wrapper = node; + canvas = node.firstChild; + } + else { + wrapper = document.createElement('cufon'); + wrapper.className = 'cufon cufon-vml'; + wrapper.alt = text; + + canvas = document.createElement('cufoncanvas'); + wrapper.appendChild(canvas); + + if (options.printable) { + var print = document.createElement('cufontext'); + print.appendChild(document.createTextNode(text)); + wrapper.appendChild(print); + } + + // ie6, for some reason, has trouble rendering the last VML element in the document. + // we can work around this by injecting a dummy element where needed. + // @todo find a better solution + if (!hasNext) wrapper.appendChild(document.createElement('cvml:shape')); + } + + var wStyle = wrapper.style; + var cStyle = canvas.style; + + var height = size.convert(viewBox.height), roundedHeight = Math.ceil(height); + var roundingFactor = roundedHeight / height; + var stretchFactor = roundingFactor * Cufon.CSS.fontStretch(style.get('fontStretch')); + var minX = viewBox.minX, minY = viewBox.minY; + + cStyle.height = roundedHeight; + cStyle.top = Math.round(size.convert(minY - font.ascent)); + cStyle.left = Math.round(size.convert(minX)); + + wStyle.height = size.convert(font.height) + 'px'; + + var color = style.get('color'); + var chars = Cufon.CSS.textTransform(text, style).split(''); + + var jumps = font.spacing(chars, + getSpacingValue(el, style, size, 'letterSpacing'), + getSpacingValue(el, style, size, 'wordSpacing') + ); + + if (!jumps.length) return null; + + var width = jumps.total; + var fullWidth = -minX + width + (viewBox.width - jumps[jumps.length - 1]); + + var shapeWidth = size.convert(fullWidth * stretchFactor), roundedShapeWidth = Math.round(shapeWidth); + + var coordSize = fullWidth + ',' + viewBox.height, coordOrigin; + var stretch = 'r' + coordSize + 'ns'; + + var fill = options.textGradient && gradientFill(options.textGradient); + + var glyphs = font.glyphs, offsetX = 0; + var shadows = options.textShadow; + var i = -1, j = 0, chr; + + while (chr = chars[++i]) { + + var glyph = glyphs[chars[i]] || font.missingGlyph, shape; + if (!glyph) continue; + + if (redraw) { + // some glyphs may be missing so we can't use i + shape = canvas.childNodes[j]; + while (shape.firstChild) shape.removeChild(shape.firstChild); // shadow, fill + } + else { + shape = document.createElement('cvml:shape'); + canvas.appendChild(shape); + } + + shape.stroked = 'f'; + shape.coordsize = coordSize; + shape.coordorigin = coordOrigin = (minX - offsetX) + ',' + minY; + shape.path = (glyph.d ? 'm' + glyph.d + 'xe' : '') + 'm' + coordOrigin + stretch; + shape.fillcolor = color; + + if (fill) shape.appendChild(fill.cloneNode(false)); + + // it's important to not set top/left or IE8 will grind to a halt + var sStyle = shape.style; + sStyle.width = roundedShapeWidth; + sStyle.height = roundedHeight; + + if (shadows) { + // due to the limitations of the VML shadow element there + // can only be two visible shadows. opacity is shared + // for all shadows. + var shadow1 = shadows[0], shadow2 = shadows[1]; + var color1 = Cufon.CSS.color(shadow1.color), color2; + var shadow = document.createElement('cvml:shadow'); + shadow.on = 't'; + shadow.color = color1.color; + shadow.offset = shadow1.offX + ',' + shadow1.offY; + if (shadow2) { + color2 = Cufon.CSS.color(shadow2.color); + shadow.type = 'double'; + shadow.color2 = color2.color; + shadow.offset2 = shadow2.offX + ',' + shadow2.offY; + } + shadow.opacity = color1.opacity || (color2 && color2.opacity) || 1; + shape.appendChild(shadow); + } + + offsetX += jumps[j++]; + } + + // addresses flickering issues on :hover + + var cover = shape.nextSibling, coverFill, vStyle; + + if (options.forceHitArea) { + + if (!cover) { + cover = document.createElement('cvml:rect'); + cover.stroked = 'f'; + cover.className = 'cufon-vml-cover'; + coverFill = document.createElement('cvml:fill'); + coverFill.opacity = 0; + cover.appendChild(coverFill); + canvas.appendChild(cover); + } + + vStyle = cover.style; + + vStyle.width = roundedShapeWidth; + vStyle.height = roundedHeight; + + } + else if (cover) canvas.removeChild(cover); + + wStyle.width = Math.max(Math.ceil(size.convert(width * stretchFactor)), 0); + + if (HAS_BROKEN_LINEHEIGHT) { + + var yAdjust = style.computedYAdjust; + + if (yAdjust === undefined) { + var lineHeight = style.get('lineHeight'); + if (lineHeight == 'normal') lineHeight = '1em'; + else if (!isNaN(lineHeight)) lineHeight += 'em'; // no unit + style.computedYAdjust = yAdjust = 0.5 * (getSizeInPixels(el, lineHeight) - parseFloat(wStyle.height)); + } + + if (yAdjust) { + wStyle.marginTop = Math.ceil(yAdjust) + 'px'; + wStyle.marginBottom = yAdjust + 'px'; + } + + } + + return wrapper; + + }; + +})()); \ No newline at end of file diff --git a/storefront/static/common/js/function.js b/storefront/static/common/js/function.js new file mode 100644 index 0000000..42b5f5c --- /dev/null +++ b/storefront/static/common/js/function.js @@ -0,0 +1,14 @@ +$(document).ready(function() { + $('.tab ul li').click(function() { + $('.tab ul li').children('a').removeClass('active'); + $(this).children('a').addClass('active'); + for ( var i = 1; i < 4; i++) { + $('#content' + i).hide(); + } + $('#conten' + $(this).children('a').attr('rel')).show(); + }); +}); + +function go_chat() { + $('#habla_topbar_div').click(); $('#habla_wcsend_input').focus() +} \ No newline at end of file diff --git a/storefront/static/common/js/function01.js b/storefront/static/common/js/function01.js new file mode 100644 index 0000000..5d182ff --- /dev/null +++ b/storefront/static/common/js/function01.js @@ -0,0 +1,15 @@ +$(document).ready(function(){ + $('.work_tab ul li').click(function(){ + var tabs = $(this).parent().children('.work_tab ul li').children('a'); + tabs.removeClass('active'); + var tgt = $(this).children('a').attr('rel'); + $('.'+tgt).show(); + tabs.each(function() { + var rel = $(this).attr('rel'); + if (rel != tgt) { + $('.'+rel).hide(); + } + }); + $(this).children('a').addClass('active'); + }); +}); diff --git a/storefront/static/common/js/jquery-1.6.1.min.js b/storefront/static/common/js/jquery-1.6.1.min.js new file mode 100644 index 0000000..b2ac174 --- /dev/null +++ b/storefront/static/common/js/jquery-1.6.1.min.js @@ -0,0 +1,18 @@ +/*! + * jQuery JavaScript Library v1.6.1 + * http://jquery.com/ + * + * Copyright 2011, John Resig + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * Includes Sizzle.js + * http://sizzlejs.com/ + * Copyright 2011, The Dojo Foundation + * Released under the MIT, BSD, and GPL Licenses. + * + * Date: Thu May 12 15:04:36 2011 -0400 + */ +(function(a,b){function cy(a){return f.isWindow(a)?a:a.nodeType===9?a.defaultView||a.parentWindow:!1}function cv(a){if(!cj[a]){var b=f("<"+a+">").appendTo("body"),d=b.css("display");b.remove();if(d==="none"||d===""){ck||(ck=c.createElement("iframe"),ck.frameBorder=ck.width=ck.height=0),c.body.appendChild(ck);if(!cl||!ck.createElement)cl=(ck.contentWindow||ck.contentDocument).document,cl.write("");b=cl.createElement(a),cl.body.appendChild(b),d=f.css(b,"display"),c.body.removeChild(ck)}cj[a]=d}return cj[a]}function cu(a,b){var c={};f.each(cp.concat.apply([],cp.slice(0,b)),function(){c[this]=a});return c}function ct(){cq=b}function cs(){setTimeout(ct,0);return cq=f.now()}function ci(){try{return new a.ActiveXObject("Microsoft.XMLHTTP")}catch(b){}}function ch(){try{return new a.XMLHttpRequest}catch(b){}}function cb(a,c){a.dataFilter&&(c=a.dataFilter(c,a.dataType));var d=a.dataTypes,e={},g,h,i=d.length,j,k=d[0],l,m,n,o,p;for(g=1;g=0===c})}function W(a){return!a||!a.parentNode||a.parentNode.nodeType===11}function O(a,b){return(a&&a!=="*"?a+".":"")+b.replace(A,"`").replace(B,"&")}function N(a){var b,c,d,e,g,h,i,j,k,l,m,n,o,p=[],q=[],r=f._data(this,"events");if(!(a.liveFired===this||!r||!r.live||a.target.disabled||a.button&&a.type==="click")){a.namespace&&(n=new RegExp("(^|\\.)"+a.namespace.split(".").join("\\.(?:.*\\.)?")+"(\\.|$)")),a.liveFired=this;var s=r.live.slice(0);for(i=0;ic)break;a.currentTarget=e.elem,a.data=e.handleObj.data,a.handleObj=e.handleObj,o=e.handleObj.origHandler.apply(e.elem,arguments);if(o===!1||a.isPropagationStopped()){c=e.level,o===!1&&(b=!1);if(a.isImmediatePropagationStopped())break}}return b}}function L(a,c,d){var e=f.extend({},d[0]);e.type=a,e.originalEvent={},e.liveFired=b,f.event.handle.call(c,e),e.isDefaultPrevented()&&d[0].preventDefault()}function F(){return!0}function E(){return!1}function m(a,c,d){var e=c+"defer",g=c+"queue",h=c+"mark",i=f.data(a,e,b,!0);i&&(d==="queue"||!f.data(a,g,b,!0))&&(d==="mark"||!f.data(a,h,b,!0))&&setTimeout(function(){!f.data(a,g,b,!0)&&!f.data(a,h,b,!0)&&(f.removeData(a,e,!0),i.resolve())},0)}function l(a){for(var b in a)if(b!=="toJSON")return!1;return!0}function k(a,c,d){if(d===b&&a.nodeType===1){var e="data-"+c.replace(j,"$1-$2").toLowerCase();d=a.getAttribute(e);if(typeof d=="string"){try{d=d==="true"?!0:d==="false"?!1:d==="null"?null:f.isNaN(d)?i.test(d)?f.parseJSON(d):d:parseFloat(d)}catch(g){}f.data(a,c,d)}else d=b}return d}var c=a.document,d=a.navigator,e=a.location,f=function(){function H(){if(!e.isReady){try{c.documentElement.doScroll("left")}catch(a){setTimeout(H,1);return}e.ready()}}var e=function(a,b){return new e.fn.init(a,b,h)},f=a.jQuery,g=a.$,h,i=/^(?:[^<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/,j=/\S/,k=/^\s+/,l=/\s+$/,m=/\d/,n=/^<(\w+)\s*\/?>(?:<\/\1>)?$/,o=/^[\],:{}\s]*$/,p=/\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g,q=/"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g,r=/(?:^|:|,)(?:\s*\[)+/g,s=/(webkit)[ \/]([\w.]+)/,t=/(opera)(?:.*version)?[ \/]([\w.]+)/,u=/(msie) ([\w.]+)/,v=/(mozilla)(?:.*? rv:([\w.]+))?/,w=d.userAgent,x,y,z,A=Object.prototype.toString,B=Object.prototype.hasOwnProperty,C=Array.prototype.push,D=Array.prototype.slice,E=String.prototype.trim,F=Array.prototype.indexOf,G={};e.fn=e.prototype={constructor:e,init:function(a,d,f){var g,h,j,k;if(!a)return this;if(a.nodeType){this.context=this[0]=a,this.length=1;return this}if(a==="body"&&!d&&c.body){this.context=c,this[0]=c.body,this.selector=a,this.length=1;return this}if(typeof a=="string"){a.charAt(0)!=="<"||a.charAt(a.length-1)!==">"||a.length<3?g=i.exec(a):g=[null,a,null];if(g&&(g[1]||!d)){if(g[1]){d=d instanceof e?d[0]:d,k=d?d.ownerDocument||d:c,j=n.exec(a),j?e.isPlainObject(d)?(a=[c.createElement(j[1])],e.fn.attr.call(a,d,!0)):a=[k.createElement(j[1])]:(j=e.buildFragment([g[1]],[k]),a=(j.cacheable?e.clone(j.fragment):j.fragment).childNodes);return e.merge(this,a)}h=c.getElementById(g[2]);if(h&&h.parentNode){if(h.id!==g[2])return f.find(a);this.length=1,this[0]=h}this.context=c,this.selector=a;return this}return!d||d.jquery?(d||f).find(a):this.constructor(d).find(a)}if(e.isFunction(a))return f.ready(a);a.selector!==b&&(this.selector=a.selector,this.context=a.context);return e.makeArray(a,this)},selector:"",jquery:"1.6.1",length:0,size:function(){return this.length},toArray:function(){return D.call(this,0)},get:function(a){return a==null?this.toArray():a<0?this[this.length+a]:this[a]},pushStack:function(a,b,c){var d=this.constructor();e.isArray(a)?C.apply(d,a):e.merge(d,a),d.prevObject=this,d.context=this.context,b==="find"?d.selector=this.selector+(this.selector?" ":"")+c:b&&(d.selector=this.selector+"."+b+"("+c+")");return d},each:function(a,b){return e.each(this,a,b)},ready:function(a){e.bindReady(),y.done(a);return this},eq:function(a){return a===-1?this.slice(a):this.slice(a,+a+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(D.apply(this,arguments),"slice",D.call(arguments).join(","))},map:function(a){return this.pushStack(e.map(this,function(b,c){return a.call(b,c,b)}))},end:function(){return this.prevObject||this.constructor(null)},push:C,sort:[].sort,splice:[].splice},e.fn.init.prototype=e.fn,e.extend=e.fn.extend=function(){var a,c,d,f,g,h,i=arguments[0]||{},j=1,k=arguments.length,l=!1;typeof i=="boolean"&&(l=i,i=arguments[1]||{},j=2),typeof i!="object"&&!e.isFunction(i)&&(i={}),k===j&&(i=this,--j);for(;j0)return;y.resolveWith(c,[e]),e.fn.trigger&&e(c).trigger("ready").unbind("ready")}},bindReady:function(){if(!y){y=e._Deferred();if(c.readyState==="complete")return setTimeout(e.ready,1);if(c.addEventListener)c.addEventListener("DOMContentLoaded",z,!1),a.addEventListener("load",e.ready,!1);else if(c.attachEvent){c.attachEvent("onreadystatechange",z),a.attachEvent("onload",e.ready);var b=!1;try{b=a.frameElement==null}catch(d){}c.documentElement.doScroll&&b&&H()}}},isFunction:function(a){return e.type(a)==="function"},isArray:Array.isArray||function(a){return e.type(a)==="array"},isWindow:function(a){return a&&typeof a=="object"&&"setInterval"in a},isNaN:function(a){return a==null||!m.test(a)||isNaN(a)},type:function(a){return a==null?String(a):G[A.call(a)]||"object"},isPlainObject:function(a){if(!a||e.type(a)!=="object"||a.nodeType||e.isWindow(a))return!1;if(a.constructor&&!B.call(a,"constructor")&&!B.call(a.constructor.prototype,"isPrototypeOf"))return!1;var c;for(c in a);return c===b||B.call(a,c)},isEmptyObject:function(a){for(var b in a)return!1;return!0},error:function(a){throw a},parseJSON:function(b){if(typeof b!="string"||!b)return null;b=e.trim(b);if(a.JSON&&a.JSON.parse)return a.JSON.parse(b);if(o.test(b.replace(p,"@").replace(q,"]").replace(r,"")))return(new Function("return "+b))();e.error("Invalid JSON: "+b)},parseXML:function(b,c,d){a.DOMParser?(d=new DOMParser,c=d.parseFromString(b,"text/xml")):(c=new ActiveXObject("Microsoft.XMLDOM"),c.async="false",c.loadXML(b)),d=c.documentElement,(!d||!d.nodeName||d.nodeName==="parsererror")&&e.error("Invalid XML: "+b);return c},noop:function(){},globalEval:function(b){b&&j.test(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toUpperCase()===b.toUpperCase()},each:function(a,c,d){var f,g=0,h=a.length,i=h===b||e.isFunction(a);if(d){if(i){for(f in a)if(c.apply(a[f],d)===!1)break}else for(;g0&&a[0]&&a[j-1]||j===0||e.isArray(a));if(k)for(;i1?h.call(arguments,0):c,--e||g.resolveWith(g,h.call(b,0))}}var b=arguments,c=0,d=b.length,e=d,g=d<=1&&a&&f.isFunction(a.promise)?a:f.Deferred();if(d>1){for(;c
    a",d=a.getElementsByTagName("*"),e=a.getElementsByTagName("a")[0];if(!d||!d.length||!e)return{};f=c.createElement("select"),g=f.appendChild(c.createElement("option")),h=a.getElementsByTagName("input")[0],j={leadingWhitespace:a.firstChild.nodeType===3,tbody:!a.getElementsByTagName("tbody").length,htmlSerialize:!!a.getElementsByTagName("link").length,style:/top/.test(e.getAttribute("style")),hrefNormalized:e.getAttribute("href")==="/a",opacity:/^0.55$/.test(e.style.opacity),cssFloat:!!e.style.cssFloat,checkOn:h.value==="on",optSelected:g.selected,getSetAttribute:a.className!=="t",submitBubbles:!0,changeBubbles:!0,focusinBubbles:!1,deleteExpando:!0,noCloneEvent:!0,inlineBlockNeedsLayout:!1,shrinkWrapBlocks:!1,reliableMarginRight:!0},h.checked=!0,j.noCloneChecked=h.cloneNode(!0).checked,f.disabled=!0,j.optDisabled=!g.disabled;try{delete a.test}catch(s){j.deleteExpando=!1}!a.addEventListener&&a.attachEvent&&a.fireEvent&&(a.attachEvent("onclick",function b(){j.noCloneEvent=!1,a.detachEvent("onclick",b)}),a.cloneNode(!0).fireEvent("onclick")),h=c.createElement("input"),h.value="t",h.setAttribute("type","radio"),j.radioValue=h.value==="t",h.setAttribute("checked","checked"),a.appendChild(h),k=c.createDocumentFragment(),k.appendChild(a.firstChild),j.checkClone=k.cloneNode(!0).cloneNode(!0).lastChild.checked,a.innerHTML="",a.style.width=a.style.paddingLeft="1px",l=c.createElement("body"),m={visibility:"hidden",width:0,height:0,border:0,margin:0,background:"none"};for(q in m)l.style[q]=m[q];l.appendChild(a),b.insertBefore(l,b.firstChild),j.appendChecked=h.checked,j.boxModel=a.offsetWidth===2,"zoom"in a.style&&(a.style.display="inline",a.style.zoom=1,j.inlineBlockNeedsLayout=a.offsetWidth===2,a.style.display="",a.innerHTML="
    ",j.shrinkWrapBlocks=a.offsetWidth!==2),a.innerHTML="
    t
    ",n=a.getElementsByTagName("td"),r=n[0].offsetHeight===0,n[0].style.display="",n[1].style.display="none",j.reliableHiddenOffsets=r&&n[0].offsetHeight===0,a.innerHTML="",c.defaultView&&c.defaultView.getComputedStyle&&(i=c.createElement("div"),i.style.width="0",i.style.marginRight="0",a.appendChild(i),j.reliableMarginRight=(parseInt((c.defaultView.getComputedStyle(i,null)||{marginRight:0}).marginRight,10)||0)===0),l.innerHTML="",b.removeChild(l);if(a.attachEvent)for(q in{submit:1,change:1,focusin:1})p="on"+q,r=p in a,r||(a.setAttribute(p,"return;"),r=typeof a[p]=="function"),j[q+"Bubbles"]=r;return j}(),f.boxModel=f.support.boxModel;var i=/^(?:\{.*\}|\[.*\])$/,j=/([a-z])([A-Z])/g;f.extend({cache:{},uuid:0,expando:"jQuery"+(f.fn.jquery+Math.random()).replace(/\D/g,""),noData:{embed:!0,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",applet:!0},hasData:function(a){a=a.nodeType?f.cache[a[f.expando]]:a[f.expando];return!!a&&!l(a)},data:function(a,c,d,e){if(!!f.acceptData(a)){var g=f.expando,h=typeof c=="string",i,j=a.nodeType,k=j?f.cache:a,l=j?a[f.expando]:a[f.expando]&&f.expando;if((!l||e&&l&&!k[l][g])&&h&&d===b)return;l||(j?a[f.expando]=l=++f.uuid:l=f.expando),k[l]||(k[l]={},j||(k[l].toJSON=f.noop));if(typeof c=="object"||typeof c=="function")e?k[l][g]=f.extend(k[l][g],c):k[l]=f.extend(k[l],c);i=k[l],e&&(i[g]||(i[g]={}),i=i[g]),d!==b&&(i[f.camelCase(c)]=d);if(c==="events"&&!i[c])return i[g]&&i[g].events;return h?i[f.camelCase(c)]:i}},removeData:function(b,c,d){if(!!f.acceptData(b)){var e=f.expando,g=b.nodeType,h=g?f.cache:b,i=g?b[f.expando]:f.expando;if(!h[i])return;if(c){var j=d?h[i][e]:h[i];if(j){delete j[c];if(!l(j))return}}if(d){delete h[i][e];if(!l(h[i]))return}var k=h[i][e];f.support.deleteExpando||h!=a?delete h[i]:h[i]=null,k?(h[i]={},g||(h[i].toJSON=f.noop),h[i][e]=k):g&&(f.support.deleteExpando?delete b[f.expando]:b.removeAttribute?b.removeAttribute(f.expando):b[f.expando]=null)}},_data:function(a,b,c){return f.data(a,b,c,!0)},acceptData:function(a){if(a.nodeName){var b=f.noData[a.nodeName.toLowerCase()];if(b)return b!==!0&&a.getAttribute("classid")===b}return!0}}),f.fn.extend({data:function(a,c){var d=null;if(typeof a=="undefined"){if(this.length){d=f.data(this[0]);if(this[0].nodeType===1){var e=this[0].attributes,g;for(var h=0,i=e.length;h-1)return!0;return!1},val:function(a){var c,d,e=this[0];if(!arguments.length){if(e){c=f.valHooks[e.nodeName.toLowerCase()]||f.valHooks[e.type];if(c&&"get"in c&&(d=c.get(e,"value"))!==b)return d;return(e.value||"").replace(p,"")}return b}var g=f.isFunction(a);return this.each(function(d){var e=f(this),h;if(this.nodeType===1){g?h=a.call(this,d,e.val()):h=a,h==null?h="":typeof h=="number"?h+="":f.isArray(h)&&(h=f.map(h,function(a){return a==null?"":a+""})),c=f.valHooks[this.nodeName.toLowerCase()]||f.valHooks[this.type];if(!c||!("set"in c)||c.set(this,h,"value")===b)this.value=h}})}}),f.extend({valHooks:{option:{get:function(a){var b=a.attributes.value;return!b||b.specified?a.value:a.text}},select:{get:function(a){var b,c=a.selectedIndex,d=[],e=a.options,g=a.type==="select-one";if(c<0)return null;for(var h=g?c:0,i=g?c+1:e.length;h=0}),c.length||(a.selectedIndex=-1);return c}}},attrFn:{val:!0,css:!0,html:!0,text:!0,data:!0,width:!0,height:!0,offset:!0},attrFix:{tabindex:"tabIndex"},attr:function(a,c,d,e){var g=a.nodeType;if(!a||g===3||g===8||g===2)return b;if(e&&c in f.attrFn)return f(a)[c](d);if(!("getAttribute"in a))return f.prop(a,c,d);var h,i,j=g!==1||!f.isXMLDoc(a);c=j&&f.attrFix[c]||c,i=f.attrHooks[c],i||(!t.test(c)||typeof d!="boolean"&&d!==b&&d.toLowerCase()!==c.toLowerCase()?v&&(f.nodeName(a,"form")||u.test(c))&&(i=v):i=w);if(d!==b){if(d===null){f.removeAttr(a,c);return b}if(i&&"set"in i&&j&&(h=i.set(a,d,c))!==b)return h;a.setAttribute(c,""+d);return d}if(i&&"get"in i&&j)return i.get(a,c);h=a.getAttribute(c);return h===null?b:h},removeAttr:function(a,b){var c;a.nodeType===1&&(b=f.attrFix[b]||b,f.support.getSetAttribute?a.removeAttribute(b):(f.attr(a,b,""),a.removeAttributeNode(a.getAttributeNode(b))),t.test(b)&&(c=f.propFix[b]||b)in a&&(a[c]=!1))},attrHooks:{type:{set:function(a,b){if(q.test(a.nodeName)&&a.parentNode)f.error("type property can't be changed");else if(!f.support.radioValue&&b==="radio"&&f.nodeName(a,"input")){var c=a.value;a.setAttribute("type",b),c&&(a.value=c);return b}}},tabIndex:{get:function(a){var c=a.getAttributeNode("tabIndex");return c&&c.specified?parseInt(c.value,10):r.test(a.nodeName)||s.test(a.nodeName)&&a.href?0:b}}},propFix:{tabindex:"tabIndex",readonly:"readOnly","for":"htmlFor","class":"className",maxlength:"maxLength",cellspacing:"cellSpacing",cellpadding:"cellPadding",rowspan:"rowSpan",colspan:"colSpan",usemap:"useMap",frameborder:"frameBorder",contenteditable:"contentEditable"},prop:function(a,c,d){var e=a.nodeType;if(!a||e===3||e===8||e===2)return b;var g,h,i=e!==1||!f.isXMLDoc(a);c=i&&f.propFix[c]||c,h=f.propHooks[c];return d!==b?h&&"set"in h&&(g=h.set(a,d,c))!==b?g:a[c]=d:h&&"get"in h&&(g=h.get(a,c))!==b?g:a[c]},propHooks:{}}),w={get:function(a,c){return a[f.propFix[c]||c]?c.toLowerCase():b},set:function(a,b,c){var d;b===!1?f.removeAttr(a,c):(d=f.propFix[c]||c,d in a&&(a[d]=b),a.setAttribute(c,c.toLowerCase()));return c}},f.attrHooks.value={get:function(a,b){if(v&&f.nodeName(a,"button"))return v.get(a,b);return a.value},set:function(a,b,c){if(v&&f.nodeName(a,"button"))return v.set(a,b,c);a.value=b}},f.support.getSetAttribute||(f.attrFix=f.propFix,v=f.attrHooks.name=f.valHooks.button={get:function(a,c){var d;d=a.getAttributeNode(c);return d&&d.nodeValue!==""?d.nodeValue:b},set:function(a,b,c){var d=a.getAttributeNode(c);if(d){d.nodeValue=b;return b}}},f.each(["width","height"],function(a,b){f.attrHooks[b]=f.extend(f.attrHooks[b],{set:function(a,c){if(c===""){a.setAttribute(b,"auto");return c}}})})),f.support.hrefNormalized||f.each(["href","src","width","height"],function(a,c){f.attrHooks[c]=f.extend(f.attrHooks[c],{get:function(a){var d=a.getAttribute(c,2);return d===null?b:d}})}),f.support.style||(f.attrHooks.style={get:function(a){return a.style.cssText.toLowerCase()||b},set:function(a,b){return a.style.cssText=""+b}}),f.support.optSelected||(f.propHooks.selected=f.extend(f.propHooks.selected,{get:function(a){var b=a.parentNode;b&&(b.selectedIndex,b.parentNode&&b.parentNode.selectedIndex)}})),f.support.checkOn||f.each(["radio","checkbox"],function(){f.valHooks[this]={get:function(a){return a.getAttribute("value")===null?"on":a.value}}}),f.each(["radio","checkbox"],function(){f.valHooks[this]=f.extend(f.valHooks[this],{set:function(a,b){if(f.isArray(b))return a.checked=f.inArray(f(a).val(),b)>=0}})});var x=Object.prototype.hasOwnProperty,y=/\.(.*)$/,z=/^(?:textarea|input|select)$/i,A=/\./g,B=/ /g,C=/[^\w\s.|`]/g,D=function(a){return a.replace(C,"\\$&")};f.event={add:function(a,c,d,e){if(a.nodeType!==3&&a.nodeType!==8){if(d===!1)d=E;else if(!d)return;var g,h;d.handler&&(g=d,d=g.handler),d.guid||(d.guid=f.guid++);var i=f._data(a);if(!i)return;var j=i.events,k=i.handle;j||(i.events=j={}),k||(i.handle=k=function(a){return typeof f!="undefined"&&(!a||f.event.triggered!==a.type)?f.event.handle.apply(k.elem,arguments):b}),k.elem=a,c=c.split(" ");var l,m=0,n;while(l=c[m++]){h=g?f.extend({},g):{handler:d,data:e},l.indexOf(".")>-1?(n=l.split("."),l=n.shift(),h.namespace=n.slice(0).sort().join(".")):(n=[],h.namespace=""),h.type=l,h.guid||(h.guid=d.guid);var o=j[l],p=f.event.special[l]||{};if(!o){o=j[l]=[];if(!p.setup||p.setup.call(a,e,n,k)===!1)a.addEventListener?a.addEventListener(l,k,!1):a.attachEvent&&a.attachEvent("on"+l,k)}p.add&&(p.add.call(a,h),h.handler.guid||(h.handler.guid=d.guid)),o.push(h),f.event.global[l]=!0}a=null}},global:{},remove:function(a,c,d,e){if(a.nodeType!==3&&a.nodeType!==8){d===!1&&(d=E);var g,h,i,j,k=0,l,m,n,o,p,q,r,s=f.hasData(a)&&f._data(a),t=s&&s.events;if(!s||!t)return;c&&c.type&&(d=c.handler,c=c.type);if(!c||typeof c=="string"&&c.charAt(0)==="."){c=c||"";for(h in t)f.event.remove(a,h+c);return}c=c.split(" ");while(h=c[k++]){r=h,q=null,l=h.indexOf(".")<0,m=[],l||(m=h.split("."),h=m.shift(),n=new RegExp("(^|\\.)"+f.map(m.slice(0).sort(),D).join("\\.(?:.*\\.)?")+"(\\.|$)")),p=t[h];if(!p)continue;if(!d){for(j=0;j=0&&(h=h.slice(0,-1),j=!0),h.indexOf(".")>=0&&(i=h.split("."),h=i.shift(),i.sort());if(!!e&&!f.event.customEvent[h]||!!f.event.global[h]){c=typeof c=="object"?c[f.expando]?c:new f.Event(h,c):new f.Event(h),c.type=h,c.exclusive=j,c.namespace=i.join("."),c.namespace_re=new RegExp("(^|\\.)"+i.join("\\.(?:.*\\.)?")+"(\\.|$)");if(g||!e)c.preventDefault(),c.stopPropagation();if(!e){f.each(f.cache,function(){var a=f.expando,b=this[a];b&&b.events&&b.events[h]&&f.event.trigger(c,d,b.handle.elem +)});return}if(e.nodeType===3||e.nodeType===8)return;c.result=b,c.target=e,d=d?f.makeArray(d):[],d.unshift(c);var k=e,l=h.indexOf(":")<0?"on"+h:"";do{var m=f._data(k,"handle");c.currentTarget=k,m&&m.apply(k,d),l&&f.acceptData(k)&&k[l]&&k[l].apply(k,d)===!1&&(c.result=!1,c.preventDefault()),k=k.parentNode||k.ownerDocument||k===c.target.ownerDocument&&a}while(k&&!c.isPropagationStopped());if(!c.isDefaultPrevented()){var n,o=f.event.special[h]||{};if((!o._default||o._default.call(e.ownerDocument,c)===!1)&&(h!=="click"||!f.nodeName(e,"a"))&&f.acceptData(e)){try{l&&e[h]&&(n=e[l],n&&(e[l]=null),f.event.triggered=h,e[h]())}catch(p){}n&&(e[l]=n),f.event.triggered=b}}return c.result}},handle:function(c){c=f.event.fix(c||a.event);var d=((f._data(this,"events")||{})[c.type]||[]).slice(0),e=!c.exclusive&&!c.namespace,g=Array.prototype.slice.call(arguments,0);g[0]=c,c.currentTarget=this;for(var h=0,i=d.length;h-1?f.map(a.options,function(a){return a.selected}).join("-"):"":f.nodeName(a,"select")&&(c=a.selectedIndex);return c},K=function(c){var d=c.target,e,g;if(!!z.test(d.nodeName)&&!d.readOnly){e=f._data(d,"_change_data"),g=J(d),(c.type!=="focusout"||d.type!=="radio")&&f._data(d,"_change_data",g);if(e===b||g===e)return;if(e!=null||g)c.type="change",c.liveFired=b,f.event.trigger(c,arguments[1],d)}};f.event.special.change={filters:{focusout:K,beforedeactivate:K,click:function(a){var b=a.target,c=f.nodeName(b,"input")?b.type:"";(c==="radio"||c==="checkbox"||f.nodeName(b,"select"))&&K.call(this,a)},keydown:function(a){var b=a.target,c=f.nodeName(b,"input")?b.type:"";(a.keyCode===13&&!f.nodeName(b,"textarea")||a.keyCode===32&&(c==="checkbox"||c==="radio")||c==="select-multiple")&&K.call(this,a)},beforeactivate:function(a){var b=a.target;f._data(b,"_change_data",J(b))}},setup:function(a,b){if(this.type==="file")return!1;for(var c in I)f.event.add(this,c+".specialChange",I[c]);return z.test(this.nodeName)},teardown:function(a){f.event.remove(this,".specialChange");return z.test(this.nodeName)}},I=f.event.special.change.filters,I.focus=I.beforeactivate}f.support.focusinBubbles||f.each({focus:"focusin",blur:"focusout"},function(a,b){function e(a){var c=f.event.fix(a);c.type=b,c.originalEvent={},f.event.trigger(c,null,c.target),c.isDefaultPrevented()&&a.preventDefault()}var d=0;f.event.special[b]={setup:function(){d++===0&&c.addEventListener(a,e,!0)},teardown:function(){--d===0&&c.removeEventListener(a,e,!0)}}}),f.each(["bind","one"],function(a,c){f.fn[c]=function(a,d,e){var g;if(typeof a=="object"){for(var h in a)this[c](h,d,a[h],e);return this}if(arguments.length===2||d===!1)e=d,d=b;c==="one"?(g=function(a){f(this).unbind(a,g);return e.apply(this,arguments)},g.guid=e.guid||f.guid++):g=e;if(a==="unload"&&c!=="one")this.one(a,d,e);else for(var i=0,j=this.length;i0?this.bind(b,a,c):this.trigger(b)},f.attrFn&&(f.attrFn[b]=!0)}),function(){function u(a,b,c,d,e,f){for(var g=0,h=d.length;g0){j=i;break}}i=i[a]}d[g]=j}}}function t(a,b,c,d,e,f){for(var g=0,h=d.length;g+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g,d=0,e=Object.prototype.toString,g=!1,h=!0,i=/\\/g,j=/\W/;[0,0].sort(function(){h=!1;return 0});var k=function(b,d,f,g){f=f||[],d=d||c;var h=d;if(d.nodeType!==1&&d.nodeType!==9)return[];if(!b||typeof b!="string")return f;var i,j,n,o,q,r,s,t,u=!0,w=k.isXML(d),x=[],y=b;do{a.exec(""),i=a.exec(y);if(i){y=i[3],x.push(i[1]);if(i[2]){o=i[3];break}}}while(i);if(x.length>1&&m.exec(b))if(x.length===2&&l.relative[x[0]])j=v(x[0]+x[1],d);else{j=l.relative[x[0]]?[d]:k(x.shift(),d);while(x.length)b=x.shift(),l.relative[b]&&(b+=x.shift()),j=v(b,j)}else{!g&&x.length>1&&d.nodeType===9&&!w&&l.match.ID.test(x[0])&&!l.match.ID.test(x[x.length-1])&&(q=k.find(x.shift(),d,w),d=q.expr?k.filter(q.expr,q.set)[0]:q.set[0]);if(d){q=g?{expr:x.pop(),set:p(g)}:k.find(x.pop(),x.length===1&&(x[0]==="~"||x[0]==="+")&&d.parentNode?d.parentNode:d,w),j=q.expr?k.filter(q.expr,q.set):q.set,x.length>0?n=p(j):u=!1;while(x.length)r=x.pop(),s=r,l.relative[r]?s=x.pop():r="",s==null&&(s=d),l.relative[r](n,s,w)}else n=x=[]}n||(n=j),n||k.error(r||b);if(e.call(n)==="[object Array]")if(!u)f.push.apply(f,n);else if(d&&d.nodeType===1)for(t=0;n[t]!=null;t++)n[t]&&(n[t]===!0||n[t].nodeType===1&&k.contains(d,n[t]))&&f.push(j[t]);else for(t=0;n[t]!=null;t++)n[t]&&n[t].nodeType===1&&f.push(j[t]);else p(n,f);o&&(k(o,h,f,g),k.uniqueSort(f));return f};k.uniqueSort=function(a){if(r){g=h,a.sort(r);if(g)for(var b=1;b0},k.find=function(a,b,c){var d;if(!a)return[];for(var e=0,f=l.order.length;e":function(a,b){var c,d=typeof b=="string",e=0,f=a.length;if(d&&!j.test(b)){b=b.toLowerCase();for(;e=0)?c||d.push(h):c&&(b[g]=!1));return!1},ID:function(a){return a[1].replace(i,"")},TAG:function(a,b){return a[1].replace(i,"").toLowerCase()},CHILD:function(a){if(a[1]==="nth"){a[2]||k.error(a[0]),a[2]=a[2].replace(/^\+|\s*/g,"");var b=/(-?)(\d*)(?:n([+\-]?\d*))?/.exec(a[2]==="even"&&"2n"||a[2]==="odd"&&"2n+1"||!/\D/.test(a[2])&&"0n+"+a[2]||a[2]);a[2]=b[1]+(b[2]||1)-0,a[3]=b[3]-0}else a[2]&&k.error(a[0]);a[0]=d++;return a},ATTR:function(a,b,c,d,e,f){var g=a[1]=a[1].replace(i,"");!f&&l.attrMap[g]&&(a[1]=l.attrMap[g]),a[4]=(a[4]||a[5]||"").replace(i,""),a[2]==="~="&&(a[4]=" "+a[4]+" ");return a},PSEUDO:function(b,c,d,e,f){if(b[1]==="not")if((a.exec(b[3])||"").length>1||/^\w/.test(b[3]))b[3]=k(b[3],null,null,c);else{var g=k.filter(b[3],c,d,!0^f);d||e.push.apply(e,g);return!1}else if(l.match.POS.test(b[0])||l.match.CHILD.test(b[0]))return!0;return b},POS:function(a){a.unshift(!0);return a}},filters:{enabled:function(a){return a.disabled===!1&&a.type!=="hidden"},disabled:function(a){return a.disabled===!0},checked:function(a){return a.checked===!0},selected:function(a){a.parentNode&&a.parentNode.selectedIndex;return a.selected===!0},parent:function(a){return!!a.firstChild},empty:function(a){return!a.firstChild},has:function(a,b,c){return!!k(c[3],a).length},header:function(a){return/h\d/i.test(a.nodeName)},text:function(a){var b=a.getAttribute("type"),c=a.type;return a.nodeName.toLowerCase()==="input"&&"text"===c&&(b===c||b===null)},radio:function(a){return a.nodeName.toLowerCase()==="input"&&"radio"===a.type},checkbox:function(a){return a.nodeName.toLowerCase()==="input"&&"checkbox"===a.type},file:function(a){return a.nodeName.toLowerCase()==="input"&&"file"===a.type},password:function(a){return a.nodeName.toLowerCase()==="input"&&"password"===a.type},submit:function(a){var b=a.nodeName.toLowerCase();return(b==="input"||b==="button")&&"submit"===a.type},image:function(a){return a.nodeName.toLowerCase()==="input"&&"image"===a.type},reset:function(a){var b=a.nodeName.toLowerCase();return(b==="input"||b==="button")&&"reset"===a.type},button:function(a){var b=a.nodeName.toLowerCase();return b==="input"&&"button"===a.type||b==="button"},input:function(a){return/input|select|textarea|button/i.test(a.nodeName)},focus:function(a){return a===a.ownerDocument.activeElement}},setFilters:{first:function(a,b){return b===0},last:function(a,b,c,d){return b===d.length-1},even:function(a,b){return b%2===0},odd:function(a,b){return b%2===1},lt:function(a,b,c){return bc[3]-0},nth:function(a,b,c){return c[3]-0===b},eq:function(a,b,c){return c[3]-0===b}},filter:{PSEUDO:function(a,b,c,d){var e=b[1],f=l.filters[e];if(f)return f(a,c,b,d);if(e==="contains")return(a.textContent||a.innerText||k.getText([a])||"").indexOf(b[3])>=0;if(e==="not"){var g=b[3];for(var h=0,i=g.length;h=0}},ID:function(a,b){return a.nodeType===1&&a.getAttribute("id")===b},TAG:function(a,b){return b==="*"&&a.nodeType===1||a.nodeName.toLowerCase()===b},CLASS:function(a,b){return(" "+(a.className||a.getAttribute("class"))+" ").indexOf(b)>-1},ATTR:function(a,b){var c=b[1],d=l.attrHandle[c]?l.attrHandle[c](a):a[c]!=null?a[c]:a.getAttribute(c),e=d+"",f=b[2],g=b[4];return d==null?f==="!=":f==="="?e===g:f==="*="?e.indexOf(g)>=0:f==="~="?(" "+e+" ").indexOf(g)>=0:g?f==="!="?e!==g:f==="^="?e.indexOf(g)===0:f==="$="?e.substr(e.length-g.length)===g:f==="|="?e===g||e.substr(0,g.length+1)===g+"-":!1:e&&d!==!1},POS:function(a,b,c,d){var e=b[2],f=l.setFilters[e];if(f)return f(a,c,b,d)}}},m=l.match.POS,n=function(a,b){return"\\"+(b-0+1)};for(var o in l.match)l.match[o]=new RegExp(l.match[o].source+/(?![^\[]*\])(?![^\(]*\))/.source),l.leftMatch[o]=new RegExp(/(^(?:.|\r|\n)*?)/.source+l.match[o].source.replace(/\\(\d+)/g,n));var p=function(a,b){a=Array.prototype.slice.call(a,0);if(b){b.push.apply(b,a);return b}return a};try{Array.prototype.slice.call(c.documentElement.childNodes,0)[0].nodeType}catch(q){p=function(a,b){var c=0,d=b||[];if(e.call(a)==="[object Array]")Array.prototype.push.apply(d,a);else if(typeof a.length=="number")for(var f=a.length;c",e.insertBefore(a,e.firstChild),c.getElementById(d)&&(l.find.ID=function(a,c,d){if(typeof c.getElementById!="undefined"&&!d){var e=c.getElementById(a[1]);return e?e.id===a[1]||typeof e.getAttributeNode!="undefined"&&e.getAttributeNode("id").nodeValue===a[1]?[e]:b:[]}},l.filter.ID=function(a,b){var c=typeof a.getAttributeNode!="undefined"&&a.getAttributeNode("id");return a.nodeType===1&&c&&c.nodeValue===b}),e.removeChild(a),e=a=null}(),function(){var a=c.createElement("div");a.appendChild(c.createComment("")),a.getElementsByTagName("*").length>0&&(l.find.TAG=function(a,b){var c=b.getElementsByTagName(a[1]);if(a[1]==="*"){var d=[];for(var e=0;c[e];e++)c[e].nodeType===1&&d.push(c[e]);c=d}return c}),a.innerHTML="",a.firstChild&&typeof a.firstChild.getAttribute!="undefined"&&a.firstChild.getAttribute("href")!=="#"&&(l.attrHandle.href=function(a){return a.getAttribute("href",2)}),a=null}(),c.querySelectorAll&&function(){var a=k,b=c.createElement("div"),d="__sizzle__";b.innerHTML="

    ";if(!b.querySelectorAll||b.querySelectorAll(".TEST").length!==0){k=function(b,e,f,g){e=e||c;if(!g&&!k.isXML(e)){var h=/^(\w+$)|^\.([\w\-]+$)|^#([\w\-]+$)/.exec(b);if(h&&(e.nodeType===1||e.nodeType===9)){if(h[1])return p(e.getElementsByTagName(b),f);if(h[2]&&l.find.CLASS&&e.getElementsByClassName)return p(e.getElementsByClassName(h[2]),f)}if(e.nodeType===9){if(b==="body"&&e.body)return p([e.body],f);if(h&&h[3]){var i=e.getElementById(h[3]);if(!i||!i.parentNode)return p([],f);if(i.id===h[3])return p([i],f)}try{return p(e.querySelectorAll(b),f)}catch(j){}}else if(e.nodeType===1&&e.nodeName.toLowerCase()!=="object"){var m=e,n=e.getAttribute("id"),o=n||d,q=e.parentNode,r=/^\s*[+~]/.test(b);n?o=o.replace(/'/g,"\\$&"):e.setAttribute("id",o),r&&q&&(e=e.parentNode);try{if(!r||q)return p(e.querySelectorAll("[id='"+o+"'] "+b),f)}catch(s){}finally{n||m.removeAttribute("id")}}}return a(b,e,f,g)};for(var e in a)k[e]=a[e];b=null}}(),function(){var a=c.documentElement,b=a.matchesSelector||a.mozMatchesSelector||a.webkitMatchesSelector||a.msMatchesSelector;if(b){var d=!b.call(c.createElement("div"),"div"),e=!1;try{b.call(c.documentElement,"[test!='']:sizzle")}catch(f){e=!0}k.matchesSelector=function(a,c){c=c.replace(/\=\s*([^'"\]]*)\s*\]/g,"='$1']");if(!k.isXML(a))try{if(e||!l.match.PSEUDO.test(c)&&!/!=/.test(c)){var f=b.call(a,c);if(f||!d||a.document&&a.document.nodeType!==11)return f}}catch(g){}return k(c,null,null,[a]).length>0}}}(),function(){var a=c.createElement("div");a.innerHTML="
    ";if(!!a.getElementsByClassName&&a.getElementsByClassName("e").length!==0){a.lastChild.className="e";if(a.getElementsByClassName("e").length===1)return;l.order.splice(1,0,"CLASS"),l.find.CLASS=function(a,b,c){if(typeof b.getElementsByClassName!="undefined"&&!c)return b.getElementsByClassName(a[1])},a=null}}(),c.documentElement.contains?k.contains=function(a,b){return a!==b&&(a.contains?a.contains(b):!0)}:c.documentElement.compareDocumentPosition?k.contains=function(a,b){return!!(a.compareDocumentPosition(b)&16)}:k.contains=function(){return!1},k.isXML=function(a){var b=(a?a.ownerDocument||a:0).documentElement;return b?b.nodeName!=="HTML":!1};var v=function(a,b){var c,d=[],e="",f=b.nodeType?[b]:b;while(c=l.match.PSEUDO.exec(a))e+=c[0],a=a.replace(l.match.PSEUDO,"");a=l.relative[a]?a+"*":a;for(var g=0,h=f.length;g0)for(h=g;h0:this.filter(a).length>0)},closest:function(a,b){var c=[],d,e,g=this[0];if(f.isArray(a)){var h,i,j={},k=1;if(g&&a.length){for(d=0,e=a.length;d-1:f(g).is(h))&&c.push({selector:i,elem:g,level:k});g=g.parentNode,k++}}return c}var l=U.test(a)||typeof a!="string"?f(a,b||this.context):0;for(d=0,e=this.length;d-1:f.find.matchesSelector(g,a)){c.push(g);break}g=g.parentNode;if(!g||!g.ownerDocument||g===b||g.nodeType===11)break}}c=c.length>1?f.unique(c):c;return this.pushStack(c,"closest",a)},index:function(a){if(!a||typeof a=="string")return f.inArray(this[0],a?f(a):this.parent().children());return f.inArray(a.jquery?a[0]:a,this)},add:function(a,b){var c=typeof a=="string"?f(a,b):f.makeArray(a&&a.nodeType?[a]:a),d=f.merge(this.get(),c);return this.pushStack(W(c[0])||W(d[0])?d:f.unique(d))},andSelf:function(){return this.add(this.prevObject)}}),f.each({parent:function(a){var b=a.parentNode;return b&&b.nodeType!==11?b:null},parents:function(a){return f.dir(a,"parentNode")},parentsUntil:function(a,b,c){return f.dir(a,"parentNode",c)},next:function(a){return f.nth(a,2,"nextSibling")},prev:function(a){return f.nth(a,2,"previousSibling")},nextAll:function(a){return f.dir(a,"nextSibling")},prevAll:function(a){return f.dir(a,"previousSibling")},nextUntil:function(a,b,c){return f.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return f.dir(a,"previousSibling",c)},siblings:function(a){return f.sibling(a.parentNode.firstChild,a)},children:function(a){return f.sibling(a.firstChild)},contents:function(a){return f.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:f.makeArray(a.childNodes)}},function(a,b){f.fn[a]=function(c,d){var e=f.map(this,b,c),g=T.call(arguments);P.test(a)||(d=c),d&&typeof d=="string"&&(e=f.filter(d,e)),e=this.length>1&&!V[a]?f.unique(e):e,(this.length>1||R.test(d))&&Q.test(a)&&(e=e.reverse());return this.pushStack(e,a,g.join(","))}}),f.extend({filter:function(a,b,c){c&&(a=":not("+a+")");return b.length===1?f.find.matchesSelector(b[0],a)?[b[0]]:[]:f.find.matches(a,b)},dir:function(a,c,d){var e=[],g=a[c];while(g&&g.nodeType!==9&&(d===b||g.nodeType!==1||!f(g).is(d)))g.nodeType===1&&e.push(g),g=g[c];return e},nth:function(a,b,c,d){b=b||1;var e=0;for(;a;a=a[c])if(a.nodeType===1&&++e===b)break;return a},sibling:function(a,b){var c=[];for(;a;a=a.nextSibling)a.nodeType===1&&a!==b&&c.push(a);return c}});var Y=/ jQuery\d+="(?:\d+|null)"/g,Z=/^\s+/,$=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/ig,_=/<([\w:]+)/,ba=/",""],legend:[1,"
    ","
    "],thead:[1,"","
    "],tr:[2,"","
    "],td:[3,"","
    "],col:[2,"","
    "],area:[1,"",""],_default:[0,"",""]};bg.optgroup=bg.option,bg.tbody=bg.tfoot=bg.colgroup=bg.caption=bg.thead,bg.th=bg.td,f.support.htmlSerialize||(bg._default=[1,"div
    ","
    "]),f.fn.extend({text:function(a){if(f.isFunction(a))return this.each(function(b){var c=f(this);c.text(a.call(this,b,c.text()))});if(typeof a!="object"&&a!==b)return this.empty().append((this[0]&&this[0].ownerDocument||c).createTextNode(a));return f.text(this)},wrapAll:function(a){if(f.isFunction(a))return this.each(function(b){f(this).wrapAll(a.call(this,b))});if(this[0]){var b=f(a,this[0].ownerDocument).eq(0).clone(!0);this[0].parentNode&&b.insertBefore(this[0]),b.map(function(){var a=this;while(a.firstChild&&a.firstChild.nodeType===1)a=a.firstChild;return a}).append(this)}return this},wrapInner:function(a){if(f.isFunction(a))return this.each(function(b){f(this).wrapInner(a.call(this,b))});return this.each(function(){var b=f(this),c=b.contents();c.length?c.wrapAll(a):b.append(a)})},wrap:function(a){return this.each(function(){f(this).wrapAll(a)})},unwrap:function(){return this.parent().each(function(){f.nodeName(this,"body")||f(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,!0,function(a){this.nodeType===1&&this.appendChild(a)})},prepend:function(){return this.domManip(arguments,!0,function(a){this.nodeType===1&&this.insertBefore(a,this.firstChild)})},before:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,!1,function(a){this.parentNode.insertBefore(a,this)});if(arguments.length){var a=f(arguments[0]);a.push.apply(a,this.toArray());return this.pushStack(a,"before",arguments)}},after:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,!1,function(a){this.parentNode.insertBefore(a,this.nextSibling)});if(arguments.length){var a=this.pushStack(this,"after",arguments);a.push.apply(a,f(arguments[0]).toArray());return a}},remove:function(a,b){for(var c=0,d;(d=this[c])!=null;c++)if(!a||f.filter(a,[d]).length)!b&&d.nodeType===1&&(f.cleanData(d.getElementsByTagName("*")),f.cleanData([d])),d.parentNode&&d.parentNode.removeChild(d);return this},empty:function(){for(var a=0,b;(b=this[a])!=null;a++){b.nodeType===1&&f.cleanData(b.getElementsByTagName("*"));while(b.firstChild)b.removeChild(b.firstChild)}return this},clone:function(a,b){a=a==null?!1:a,b=b==null?a:b;return this.map(function(){return f.clone(this,a,b)})},html:function(a){if(a===b)return this[0]&&this[0].nodeType===1?this[0].innerHTML.replace(Y,""):null;if(typeof a=="string"&&!bc.test(a)&&(f.support.leadingWhitespace||!Z.test(a))&&!bg[(_.exec(a)||["",""])[1].toLowerCase()]){a=a.replace($,"<$1>");try{for(var c=0,d=this.length;c1&&l0?this.clone(!0):this).get();f(e[h])[b](j),d=d.concat(j)}return this.pushStack(d,a,e.selector)}}),f.extend({clone:function(a,b,c){var d=a.cloneNode(!0),e,g,h;if((!f.support.noCloneEvent||!f.support.noCloneChecked)&&(a.nodeType===1||a.nodeType===11)&&!f.isXMLDoc(a)){bj(a,d),e=bk(a),g=bk(d);for(h=0;e[h];++h)bj(e[h],g[h])}if(b){bi(a,d);if(c){e=bk(a),g=bk(d);for(h=0;e[h];++h)bi(e[h],g[h])}}return d},clean:function(a,b,d,e){var g;b=b||c,typeof b.createElement=="undefined"&&(b=b.ownerDocument|| +b[0]&&b[0].ownerDocument||c);var h=[],i;for(var j=0,k;(k=a[j])!=null;j++){typeof k=="number"&&(k+="");if(!k)continue;if(typeof k=="string")if(!bb.test(k))k=b.createTextNode(k);else{k=k.replace($,"<$1>");var l=(_.exec(k)||["",""])[1].toLowerCase(),m=bg[l]||bg._default,n=m[0],o=b.createElement("div");o.innerHTML=m[1]+k+m[2];while(n--)o=o.lastChild;if(!f.support.tbody){var p=ba.test(k),q=l==="table"&&!p?o.firstChild&&o.firstChild.childNodes:m[1]===""&&!p?o.childNodes:[];for(i=q.length-1;i>=0;--i)f.nodeName(q[i],"tbody")&&!q[i].childNodes.length&&q[i].parentNode.removeChild(q[i])}!f.support.leadingWhitespace&&Z.test(k)&&o.insertBefore(b.createTextNode(Z.exec(k)[0]),o.firstChild),k=o.childNodes}var r;if(!f.support.appendChecked)if(k[0]&&typeof (r=k.length)=="number")for(i=0;i=0)return b+"px"}}}),f.support.opacity||(f.cssHooks.opacity={get:function(a,b){return bp.test((b&&a.currentStyle?a.currentStyle.filter:a.style.filter)||"")?parseFloat(RegExp.$1)/100+"":b?"1":""},set:function(a,b){var c=a.style,d=a.currentStyle;c.zoom=1;var e=f.isNaN(b)?"":"alpha(opacity="+b*100+")",g=d&&d.filter||c.filter||"";c.filter=bo.test(g)?g.replace(bo,e):g+" "+e}}),f(function(){f.support.reliableMarginRight||(f.cssHooks.marginRight={get:function(a,b){var c;f.swap(a,{display:"inline-block"},function(){b?c=bz(a,"margin-right","marginRight"):c=a.style.marginRight});return c}})}),c.defaultView&&c.defaultView.getComputedStyle&&(bA=function(a,c){var d,e,g;c=c.replace(br,"-$1").toLowerCase();if(!(e=a.ownerDocument.defaultView))return b;if(g=e.getComputedStyle(a,null))d=g.getPropertyValue(c),d===""&&!f.contains(a.ownerDocument.documentElement,a)&&(d=f.style(a,c));return d}),c.documentElement.currentStyle&&(bB=function(a,b){var c,d=a.currentStyle&&a.currentStyle[b],e=a.runtimeStyle&&a.runtimeStyle[b],f=a.style;!bs.test(d)&&bt.test(d)&&(c=f.left,e&&(a.runtimeStyle.left=a.currentStyle.left),f.left=b==="fontSize"?"1em":d||0,d=f.pixelLeft+"px",f.left=c,e&&(a.runtimeStyle.left=e));return d===""?"auto":d}),bz=bA||bB,f.expr&&f.expr.filters&&(f.expr.filters.hidden=function(a){var b=a.offsetWidth,c=a.offsetHeight;return b===0&&c===0||!f.support.reliableHiddenOffsets&&(a.style.display||f.css(a,"display"))==="none"},f.expr.filters.visible=function(a){return!f.expr.filters.hidden(a)});var bE=/%20/g,bF=/\[\]$/,bG=/\r?\n/g,bH=/#.*$/,bI=/^(.*?):[ \t]*([^\r\n]*)\r?$/mg,bJ=/^(?:color|date|datetime|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i,bK=/^(?:about|app|app\-storage|.+\-extension|file|widget):$/,bL=/^(?:GET|HEAD)$/,bM=/^\/\//,bN=/\?/,bO=/)<[^<]*)*<\/script>/gi,bP=/^(?:select|textarea)/i,bQ=/\s+/,bR=/([?&])_=[^&]*/,bS=/^([\w\+\.\-]+:)(?:\/\/([^\/?#:]*)(?::(\d+))?)?/,bT=f.fn.load,bU={},bV={},bW,bX;try{bW=e.href}catch(bY){bW=c.createElement("a"),bW.href="",bW=bW.href}bX=bS.exec(bW.toLowerCase())||[],f.fn.extend({load:function(a,c,d){if(typeof a!="string"&&bT)return bT.apply(this,arguments);if(!this.length)return this;var e=a.indexOf(" ");if(e>=0){var g=a.slice(e,a.length);a=a.slice(0,e)}var h="GET";c&&(f.isFunction(c)?(d=c,c=b):typeof c=="object"&&(c=f.param(c,f.ajaxSettings.traditional),h="POST"));var i=this;f.ajax({url:a,type:h,dataType:"html",data:c,complete:function(a,b,c){c=a.responseText,a.isResolved()&&(a.done(function(a){c=a}),i.html(g?f("
    ").append(c.replace(bO,"")).find(g):c)),d&&i.each(d,[c,b,a])}});return this},serialize:function(){return f.param(this.serializeArray())},serializeArray:function(){return this.map(function(){return this.elements?f.makeArray(this.elements):this}).filter(function(){return this.name&&!this.disabled&&(this.checked||bP.test(this.nodeName)||bJ.test(this.type))}).map(function(a,b){var c=f(this).val();return c==null?null:f.isArray(c)?f.map(c,function(a,c){return{name:b.name,value:a.replace(bG,"\r\n")}}):{name:b.name,value:c.replace(bG,"\r\n")}}).get()}}),f.each("ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split(" "),function(a,b){f.fn[b]=function(a){return this.bind(b,a)}}),f.each(["get","post"],function(a,c){f[c]=function(a,d,e,g){f.isFunction(d)&&(g=g||e,e=d,d=b);return f.ajax({type:c,url:a,data:d,success:e,dataType:g})}}),f.extend({getScript:function(a,c){return f.get(a,b,c,"script")},getJSON:function(a,b,c){return f.get(a,b,c,"json")},ajaxSetup:function(a,b){b?f.extend(!0,a,f.ajaxSettings,b):(b=a,a=f.extend(!0,f.ajaxSettings,b));for(var c in{context:1,url:1})c in b?a[c]=b[c]:c in f.ajaxSettings&&(a[c]=f.ajaxSettings[c]);return a},ajaxSettings:{url:bW,isLocal:bK.test(bX[1]),global:!0,type:"GET",contentType:"application/x-www-form-urlencoded",processData:!0,async:!0,accepts:{xml:"application/xml, text/xml",html:"text/html",text:"text/plain",json:"application/json, text/javascript","*":"*/*"},contents:{xml:/xml/,html:/html/,json:/json/},responseFields:{xml:"responseXML",text:"responseText"},converters:{"* text":a.String,"text html":!0,"text json":f.parseJSON,"text xml":f.parseXML}},ajaxPrefilter:bZ(bU),ajaxTransport:bZ(bV),ajax:function(a,c){function w(a,c,l,m){if(s!==2){s=2,q&&clearTimeout(q),p=b,n=m||"",v.readyState=a?4:0;var o,r,u,w=l?ca(d,v,l):b,x,y;if(a>=200&&a<300||a===304){if(d.ifModified){if(x=v.getResponseHeader("Last-Modified"))f.lastModified[k]=x;if(y=v.getResponseHeader("Etag"))f.etag[k]=y}if(a===304)c="notmodified",o=!0;else try{r=cb(d,w),c="success",o=!0}catch(z){c="parsererror",u=z}}else{u=c;if(!c||a)c="error",a<0&&(a=0)}v.status=a,v.statusText=c,o?h.resolveWith(e,[r,c,v]):h.rejectWith(e,[v,c,u]),v.statusCode(j),j=b,t&&g.trigger("ajax"+(o?"Success":"Error"),[v,d,o?r:u]),i.resolveWith(e,[v,c]),t&&(g.trigger("ajaxComplete",[v,d]),--f.active||f.event.trigger("ajaxStop"))}}typeof a=="object"&&(c=a,a=b),c=c||{};var d=f.ajaxSetup({},c),e=d.context||d,g=e!==d&&(e.nodeType||e instanceof f)?f(e):f.event,h=f.Deferred(),i=f._Deferred(),j=d.statusCode||{},k,l={},m={},n,o,p,q,r,s=0,t,u,v={readyState:0,setRequestHeader:function(a,b){if(!s){var c=a.toLowerCase();a=m[c]=m[c]||a,l[a]=b}return this},getAllResponseHeaders:function(){return s===2?n:null},getResponseHeader:function(a){var c;if(s===2){if(!o){o={};while(c=bI.exec(n))o[c[1].toLowerCase()]=c[2]}c=o[a.toLowerCase()]}return c===b?null:c},overrideMimeType:function(a){s||(d.mimeType=a);return this},abort:function(a){a=a||"abort",p&&p.abort(a),w(0,a);return this}};h.promise(v),v.success=v.done,v.error=v.fail,v.complete=i.done,v.statusCode=function(a){if(a){var b;if(s<2)for(b in a)j[b]=[j[b],a[b]];else b=a[v.status],v.then(b,b)}return this},d.url=((a||d.url)+"").replace(bH,"").replace(bM,bX[1]+"//"),d.dataTypes=f.trim(d.dataType||"*").toLowerCase().split(bQ),d.crossDomain==null&&(r=bS.exec(d.url.toLowerCase()),d.crossDomain=!(!r||r[1]==bX[1]&&r[2]==bX[2]&&(r[3]||(r[1]==="http:"?80:443))==(bX[3]||(bX[1]==="http:"?80:443)))),d.data&&d.processData&&typeof d.data!="string"&&(d.data=f.param(d.data,d.traditional)),b$(bU,d,c,v);if(s===2)return!1;t=d.global,d.type=d.type.toUpperCase(),d.hasContent=!bL.test(d.type),t&&f.active++===0&&f.event.trigger("ajaxStart");if(!d.hasContent){d.data&&(d.url+=(bN.test(d.url)?"&":"?")+d.data),k=d.url;if(d.cache===!1){var x=f.now(),y=d.url.replace(bR,"$1_="+x);d.url=y+(y===d.url?(bN.test(d.url)?"&":"?")+"_="+x:"")}}(d.data&&d.hasContent&&d.contentType!==!1||c.contentType)&&v.setRequestHeader("Content-Type",d.contentType),d.ifModified&&(k=k||d.url,f.lastModified[k]&&v.setRequestHeader("If-Modified-Since",f.lastModified[k]),f.etag[k]&&v.setRequestHeader("If-None-Match",f.etag[k])),v.setRequestHeader("Accept",d.dataTypes[0]&&d.accepts[d.dataTypes[0]]?d.accepts[d.dataTypes[0]]+(d.dataTypes[0]!=="*"?", */*; q=0.01":""):d.accepts["*"]);for(u in d.headers)v.setRequestHeader(u,d.headers[u]);if(d.beforeSend&&(d.beforeSend.call(e,v,d)===!1||s===2)){v.abort();return!1}for(u in{success:1,error:1,complete:1})v[u](d[u]);p=b$(bV,d,c,v);if(!p)w(-1,"No Transport");else{v.readyState=1,t&&g.trigger("ajaxSend",[v,d]),d.async&&d.timeout>0&&(q=setTimeout(function(){v.abort("timeout")},d.timeout));try{s=1,p.send(l,w)}catch(z){status<2?w(-1,z):f.error(z)}}return v},param:function(a,c){var d=[],e=function(a,b){b=f.isFunction(b)?b():b,d[d.length]=encodeURIComponent(a)+"="+encodeURIComponent(b)};c===b&&(c=f.ajaxSettings.traditional);if(f.isArray(a)||a.jquery&&!f.isPlainObject(a))f.each(a,function(){e(this.name,this.value)});else for(var g in a)b_(g,a[g],c,e);return d.join("&").replace(bE,"+")}}),f.extend({active:0,lastModified:{},etag:{}});var cc=f.now(),cd=/(\=)\?(&|$)|\?\?/i;f.ajaxSetup({jsonp:"callback",jsonpCallback:function(){return f.expando+"_"+cc++}}),f.ajaxPrefilter("json jsonp",function(b,c,d){var e=b.contentType==="application/x-www-form-urlencoded"&&typeof b.data=="string";if(b.dataTypes[0]==="jsonp"||b.jsonp!==!1&&(cd.test(b.url)||e&&cd.test(b.data))){var g,h=b.jsonpCallback=f.isFunction(b.jsonpCallback)?b.jsonpCallback():b.jsonpCallback,i=a[h],j=b.url,k=b.data,l="$1"+h+"$2";b.jsonp!==!1&&(j=j.replace(cd,l),b.url===j&&(e&&(k=k.replace(cd,l)),b.data===k&&(j+=(/\?/.test(j)?"&":"?")+b.jsonp+"="+h))),b.url=j,b.data=k,a[h]=function(a){g=[a]},d.always(function(){a[h]=i,g&&f.isFunction(i)&&a[h](g[0])}),b.converters["script json"]=function(){g||f.error(h+" was not called");return g[0]},b.dataTypes[0]="json";return"script"}}),f.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/javascript|ecmascript/},converters:{"text script":function(a){f.globalEval(a);return a}}}),f.ajaxPrefilter("script",function(a){a.cache===b&&(a.cache=!1),a.crossDomain&&(a.type="GET",a.global=!1)}),f.ajaxTransport("script",function(a){if(a.crossDomain){var d,e=c.head||c.getElementsByTagName("head")[0]||c.documentElement;return{send:function(f,g){d=c.createElement("script"),d.async="async",a.scriptCharset&&(d.charset=a.scriptCharset),d.src=a.url,d.onload=d.onreadystatechange=function(a,c){if(c||!d.readyState||/loaded|complete/.test(d.readyState))d.onload=d.onreadystatechange=null,e&&d.parentNode&&e.removeChild(d),d=b,c||g(200,"success")},e.insertBefore(d,e.firstChild)},abort:function(){d&&d.onload(0,1)}}}});var ce=a.ActiveXObject?function(){for(var a in cg)cg[a](0,1)}:!1,cf=0,cg;f.ajaxSettings.xhr=a.ActiveXObject?function(){return!this.isLocal&&ch()||ci()}:ch,function(a){f.extend(f.support,{ajax:!!a,cors:!!a&&"withCredentials"in a})}(f.ajaxSettings.xhr()),f.support.ajax&&f.ajaxTransport(function(c){if(!c.crossDomain||f.support.cors){var d;return{send:function(e,g){var h=c.xhr(),i,j;c.username?h.open(c.type,c.url,c.async,c.username,c.password):h.open(c.type,c.url,c.async);if(c.xhrFields)for(j in c.xhrFields)h[j]=c.xhrFields[j];c.mimeType&&h.overrideMimeType&&h.overrideMimeType(c.mimeType),!c.crossDomain&&!e["X-Requested-With"]&&(e["X-Requested-With"]="XMLHttpRequest");try{for(j in e)h.setRequestHeader(j,e[j])}catch(k){}h.send(c.hasContent&&c.data||null),d=function(a,e){var j,k,l,m,n;try{if(d&&(e||h.readyState===4)){d=b,i&&(h.onreadystatechange=f.noop,ce&&delete cg[i]);if(e)h.readyState!==4&&h.abort();else{j=h.status,l=h.getAllResponseHeaders(),m={},n=h.responseXML,n&&n.documentElement&&(m.xml=n),m.text=h.responseText;try{k=h.statusText}catch(o){k=""}!j&&c.isLocal&&!c.crossDomain?j=m.text?200:404:j===1223&&(j=204)}}}catch(p){e||g(-1,p)}m&&g(j,k,m,l)},!c.async||h.readyState===4?d():(i=++cf,ce&&(cg||(cg={},f(a).unload(ce)),cg[i]=d),h.onreadystatechange=d)},abort:function(){d&&d(0,1)}}}});var cj={},ck,cl,cm=/^(?:toggle|show|hide)$/,cn=/^([+\-]=)?([\d+.\-]+)([a-z%]*)$/i,co,cp=[["height","marginTop","marginBottom","paddingTop","paddingBottom"],["width","marginLeft","marginRight","paddingLeft","paddingRight"],["opacity"]],cq,cr=a.webkitRequestAnimationFrame||a.mozRequestAnimationFrame||a.oRequestAnimationFrame;f.fn.extend({show:function(a,b,c){var d,e;if(a||a===0)return this.animate(cu("show",3),a,b,c);for(var g=0,h=this.length;g=e.duration+this.startTime){this.now=this.end,this.pos=this.state=1,this.update(),e.animatedProperties[this.prop]=!0;for(g in e.animatedProperties)e.animatedProperties[g]!==!0&&(c=!1);if(c){e.overflow!=null&&!f.support.shrinkWrapBlocks&&f.each(["","X","Y"],function(a,b){d.style["overflow"+b]=e.overflow[a]}),e.hide&&f(d).hide();if(e.hide||e.show)for(var i in e.animatedProperties)f.style(d,i,e.orig[i]);e.complete.call(d)}return!1}e.duration==Infinity?this.now=b:(h=b-this.startTime,this.state=h/e.duration,this.pos=f.easing[e.animatedProperties[this.prop]](this.state,h,0,1,e.duration),this.now=this.start+(this.end-this.start)*this.pos),this.update();return!0}},f.extend(f.fx,{tick:function(){for(var a=f.timers,b=0;b
    ";f.extend(b.style,{position:"absolute",top:0,left:0,margin:0,border:0,width:"1px",height:"1px",visibility:"hidden"}),b.innerHTML=j,a.insertBefore(b,a.firstChild),d=b.firstChild,e=d.firstChild,h=d.nextSibling.firstChild.firstChild,this.doesNotAddBorder=e.offsetTop!==5,this.doesAddBorderForTableAndCells=h.offsetTop===5,e.style.position="fixed",e.style.top="20px",this.supportsFixedPosition=e.offsetTop===20||e.offsetTop===15,e.style.position=e.style.top="",d.style.overflow="hidden",d.style.position="relative",this.subtractsBorderForOverflowNotVisible=e.offsetTop===-5,this.doesNotIncludeMarginInBodyOffset=a.offsetTop!==i,a.removeChild(b),f.offset.initialize=f.noop},bodyOffset:function(a){var b=a.offsetTop,c=a.offsetLeft;f.offset.initialize(),f.offset.doesNotIncludeMarginInBodyOffset&&(b+=parseFloat(f.css(a,"marginTop"))||0,c+=parseFloat(f.css(a,"marginLeft"))||0);return{top:b,left:c}},setOffset:function(a,b,c){var d=f.css(a,"position");d==="static"&&(a.style.position="relative");var e=f(a),g=e.offset(),h=f.css(a,"top"),i=f.css(a,"left"),j=(d==="absolute"||d==="fixed")&&f.inArray("auto",[h,i])>-1,k={},l={},m,n;j?(l=e.position(),m=l.top,n=l.left):(m=parseFloat(h)||0,n=parseFloat(i)||0),f.isFunction(b)&&(b=b.call(a,c,g)),b.top!=null&&(k.top=b.top-g.top+m),b.left!=null&&(k.left=b.left-g.left+n),"using"in b?b.using.call(a,k):e.css(k)}},f.fn.extend({position:function(){if(!this[0])return null;var a=this[0],b=this.offsetParent(),c=this.offset(),d=cx.test(b[0].nodeName)?{top:0,left:0}:b.offset();c.top-=parseFloat(f.css(a,"marginTop"))||0,c.left-=parseFloat(f.css(a,"marginLeft"))||0,d.top+=parseFloat(f.css(b[0],"borderTopWidth"))||0,d.left+=parseFloat(f.css(b[0],"borderLeftWidth"))||0;return{top:c.top-d.top,left:c.left-d.left}},offsetParent:function(){return this.map(function(){var a=this.offsetParent||c.body;while(a&&!cx.test(a.nodeName)&&f.css(a,"position")==="static")a=a.offsetParent;return a})}}),f.each(["Left","Top"],function(a,c){var d="scroll"+c;f.fn[d]=function(c){var e,g;if(c===b){e=this[0];if(!e)return null;g=cy(e);return g?"pageXOffset"in g?g[a?"pageYOffset":"pageXOffset"]:f.support.boxModel&&g.document.documentElement[d]||g.document.body[d]:e[d]}return this.each(function(){g=cy(this),g?g.scrollTo(a?f(g).scrollLeft():c,a?c:f(g).scrollTop()):this[d]=c})}}),f.each(["Height","Width"],function(a,c){var d=c.toLowerCase();f.fn["inner"+c]=function(){return this[0]?parseFloat(f.css(this[0],d,"padding")):null},f.fn["outer"+c]=function(a){return this[0]?parseFloat(f.css(this[0],d,a?"margin":"border")):null},f.fn[d]=function(a){var e=this[0];if(!e)return a==null?null:this;if(f.isFunction(a))return this.each(function(b){var c=f(this);c[d](a.call(this,b,c[d]()))});if(f.isWindow(e)){var g=e.document.documentElement["client"+c];return e.document.compatMode==="CSS1Compat"&&g||e.document.body["client"+c]||g}if(e.nodeType===9)return Math.max(e.documentElement["client"+c],e.body["scroll"+c],e.documentElement["scroll"+c],e.body["offset"+c],e.documentElement["offset"+c]);if(a===b){var h=f.css(e,d),i=parseFloat(h);return f.isNaN(i)?h:i}return this.css(d,typeof a=="string"?a:a+"px")}}),a.jQuery=a.$=f})(window); \ No newline at end of file diff --git a/storefront/static/common/js/jquery-ui-1.8.11.min.js b/storefront/static/common/js/jquery-ui-1.8.11.min.js new file mode 100644 index 0000000..e3badaa --- /dev/null +++ b/storefront/static/common/js/jquery-ui-1.8.11.min.js @@ -0,0 +1,406 @@ +/*! + * jQuery UI 1.8.11 + * + * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI + */ +(function(b,d){function e(g){return!b(g).parents().andSelf().filter(function(){return b.curCSS(this,"visibility")==="hidden"||b.expr.filters.hidden(this)}).length}b.ui=b.ui||{};if(!b.ui.version){b.extend(b.ui,{version:"1.8.11",keyCode:{ALT:18,BACKSPACE:8,CAPS_LOCK:20,COMMA:188,COMMAND:91,COMMAND_LEFT:91,COMMAND_RIGHT:93,CONTROL:17,DELETE:46,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,INSERT:45,LEFT:37,MENU:93,NUMPAD_ADD:107,NUMPAD_DECIMAL:110,NUMPAD_DIVIDE:111,NUMPAD_ENTER:108,NUMPAD_MULTIPLY:106, +NUMPAD_SUBTRACT:109,PAGE_DOWN:34,PAGE_UP:33,PERIOD:190,RIGHT:39,SHIFT:16,SPACE:32,TAB:9,UP:38,WINDOWS:91}});b.fn.extend({_focus:b.fn.focus,focus:function(g,f){return typeof g==="number"?this.each(function(){var a=this;setTimeout(function(){b(a).focus();f&&f.call(a)},g)}):this._focus.apply(this,arguments)},scrollParent:function(){var g;g=b.browser.msie&&/(static|relative)/.test(this.css("position"))||/absolute/.test(this.css("position"))?this.parents().filter(function(){return/(relative|absolute|fixed)/.test(b.curCSS(this, +"position",1))&&/(auto|scroll)/.test(b.curCSS(this,"overflow",1)+b.curCSS(this,"overflow-y",1)+b.curCSS(this,"overflow-x",1))}).eq(0):this.parents().filter(function(){return/(auto|scroll)/.test(b.curCSS(this,"overflow",1)+b.curCSS(this,"overflow-y",1)+b.curCSS(this,"overflow-x",1))}).eq(0);return/fixed/.test(this.css("position"))||!g.length?b(document):g},zIndex:function(g){if(g!==d)return this.css("zIndex",g);if(this.length){g=b(this[0]);for(var f;g.length&&g[0]!==document;){f=g.css("position"); +if(f==="absolute"||f==="relative"||f==="fixed"){f=parseInt(g.css("zIndex"),10);if(!isNaN(f)&&f!==0)return f}g=g.parent()}}return 0},disableSelection:function(){return this.bind((b.support.selectstart?"selectstart":"mousedown")+".ui-disableSelection",function(g){g.preventDefault()})},enableSelection:function(){return this.unbind(".ui-disableSelection")}});b.each(["Width","Height"],function(g,f){function a(j,n,p,l){b.each(c,function(){n-=parseFloat(b.curCSS(j,"padding"+this,true))||0;if(p)n-=parseFloat(b.curCSS(j, +"border"+this+"Width",true))||0;if(l)n-=parseFloat(b.curCSS(j,"margin"+this,true))||0});return n}var c=f==="Width"?["Left","Right"]:["Top","Bottom"],h=f.toLowerCase(),i={innerWidth:b.fn.innerWidth,innerHeight:b.fn.innerHeight,outerWidth:b.fn.outerWidth,outerHeight:b.fn.outerHeight};b.fn["inner"+f]=function(j){if(j===d)return i["inner"+f].call(this);return this.each(function(){b(this).css(h,a(this,j)+"px")})};b.fn["outer"+f]=function(j,n){if(typeof j!=="number")return i["outer"+f].call(this,j);return this.each(function(){b(this).css(h, +a(this,j,true,n)+"px")})}});b.extend(b.expr[":"],{data:function(g,f,a){return!!b.data(g,a[3])},focusable:function(g){var f=g.nodeName.toLowerCase(),a=b.attr(g,"tabindex");if("area"===f){f=g.parentNode;a=f.name;if(!g.href||!a||f.nodeName.toLowerCase()!=="map")return false;g=b("img[usemap=#"+a+"]")[0];return!!g&&e(g)}return(/input|select|textarea|button|object/.test(f)?!g.disabled:"a"==f?g.href||!isNaN(a):!isNaN(a))&&e(g)},tabbable:function(g){var f=b.attr(g,"tabindex");return(isNaN(f)||f>=0)&&b(g).is(":focusable")}}); +b(function(){var g=document.body,f=g.appendChild(f=document.createElement("div"));b.extend(f.style,{minHeight:"100px",height:"auto",padding:0,borderWidth:0});b.support.minHeight=f.offsetHeight===100;b.support.selectstart="onselectstart"in f;g.removeChild(f).style.display="none"});b.extend(b.ui,{plugin:{add:function(g,f,a){g=b.ui[g].prototype;for(var c in a){g.plugins[c]=g.plugins[c]||[];g.plugins[c].push([f,a[c]])}},call:function(g,f,a){if((f=g.plugins[f])&&g.element[0].parentNode)for(var c=0;c0)return true;g[f]=1;a=g[f]>0;g[f]=0;return a},isOverAxis:function(g,f,a){return g>f&&g=9)&&!d.button)return this._mouseUp(d);if(this._mouseStarted){this._mouseDrag(d);return d.preventDefault()}if(this._mouseDistanceMet(d)&&this._mouseDelayMet(d))(this._mouseStarted=this._mouseStart(this._mouseDownEvent,d)!==false)?this._mouseDrag(d):this._mouseUp(d);return!this._mouseStarted},_mouseUp:function(d){b(document).unbind("mousemove."+this.widgetName,this._mouseMoveDelegate).unbind("mouseup."+this.widgetName,this._mouseUpDelegate); +if(this._mouseStarted){this._mouseStarted=false;d.target==this._mouseDownEvent.target&&b.data(d.target,this.widgetName+".preventClickEvent",true);this._mouseStop(d)}return false},_mouseDistanceMet:function(d){return Math.max(Math.abs(this._mouseDownEvent.pageX-d.pageX),Math.abs(this._mouseDownEvent.pageY-d.pageY))>=this.options.distance},_mouseDelayMet:function(){return this.mouseDelayMet},_mouseStart:function(){},_mouseDrag:function(){},_mouseStop:function(){},_mouseCapture:function(){return true}})})(jQuery); +(function(b){b.widget("ui.draggable",b.ui.mouse,{widgetEventPrefix:"drag",options:{addClasses:true,appendTo:"parent",axis:false,connectToSortable:false,containment:false,cursor:"auto",cursorAt:false,grid:false,handle:false,helper:"original",iframeFix:false,opacity:false,refreshPositions:false,revert:false,revertDuration:500,scope:"default",scroll:true,scrollSensitivity:20,scrollSpeed:20,snap:false,snapMode:"both",snapTolerance:20,stack:false,zIndex:false},_create:function(){if(this.options.helper== +"original"&&!/^(?:r|a|f)/.test(this.element.css("position")))this.element[0].style.position="relative";this.options.addClasses&&this.element.addClass("ui-draggable");this.options.disabled&&this.element.addClass("ui-draggable-disabled");this._mouseInit()},destroy:function(){if(this.element.data("draggable")){this.element.removeData("draggable").unbind(".draggable").removeClass("ui-draggable ui-draggable-dragging ui-draggable-disabled");this._mouseDestroy();return this}},_mouseCapture:function(d){var e= +this.options;if(this.helper||e.disabled||b(d.target).is(".ui-resizable-handle"))return false;this.handle=this._getHandle(d);if(!this.handle)return false;return true},_mouseStart:function(d){var e=this.options;this.helper=this._createHelper(d);this._cacheHelperProportions();if(b.ui.ddmanager)b.ui.ddmanager.current=this;this._cacheMargins();this.cssPosition=this.helper.css("position");this.scrollParent=this.helper.scrollParent();this.offset=this.positionAbs=this.element.offset();this.offset={top:this.offset.top- +this.margins.top,left:this.offset.left-this.margins.left};b.extend(this.offset,{click:{left:d.pageX-this.offset.left,top:d.pageY-this.offset.top},parent:this._getParentOffset(),relative:this._getRelativeOffset()});this.originalPosition=this.position=this._generatePosition(d);this.originalPageX=d.pageX;this.originalPageY=d.pageY;e.cursorAt&&this._adjustOffsetFromHelper(e.cursorAt);e.containment&&this._setContainment();if(this._trigger("start",d)===false){this._clear();return false}this._cacheHelperProportions(); +b.ui.ddmanager&&!e.dropBehaviour&&b.ui.ddmanager.prepareOffsets(this,d);this.helper.addClass("ui-draggable-dragging");this._mouseDrag(d,true);return true},_mouseDrag:function(d,e){this.position=this._generatePosition(d);this.positionAbs=this._convertPositionTo("absolute");if(!e){e=this._uiHash();if(this._trigger("drag",d,e)===false){this._mouseUp({});return false}this.position=e.position}if(!this.options.axis||this.options.axis!="y")this.helper[0].style.left=this.position.left+"px";if(!this.options.axis|| +this.options.axis!="x")this.helper[0].style.top=this.position.top+"px";b.ui.ddmanager&&b.ui.ddmanager.drag(this,d);return false},_mouseStop:function(d){var e=false;if(b.ui.ddmanager&&!this.options.dropBehaviour)e=b.ui.ddmanager.drop(this,d);if(this.dropped){e=this.dropped;this.dropped=false}if((!this.element[0]||!this.element[0].parentNode)&&this.options.helper=="original")return false;if(this.options.revert=="invalid"&&!e||this.options.revert=="valid"&&e||this.options.revert===true||b.isFunction(this.options.revert)&& +this.options.revert.call(this.element,e)){var g=this;b(this.helper).animate(this.originalPosition,parseInt(this.options.revertDuration,10),function(){g._trigger("stop",d)!==false&&g._clear()})}else this._trigger("stop",d)!==false&&this._clear();return false},cancel:function(){this.helper.is(".ui-draggable-dragging")?this._mouseUp({}):this._clear();return this},_getHandle:function(d){var e=!this.options.handle||!b(this.options.handle,this.element).length?true:false;b(this.options.handle,this.element).find("*").andSelf().each(function(){if(this== +d.target)e=true});return e},_createHelper:function(d){var e=this.options;d=b.isFunction(e.helper)?b(e.helper.apply(this.element[0],[d])):e.helper=="clone"?this.element.clone():this.element;d.parents("body").length||d.appendTo(e.appendTo=="parent"?this.element[0].parentNode:e.appendTo);d[0]!=this.element[0]&&!/(fixed|absolute)/.test(d.css("position"))&&d.css("position","absolute");return d},_adjustOffsetFromHelper:function(d){if(typeof d=="string")d=d.split(" ");if(b.isArray(d))d={left:+d[0],top:+d[1]|| +0};if("left"in d)this.offset.click.left=d.left+this.margins.left;if("right"in d)this.offset.click.left=this.helperProportions.width-d.right+this.margins.left;if("top"in d)this.offset.click.top=d.top+this.margins.top;if("bottom"in d)this.offset.click.top=this.helperProportions.height-d.bottom+this.margins.top},_getParentOffset:function(){this.offsetParent=this.helper.offsetParent();var d=this.offsetParent.offset();if(this.cssPosition=="absolute"&&this.scrollParent[0]!=document&&b.ui.contains(this.scrollParent[0], +this.offsetParent[0])){d.left+=this.scrollParent.scrollLeft();d.top+=this.scrollParent.scrollTop()}if(this.offsetParent[0]==document.body||this.offsetParent[0].tagName&&this.offsetParent[0].tagName.toLowerCase()=="html"&&b.browser.msie)d={top:0,left:0};return{top:d.top+(parseInt(this.offsetParent.css("borderTopWidth"),10)||0),left:d.left+(parseInt(this.offsetParent.css("borderLeftWidth"),10)||0)}},_getRelativeOffset:function(){if(this.cssPosition=="relative"){var d=this.element.position();return{top:d.top- +(parseInt(this.helper.css("top"),10)||0)+this.scrollParent.scrollTop(),left:d.left-(parseInt(this.helper.css("left"),10)||0)+this.scrollParent.scrollLeft()}}else return{top:0,left:0}},_cacheMargins:function(){this.margins={left:parseInt(this.element.css("marginLeft"),10)||0,top:parseInt(this.element.css("marginTop"),10)||0,right:parseInt(this.element.css("marginRight"),10)||0,bottom:parseInt(this.element.css("marginBottom"),10)||0}},_cacheHelperProportions:function(){this.helperProportions={width:this.helper.outerWidth(), +height:this.helper.outerHeight()}},_setContainment:function(){var d=this.options;if(d.containment=="parent")d.containment=this.helper[0].parentNode;if(d.containment=="document"||d.containment=="window")this.containment=[(d.containment=="document"?0:b(window).scrollLeft())-this.offset.relative.left-this.offset.parent.left,(d.containment=="document"?0:b(window).scrollTop())-this.offset.relative.top-this.offset.parent.top,(d.containment=="document"?0:b(window).scrollLeft())+b(d.containment=="document"? +document:window).width()-this.helperProportions.width-this.margins.left,(d.containment=="document"?0:b(window).scrollTop())+(b(d.containment=="document"?document:window).height()||document.body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top];if(!/^(document|window|parent)$/.test(d.containment)&&d.containment.constructor!=Array){var e=b(d.containment)[0];if(e){d=b(d.containment).offset();var g=b(e).css("overflow")!="hidden";this.containment=[d.left+(parseInt(b(e).css("borderLeftWidth"), +10)||0)+(parseInt(b(e).css("paddingLeft"),10)||0),d.top+(parseInt(b(e).css("borderTopWidth"),10)||0)+(parseInt(b(e).css("paddingTop"),10)||0),d.left+(g?Math.max(e.scrollWidth,e.offsetWidth):e.offsetWidth)-(parseInt(b(e).css("borderLeftWidth"),10)||0)-(parseInt(b(e).css("paddingRight"),10)||0)-this.helperProportions.width-this.margins.left-this.margins.right,d.top+(g?Math.max(e.scrollHeight,e.offsetHeight):e.offsetHeight)-(parseInt(b(e).css("borderTopWidth"),10)||0)-(parseInt(b(e).css("paddingBottom"), +10)||0)-this.helperProportions.height-this.margins.top-this.margins.bottom]}}else if(d.containment.constructor==Array)this.containment=d.containment},_convertPositionTo:function(d,e){if(!e)e=this.position;d=d=="absolute"?1:-1;var g=this.cssPosition=="absolute"&&!(this.scrollParent[0]!=document&&b.ui.contains(this.scrollParent[0],this.offsetParent[0]))?this.offsetParent:this.scrollParent,f=/(html|body)/i.test(g[0].tagName);return{top:e.top+this.offset.relative.top*d+this.offset.parent.top*d-(b.browser.safari&& +b.browser.version<526&&this.cssPosition=="fixed"?0:(this.cssPosition=="fixed"?-this.scrollParent.scrollTop():f?0:g.scrollTop())*d),left:e.left+this.offset.relative.left*d+this.offset.parent.left*d-(b.browser.safari&&b.browser.version<526&&this.cssPosition=="fixed"?0:(this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():f?0:g.scrollLeft())*d)}},_generatePosition:function(d){var e=this.options,g=this.cssPosition=="absolute"&&!(this.scrollParent[0]!=document&&b.ui.contains(this.scrollParent[0], +this.offsetParent[0]))?this.offsetParent:this.scrollParent,f=/(html|body)/i.test(g[0].tagName),a=d.pageX,c=d.pageY;if(this.originalPosition){if(this.containment){if(d.pageX-this.offset.click.leftthis.containment[2])a=this.containment[2]+this.offset.click.left;if(d.pageY-this.offset.click.top>this.containment[3])c= +this.containment[3]+this.offset.click.top}if(e.grid){c=this.originalPageY+Math.round((c-this.originalPageY)/e.grid[1])*e.grid[1];c=this.containment?!(c-this.offset.click.topthis.containment[3])?c:!(c-this.offset.click.topthis.containment[2])? +a:!(a-this.offset.click.left').css({width:this.offsetWidth+ +"px",height:this.offsetHeight+"px",position:"absolute",opacity:"0.001",zIndex:1E3}).css(b(this).offset()).appendTo("body")})},stop:function(){b("div.ui-draggable-iframeFix").each(function(){this.parentNode.removeChild(this)})}});b.ui.plugin.add("draggable","opacity",{start:function(d,e){d=b(e.helper);e=b(this).data("draggable").options;if(d.css("opacity"))e._opacity=d.css("opacity");d.css("opacity",e.opacity)},stop:function(d,e){d=b(this).data("draggable").options;d._opacity&&b(e.helper).css("opacity", +d._opacity)}});b.ui.plugin.add("draggable","scroll",{start:function(){var d=b(this).data("draggable");if(d.scrollParent[0]!=document&&d.scrollParent[0].tagName!="HTML")d.overflowOffset=d.scrollParent.offset()},drag:function(d){var e=b(this).data("draggable"),g=e.options,f=false;if(e.scrollParent[0]!=document&&e.scrollParent[0].tagName!="HTML"){if(!g.axis||g.axis!="x")if(e.overflowOffset.top+e.scrollParent[0].offsetHeight-d.pageY=0;n--){var p=g.snapElements[n].left,l=p+g.snapElements[n].width,k=g.snapElements[n].top,m=k+g.snapElements[n].height;if(p-a=n&&c<=p||h>=n&&h<=p||cp)&&(f>= +i&&f<=j||a>=i&&a<=j||fj);default:return false}};b.ui.ddmanager={current:null,droppables:{"default":[]},prepareOffsets:function(d,e){var g=b.ui.ddmanager.droppables[d.options.scope]||[],f=e?e.type:null,a=(d.currentItem||d.element).find(":data(droppable)").andSelf(),c=0;a:for(;c').css({position:this.element.css("position"),width:this.element.outerWidth(),height:this.element.outerHeight(), +top:this.element.css("top"),left:this.element.css("left")}));this.element=this.element.parent().data("resizable",this.element.data("resizable"));this.elementIsWrapper=true;this.element.css({marginLeft:this.originalElement.css("marginLeft"),marginTop:this.originalElement.css("marginTop"),marginRight:this.originalElement.css("marginRight"),marginBottom:this.originalElement.css("marginBottom")});this.originalElement.css({marginLeft:0,marginTop:0,marginRight:0,marginBottom:0});this.originalResizeStyle= +this.originalElement.css("resize");this.originalElement.css("resize","none");this._proportionallyResizeElements.push(this.originalElement.css({position:"static",zoom:1,display:"block"}));this.originalElement.css({margin:this.originalElement.css("margin")});this._proportionallyResize()}this.handles=f.handles||(!b(".ui-resizable-handle",this.element).length?"e,s,se":{n:".ui-resizable-n",e:".ui-resizable-e",s:".ui-resizable-s",w:".ui-resizable-w",se:".ui-resizable-se",sw:".ui-resizable-sw",ne:".ui-resizable-ne", +nw:".ui-resizable-nw"});if(this.handles.constructor==String){if(this.handles=="all")this.handles="n,e,s,w,se,sw,ne,nw";var a=this.handles.split(",");this.handles={};for(var c=0;c');/sw|se|ne|nw/.test(h)&&i.css({zIndex:++f.zIndex});"se"==h&&i.addClass("ui-icon ui-icon-gripsmall-diagonal-se");this.handles[h]=".ui-resizable-"+h;this.element.append(i)}}this._renderAxis=function(j){j=j||this.element;for(var n in this.handles){if(this.handles[n].constructor== +String)this.handles[n]=b(this.handles[n],this.element).show();if(this.elementIsWrapper&&this.originalElement[0].nodeName.match(/textarea|input|select|button/i)){var p=b(this.handles[n],this.element),l=0;l=/sw|ne|nw|se|n|s/.test(n)?p.outerHeight():p.outerWidth();p=["padding",/ne|nw|n/.test(n)?"Top":/se|sw|s/.test(n)?"Bottom":/^e$/.test(n)?"Right":"Left"].join("");j.css(p,l);this._proportionallyResize()}b(this.handles[n])}};this._renderAxis(this.element);this._handles=b(".ui-resizable-handle",this.element).disableSelection(); +this._handles.mouseover(function(){if(!g.resizing){if(this.className)var j=this.className.match(/ui-resizable-(se|sw|ne|nw|n|e|s|w)/i);g.axis=j&&j[1]?j[1]:"se"}});if(f.autoHide){this._handles.hide();b(this.element).addClass("ui-resizable-autohide").hover(function(){b(this).removeClass("ui-resizable-autohide");g._handles.show()},function(){if(!g.resizing){b(this).addClass("ui-resizable-autohide");g._handles.hide()}})}this._mouseInit()},destroy:function(){this._mouseDestroy();var g=function(a){b(a).removeClass("ui-resizable ui-resizable-disabled ui-resizable-resizing").removeData("resizable").unbind(".resizable").find(".ui-resizable-handle").remove()}; +if(this.elementIsWrapper){g(this.element);var f=this.element;f.after(this.originalElement.css({position:f.css("position"),width:f.outerWidth(),height:f.outerHeight(),top:f.css("top"),left:f.css("left")})).remove()}this.originalElement.css("resize",this.originalResizeStyle);g(this.originalElement);return this},_mouseCapture:function(g){var f=false;for(var a in this.handles)if(b(this.handles[a])[0]==g.target)f=true;return!this.options.disabled&&f},_mouseStart:function(g){var f=this.options,a=this.element.position(), +c=this.element;this.resizing=true;this.documentScroll={top:b(document).scrollTop(),left:b(document).scrollLeft()};if(c.is(".ui-draggable")||/absolute/.test(c.css("position")))c.css({position:"absolute",top:a.top,left:a.left});b.browser.opera&&/relative/.test(c.css("position"))&&c.css({position:"relative",top:"auto",left:"auto"});this._renderProxy();a=d(this.helper.css("left"));var h=d(this.helper.css("top"));if(f.containment){a+=b(f.containment).scrollLeft()||0;h+=b(f.containment).scrollTop()||0}this.offset= +this.helper.offset();this.position={left:a,top:h};this.size=this._helper?{width:c.outerWidth(),height:c.outerHeight()}:{width:c.width(),height:c.height()};this.originalSize=this._helper?{width:c.outerWidth(),height:c.outerHeight()}:{width:c.width(),height:c.height()};this.originalPosition={left:a,top:h};this.sizeDiff={width:c.outerWidth()-c.width(),height:c.outerHeight()-c.height()};this.originalMousePosition={left:g.pageX,top:g.pageY};this.aspectRatio=typeof f.aspectRatio=="number"?f.aspectRatio: +this.originalSize.width/this.originalSize.height||1;f=b(".ui-resizable-"+this.axis).css("cursor");b("body").css("cursor",f=="auto"?this.axis+"-resize":f);c.addClass("ui-resizable-resizing");this._propagate("start",g);return true},_mouseDrag:function(g){var f=this.helper,a=this.originalMousePosition,c=this._change[this.axis];if(!c)return false;a=c.apply(this,[g,g.pageX-a.left||0,g.pageY-a.top||0]);if(this._aspectRatio||g.shiftKey)a=this._updateRatio(a,g);a=this._respectSize(a,g);this._propagate("resize", +g);f.css({top:this.position.top+"px",left:this.position.left+"px",width:this.size.width+"px",height:this.size.height+"px"});!this._helper&&this._proportionallyResizeElements.length&&this._proportionallyResize();this._updateCache(a);this._trigger("resize",g,this.ui());return false},_mouseStop:function(g){this.resizing=false;var f=this.options,a=this;if(this._helper){var c=this._proportionallyResizeElements,h=c.length&&/textarea/i.test(c[0].nodeName);c=h&&b.ui.hasScroll(c[0],"left")?0:a.sizeDiff.height; +h=h?0:a.sizeDiff.width;h={width:a.helper.width()-h,height:a.helper.height()-c};c=parseInt(a.element.css("left"),10)+(a.position.left-a.originalPosition.left)||null;var i=parseInt(a.element.css("top"),10)+(a.position.top-a.originalPosition.top)||null;f.animate||this.element.css(b.extend(h,{top:i,left:c}));a.helper.height(a.size.height);a.helper.width(a.size.width);this._helper&&!f.animate&&this._proportionallyResize()}b("body").css("cursor","auto");this.element.removeClass("ui-resizable-resizing"); +this._propagate("stop",g);this._helper&&this.helper.remove();return false},_updateCache:function(g){this.offset=this.helper.offset();if(e(g.left))this.position.left=g.left;if(e(g.top))this.position.top=g.top;if(e(g.height))this.size.height=g.height;if(e(g.width))this.size.width=g.width},_updateRatio:function(g){var f=this.position,a=this.size,c=this.axis;if(g.height)g.width=a.height*this.aspectRatio;else if(g.width)g.height=a.width/this.aspectRatio;if(c=="sw"){g.left=f.left+(a.width-g.width);g.top= +null}if(c=="nw"){g.top=f.top+(a.height-g.height);g.left=f.left+(a.width-g.width)}return g},_respectSize:function(g){var f=this.options,a=this.axis,c=e(g.width)&&f.maxWidth&&f.maxWidthg.width,j=e(g.height)&&f.minHeight&&f.minHeight>g.height;if(i)g.width=f.minWidth;if(j)g.height=f.minHeight;if(c)g.width=f.maxWidth;if(h)g.height=f.maxHeight;var n=this.originalPosition.left+this.originalSize.width,p=this.position.top+ +this.size.height,l=/sw|nw|w/.test(a);a=/nw|ne|n/.test(a);if(i&&l)g.left=n-f.minWidth;if(c&&l)g.left=n-f.maxWidth;if(j&&a)g.top=p-f.minHeight;if(h&&a)g.top=p-f.maxHeight;if((f=!g.width&&!g.height)&&!g.left&&g.top)g.top=null;else if(f&&!g.top&&g.left)g.left=null;return g},_proportionallyResize:function(){if(this._proportionallyResizeElements.length)for(var g=this.helper||this.element,f=0;f');var f=b.browser.msie&&b.browser.version<7,a=f?1:0;f=f?2:-1;this.helper.addClass(this._helper).css({width:this.element.outerWidth()+f,height:this.element.outerHeight()+f,position:"absolute",left:this.elementOffset.left-a+"px",top:this.elementOffset.top-a+"px",zIndex:++g.zIndex});this.helper.appendTo("body").disableSelection()}else this.helper=this.element},_change:{e:function(g, +f){return{width:this.originalSize.width+f}},w:function(g,f){return{left:this.originalPosition.left+f,width:this.originalSize.width-f}},n:function(g,f,a){return{top:this.originalPosition.top+a,height:this.originalSize.height-a}},s:function(g,f,a){return{height:this.originalSize.height+a}},se:function(g,f,a){return b.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[g,f,a]))},sw:function(g,f,a){return b.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[g,f, +a]))},ne:function(g,f,a){return b.extend(this._change.n.apply(this,arguments),this._change.e.apply(this,[g,f,a]))},nw:function(g,f,a){return b.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[g,f,a]))}},_propagate:function(g,f){b.ui.plugin.call(this,g,[f,this.ui()]);g!="resize"&&this._trigger(g,f,this.ui())},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize, +originalPosition:this.originalPosition}}});b.extend(b.ui.resizable,{version:"1.8.11"});b.ui.plugin.add("resizable","alsoResize",{start:function(){var g=b(this).data("resizable").options,f=function(a){b(a).each(function(){var c=b(this);c.data("resizable-alsoresize",{width:parseInt(c.width(),10),height:parseInt(c.height(),10),left:parseInt(c.css("left"),10),top:parseInt(c.css("top"),10),position:c.css("position")})})};if(typeof g.alsoResize=="object"&&!g.alsoResize.parentNode)if(g.alsoResize.length){g.alsoResize= +g.alsoResize[0];f(g.alsoResize)}else b.each(g.alsoResize,function(a){f(a)});else f(g.alsoResize)},resize:function(g,f){var a=b(this).data("resizable");g=a.options;var c=a.originalSize,h=a.originalPosition,i={height:a.size.height-c.height||0,width:a.size.width-c.width||0,top:a.position.top-h.top||0,left:a.position.left-h.left||0},j=function(n,p){b(n).each(function(){var l=b(this),k=b(this).data("resizable-alsoresize"),m={},o=p&&p.length?p:l.parents(f.originalElement[0]).length?["width","height"]:["width", +"height","top","left"];b.each(o,function(q,s){if((q=(k[s]||0)+(i[s]||0))&&q>=0)m[s]=q||null});if(b.browser.opera&&/relative/.test(l.css("position"))){a._revertToRelativePosition=true;l.css({position:"absolute",top:"auto",left:"auto"})}l.css(m)})};typeof g.alsoResize=="object"&&!g.alsoResize.nodeType?b.each(g.alsoResize,function(n,p){j(n,p)}):j(g.alsoResize)},stop:function(){var g=b(this).data("resizable"),f=g.options,a=function(c){b(c).each(function(){var h=b(this);h.css({position:h.data("resizable-alsoresize").position})})}; +if(g._revertToRelativePosition){g._revertToRelativePosition=false;typeof f.alsoResize=="object"&&!f.alsoResize.nodeType?b.each(f.alsoResize,function(c){a(c)}):a(f.alsoResize)}b(this).removeData("resizable-alsoresize")}});b.ui.plugin.add("resizable","animate",{stop:function(g){var f=b(this).data("resizable"),a=f.options,c=f._proportionallyResizeElements,h=c.length&&/textarea/i.test(c[0].nodeName),i=h&&b.ui.hasScroll(c[0],"left")?0:f.sizeDiff.height;h={width:f.size.width-(h?0:f.sizeDiff.width),height:f.size.height- +i};i=parseInt(f.element.css("left"),10)+(f.position.left-f.originalPosition.left)||null;var j=parseInt(f.element.css("top"),10)+(f.position.top-f.originalPosition.top)||null;f.element.animate(b.extend(h,j&&i?{top:j,left:i}:{}),{duration:a.animateDuration,easing:a.animateEasing,step:function(){var n={width:parseInt(f.element.css("width"),10),height:parseInt(f.element.css("height"),10),top:parseInt(f.element.css("top"),10),left:parseInt(f.element.css("left"),10)};c&&c.length&&b(c[0]).css({width:n.width, +height:n.height});f._updateCache(n);f._propagate("resize",g)}})}});b.ui.plugin.add("resizable","containment",{start:function(){var g=b(this).data("resizable"),f=g.element,a=g.options.containment;if(f=a instanceof b?a.get(0):/parent/.test(a)?f.parent().get(0):a){g.containerElement=b(f);if(/document/.test(a)||a==document){g.containerOffset={left:0,top:0};g.containerPosition={left:0,top:0};g.parentData={element:b(document),left:0,top:0,width:b(document).width(),height:b(document).height()||document.body.parentNode.scrollHeight}}else{var c= +b(f),h=[];b(["Top","Right","Left","Bottom"]).each(function(n,p){h[n]=d(c.css("padding"+p))});g.containerOffset=c.offset();g.containerPosition=c.position();g.containerSize={height:c.innerHeight()-h[3],width:c.innerWidth()-h[1]};a=g.containerOffset;var i=g.containerSize.height,j=g.containerSize.width;j=b.ui.hasScroll(f,"left")?f.scrollWidth:j;i=b.ui.hasScroll(f)?f.scrollHeight:i;g.parentData={element:f,left:a.left,top:a.top,width:j,height:i}}}},resize:function(g){var f=b(this).data("resizable"),a=f.options, +c=f.containerOffset,h=f.position;g=f._aspectRatio||g.shiftKey;var i={top:0,left:0},j=f.containerElement;if(j[0]!=document&&/static/.test(j.css("position")))i=c;if(h.left<(f._helper?c.left:0)){f.size.width+=f._helper?f.position.left-c.left:f.position.left-i.left;if(g)f.size.height=f.size.width/a.aspectRatio;f.position.left=a.helper?c.left:0}if(h.top<(f._helper?c.top:0)){f.size.height+=f._helper?f.position.top-c.top:f.position.top;if(g)f.size.width=f.size.height*a.aspectRatio;f.position.top=f._helper? +c.top:0}f.offset.left=f.parentData.left+f.position.left;f.offset.top=f.parentData.top+f.position.top;a=Math.abs((f._helper?f.offset.left-i.left:f.offset.left-i.left)+f.sizeDiff.width);c=Math.abs((f._helper?f.offset.top-i.top:f.offset.top-c.top)+f.sizeDiff.height);h=f.containerElement.get(0)==f.element.parent().get(0);i=/relative|absolute/.test(f.containerElement.css("position"));if(h&&i)a-=f.parentData.left;if(a+f.size.width>=f.parentData.width){f.size.width=f.parentData.width-a;if(g)f.size.height= +f.size.width/f.aspectRatio}if(c+f.size.height>=f.parentData.height){f.size.height=f.parentData.height-c;if(g)f.size.width=f.size.height*f.aspectRatio}},stop:function(){var g=b(this).data("resizable"),f=g.options,a=g.containerOffset,c=g.containerPosition,h=g.containerElement,i=b(g.helper),j=i.offset(),n=i.outerWidth()-g.sizeDiff.width;i=i.outerHeight()-g.sizeDiff.height;g._helper&&!f.animate&&/relative/.test(h.css("position"))&&b(this).css({left:j.left-c.left-a.left,width:n,height:i});g._helper&&!f.animate&& +/static/.test(h.css("position"))&&b(this).css({left:j.left-c.left-a.left,width:n,height:i})}});b.ui.plugin.add("resizable","ghost",{start:function(){var g=b(this).data("resizable"),f=g.options,a=g.size;g.ghost=g.originalElement.clone();g.ghost.css({opacity:0.25,display:"block",position:"relative",height:a.height,width:a.width,margin:0,left:0,top:0}).addClass("ui-resizable-ghost").addClass(typeof f.ghost=="string"?f.ghost:"");g.ghost.appendTo(g.helper)},resize:function(){var g=b(this).data("resizable"); +g.ghost&&g.ghost.css({position:"relative",height:g.size.height,width:g.size.width})},stop:function(){var g=b(this).data("resizable");g.ghost&&g.helper&&g.helper.get(0).removeChild(g.ghost.get(0))}});b.ui.plugin.add("resizable","grid",{resize:function(){var g=b(this).data("resizable"),f=g.options,a=g.size,c=g.originalSize,h=g.originalPosition,i=g.axis;f.grid=typeof f.grid=="number"?[f.grid,f.grid]:f.grid;var j=Math.round((a.width-c.width)/(f.grid[0]||1))*(f.grid[0]||1);f=Math.round((a.height-c.height)/ +(f.grid[1]||1))*(f.grid[1]||1);if(/^(se|s|e)$/.test(i)){g.size.width=c.width+j;g.size.height=c.height+f}else if(/^(ne)$/.test(i)){g.size.width=c.width+j;g.size.height=c.height+f;g.position.top=h.top-f}else{if(/^(sw)$/.test(i)){g.size.width=c.width+j;g.size.height=c.height+f}else{g.size.width=c.width+j;g.size.height=c.height+f;g.position.top=h.top-f}g.position.left=h.left-j}}});var d=function(g){return parseInt(g,10)||0},e=function(g){return!isNaN(parseInt(g,10))}})(jQuery); +(function(b){b.widget("ui.selectable",b.ui.mouse,{options:{appendTo:"body",autoRefresh:true,distance:0,filter:"*",tolerance:"touch"},_create:function(){var d=this;this.element.addClass("ui-selectable");this.dragged=false;var e;this.refresh=function(){e=b(d.options.filter,d.element[0]);e.each(function(){var g=b(this),f=g.offset();b.data(this,"selectable-item",{element:this,$element:g,left:f.left,top:f.top,right:f.left+g.outerWidth(),bottom:f.top+g.outerHeight(),startselected:false,selected:g.hasClass("ui-selected"), +selecting:g.hasClass("ui-selecting"),unselecting:g.hasClass("ui-unselecting")})})};this.refresh();this.selectees=e.addClass("ui-selectee");this._mouseInit();this.helper=b("
    ")},destroy:function(){this.selectees.removeClass("ui-selectee").removeData("selectable-item");this.element.removeClass("ui-selectable ui-selectable-disabled").removeData("selectable").unbind(".selectable");this._mouseDestroy();return this},_mouseStart:function(d){var e=this;this.opos=[d.pageX, +d.pageY];if(!this.options.disabled){var g=this.options;this.selectees=b(g.filter,this.element[0]);this._trigger("start",d);b(g.appendTo).append(this.helper);this.helper.css({left:d.clientX,top:d.clientY,width:0,height:0});g.autoRefresh&&this.refresh();this.selectees.filter(".ui-selected").each(function(){var f=b.data(this,"selectable-item");f.startselected=true;if(!d.metaKey){f.$element.removeClass("ui-selected");f.selected=false;f.$element.addClass("ui-unselecting");f.unselecting=true;e._trigger("unselecting", +d,{unselecting:f.element})}});b(d.target).parents().andSelf().each(function(){var f=b.data(this,"selectable-item");if(f){var a=!d.metaKey||!f.$element.hasClass("ui-selected");f.$element.removeClass(a?"ui-unselecting":"ui-selected").addClass(a?"ui-selecting":"ui-unselecting");f.unselecting=!a;f.selecting=a;(f.selected=a)?e._trigger("selecting",d,{selecting:f.element}):e._trigger("unselecting",d,{unselecting:f.element});return false}})}},_mouseDrag:function(d){var e=this;this.dragged=true;if(!this.options.disabled){var g= +this.options,f=this.opos[0],a=this.opos[1],c=d.pageX,h=d.pageY;if(f>c){var i=c;c=f;f=i}if(a>h){i=h;h=a;a=i}this.helper.css({left:f,top:a,width:c-f,height:h-a});this.selectees.each(function(){var j=b.data(this,"selectable-item");if(!(!j||j.element==e.element[0])){var n=false;if(g.tolerance=="touch")n=!(j.left>c||j.righth||j.bottomf&&j.righta&&j.bottom *",opacity:false,placeholder:false,revert:false,scroll:true,scrollSensitivity:20,scrollSpeed:20,scope:"default",tolerance:"intersect",zIndex:1E3},_create:function(){this.containerCache={};this.element.addClass("ui-sortable"); +this.refresh();this.floating=this.items.length?/left|right/.test(this.items[0].item.css("float"))||/inline|table-cell/.test(this.items[0].item.css("display")):false;this.offset=this.element.offset();this._mouseInit()},destroy:function(){this.element.removeClass("ui-sortable ui-sortable-disabled").removeData("sortable").unbind(".sortable");this._mouseDestroy();for(var d=this.items.length-1;d>=0;d--)this.items[d].item.removeData("sortable-item");return this},_setOption:function(d,e){if(d==="disabled"){this.options[d]= +e;this.widget()[e?"addClass":"removeClass"]("ui-sortable-disabled")}else b.Widget.prototype._setOption.apply(this,arguments)},_mouseCapture:function(d,e){if(this.reverting)return false;if(this.options.disabled||this.options.type=="static")return false;this._refreshItems(d);var g=null,f=this;b(d.target).parents().each(function(){if(b.data(this,"sortable-item")==f){g=b(this);return false}});if(b.data(d.target,"sortable-item")==f)g=b(d.target);if(!g)return false;if(this.options.handle&&!e){var a=false; +b(this.options.handle,g).find("*").andSelf().each(function(){if(this==d.target)a=true});if(!a)return false}this.currentItem=g;this._removeCurrentsFromItems();return true},_mouseStart:function(d,e,g){e=this.options;var f=this;this.currentContainer=this;this.refreshPositions();this.helper=this._createHelper(d);this._cacheHelperProportions();this._cacheMargins();this.scrollParent=this.helper.scrollParent();this.offset=this.currentItem.offset();this.offset={top:this.offset.top-this.margins.top,left:this.offset.left- +this.margins.left};this.helper.css("position","absolute");this.cssPosition=this.helper.css("position");b.extend(this.offset,{click:{left:d.pageX-this.offset.left,top:d.pageY-this.offset.top},parent:this._getParentOffset(),relative:this._getRelativeOffset()});this.originalPosition=this._generatePosition(d);this.originalPageX=d.pageX;this.originalPageY=d.pageY;e.cursorAt&&this._adjustOffsetFromHelper(e.cursorAt);this.domPosition={prev:this.currentItem.prev()[0],parent:this.currentItem.parent()[0]}; +this.helper[0]!=this.currentItem[0]&&this.currentItem.hide();this._createPlaceholder();e.containment&&this._setContainment();if(e.cursor){if(b("body").css("cursor"))this._storedCursor=b("body").css("cursor");b("body").css("cursor",e.cursor)}if(e.opacity){if(this.helper.css("opacity"))this._storedOpacity=this.helper.css("opacity");this.helper.css("opacity",e.opacity)}if(e.zIndex){if(this.helper.css("zIndex"))this._storedZIndex=this.helper.css("zIndex");this.helper.css("zIndex",e.zIndex)}if(this.scrollParent[0]!= +document&&this.scrollParent[0].tagName!="HTML")this.overflowOffset=this.scrollParent.offset();this._trigger("start",d,this._uiHash());this._preserveHelperProportions||this._cacheHelperProportions();if(!g)for(g=this.containers.length-1;g>=0;g--)this.containers[g]._trigger("activate",d,f._uiHash(this));if(b.ui.ddmanager)b.ui.ddmanager.current=this;b.ui.ddmanager&&!e.dropBehaviour&&b.ui.ddmanager.prepareOffsets(this,d);this.dragging=true;this.helper.addClass("ui-sortable-helper");this._mouseDrag(d); +return true},_mouseDrag:function(d){this.position=this._generatePosition(d);this.positionAbs=this._convertPositionTo("absolute");if(!this.lastPositionAbs)this.lastPositionAbs=this.positionAbs;if(this.options.scroll){var e=this.options,g=false;if(this.scrollParent[0]!=document&&this.scrollParent[0].tagName!="HTML"){if(this.overflowOffset.top+this.scrollParent[0].offsetHeight-d.pageY=0;e--){g=this.items[e];var f=g.item[0],a=this._intersectsWithPointer(g);if(a)if(f!=this.currentItem[0]&&this.placeholder[a==1?"next":"prev"]()[0]!=f&&!b.ui.contains(this.placeholder[0],f)&&(this.options.type=="semi-dynamic"?!b.ui.contains(this.element[0], +f):true)){this.direction=a==1?"down":"up";if(this.options.tolerance=="pointer"||this._intersectsWithSides(g))this._rearrange(d,g);else break;this._trigger("change",d,this._uiHash());break}}this._contactContainers(d);b.ui.ddmanager&&b.ui.ddmanager.drag(this,d);this._trigger("sort",d,this._uiHash());this.lastPositionAbs=this.positionAbs;return false},_mouseStop:function(d,e){if(d){b.ui.ddmanager&&!this.options.dropBehaviour&&b.ui.ddmanager.drop(this,d);if(this.options.revert){var g=this;e=g.placeholder.offset(); +g.reverting=true;b(this.helper).animate({left:e.left-this.offset.parent.left-g.margins.left+(this.offsetParent[0]==document.body?0:this.offsetParent[0].scrollLeft),top:e.top-this.offset.parent.top-g.margins.top+(this.offsetParent[0]==document.body?0:this.offsetParent[0].scrollTop)},parseInt(this.options.revert,10)||500,function(){g._clear(d)})}else this._clear(d,e);return false}},cancel:function(){var d=this;if(this.dragging){this._mouseUp({target:null});this.options.helper=="original"?this.currentItem.css(this._storedCSS).removeClass("ui-sortable-helper"): +this.currentItem.show();for(var e=this.containers.length-1;e>=0;e--){this.containers[e]._trigger("deactivate",null,d._uiHash(this));if(this.containers[e].containerCache.over){this.containers[e]._trigger("out",null,d._uiHash(this));this.containers[e].containerCache.over=0}}}if(this.placeholder){this.placeholder[0].parentNode&&this.placeholder[0].parentNode.removeChild(this.placeholder[0]);this.options.helper!="original"&&this.helper&&this.helper[0].parentNode&&this.helper.remove();b.extend(this,{helper:null, +dragging:false,reverting:false,_noFinalSort:null});this.domPosition.prev?b(this.domPosition.prev).after(this.currentItem):b(this.domPosition.parent).prepend(this.currentItem)}return this},serialize:function(d){var e=this._getItemsAsjQuery(d&&d.connected),g=[];d=d||{};b(e).each(function(){var f=(b(d.item||this).attr(d.attribute||"id")||"").match(d.expression||/(.+)[-=_](.+)/);if(f)g.push((d.key||f[1]+"[]")+"="+(d.key&&d.expression?f[1]:f[2]))});!g.length&&d.key&&g.push(d.key+"=");return g.join("&")}, +toArray:function(d){var e=this._getItemsAsjQuery(d&&d.connected),g=[];d=d||{};e.each(function(){g.push(b(d.item||this).attr(d.attribute||"id")||"")});return g},_intersectsWith:function(d){var e=this.positionAbs.left,g=e+this.helperProportions.width,f=this.positionAbs.top,a=f+this.helperProportions.height,c=d.left,h=c+d.width,i=d.top,j=i+d.height,n=this.offset.click.top,p=this.offset.click.left;n=f+n>i&&f+nc&&e+pd[this.floating?"width":"height"]?n:c0?"down":"up")},_getDragHorizontalDirection:function(){var d=this.positionAbs.left-this.lastPositionAbs.left;return d!=0&&(d>0?"right":"left")},refresh:function(d){this._refreshItems(d);this.refreshPositions();return this},_connectWith:function(){var d=this.options;return d.connectWith.constructor==String?[d.connectWith]:d.connectWith},_getItemsAsjQuery:function(d){var e=[],g=[],f=this._connectWith(); +if(f&&d)for(d=f.length-1;d>=0;d--)for(var a=b(f[d]),c=a.length-1;c>=0;c--){var h=b.data(a[c],"sortable");if(h&&h!=this&&!h.options.disabled)g.push([b.isFunction(h.options.items)?h.options.items.call(h.element):b(h.options.items,h.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),h])}g.push([b.isFunction(this.options.items)?this.options.items.call(this.element,null,{options:this.options,item:this.currentItem}):b(this.options.items,this.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"), +this]);for(d=g.length-1;d>=0;d--)g[d][0].each(function(){e.push(this)});return b(e)},_removeCurrentsFromItems:function(){for(var d=this.currentItem.find(":data(sortable-item)"),e=0;e=0;a--)for(var c=b(f[a]),h=c.length-1;h>=0;h--){var i=b.data(c[h],"sortable");if(i&&i!=this&&!i.options.disabled){g.push([b.isFunction(i.options.items)?i.options.items.call(i.element[0],d,{item:this.currentItem}):b(i.options.items,i.element),i]);this.containers.push(i)}}for(a=g.length-1;a>=0;a--){d=g[a][1];f=g[a][0];h=0;for(c=f.length;h=0;e--){var g=this.items[e],f=this.options.toleranceElement?b(this.options.toleranceElement,g.item):g.item;if(!d){g.width=f.outerWidth();g.height=f.outerHeight()}f=f.offset();g.left=f.left;g.top=f.top}if(this.options.custom&&this.options.custom.refreshContainers)this.options.custom.refreshContainers.call(this);else for(e=this.containers.length-1;e>=0;e--){f=this.containers[e].element.offset();this.containers[e].containerCache.left= +f.left;this.containers[e].containerCache.top=f.top;this.containers[e].containerCache.width=this.containers[e].element.outerWidth();this.containers[e].containerCache.height=this.containers[e].element.outerHeight()}return this},_createPlaceholder:function(d){var e=d||this,g=e.options;if(!g.placeholder||g.placeholder.constructor==String){var f=g.placeholder;g.placeholder={element:function(){var a=b(document.createElement(e.currentItem[0].nodeName)).addClass(f||e.currentItem[0].className+" ui-sortable-placeholder").removeClass("ui-sortable-helper")[0]; +if(!f)a.style.visibility="hidden";return a},update:function(a,c){if(!(f&&!g.forcePlaceholderSize)){c.height()||c.height(e.currentItem.innerHeight()-parseInt(e.currentItem.css("paddingTop")||0,10)-parseInt(e.currentItem.css("paddingBottom")||0,10));c.width()||c.width(e.currentItem.innerWidth()-parseInt(e.currentItem.css("paddingLeft")||0,10)-parseInt(e.currentItem.css("paddingRight")||0,10))}}}}e.placeholder=b(g.placeholder.element.call(e.element,e.currentItem));e.currentItem.after(e.placeholder); +g.placeholder.update(e,e.placeholder)},_contactContainers:function(d){for(var e=null,g=null,f=this.containers.length-1;f>=0;f--)if(!b.ui.contains(this.currentItem[0],this.containers[f].element[0]))if(this._intersectsWith(this.containers[f].containerCache)){if(!(e&&b.ui.contains(this.containers[f].element[0],e.element[0]))){e=this.containers[f];g=f}}else if(this.containers[f].containerCache.over){this.containers[f]._trigger("out",d,this._uiHash(this));this.containers[f].containerCache.over=0}if(e)if(this.containers.length=== +1){this.containers[g]._trigger("over",d,this._uiHash(this));this.containers[g].containerCache.over=1}else if(this.currentContainer!=this.containers[g]){e=1E4;f=null;for(var a=this.positionAbs[this.containers[g].floating?"left":"top"],c=this.items.length-1;c>=0;c--)if(b.ui.contains(this.containers[g].element[0],this.items[c].item[0])){var h=this.items[c][this.containers[g].floating?"left":"top"];if(Math.abs(h-a)this.containment[2])a=this.containment[2]+this.offset.click.left;if(d.pageY-this.offset.click.top>this.containment[3])c=this.containment[3]+this.offset.click.top}if(e.grid){c=this.originalPageY+Math.round((c-this.originalPageY)/e.grid[1])*e.grid[1];c=this.containment?!(c-this.offset.click.top< +this.containment[1]||c-this.offset.click.top>this.containment[3])?c:!(c-this.offset.click.topthis.containment[2])?a:!(a-this.offset.click.left=0;f--)if(b.ui.contains(this.containers[f].element[0], +this.currentItem[0])&&!e){g.push(function(a){return function(c){a._trigger("receive",c,this._uiHash(this))}}.call(this,this.containers[f]));g.push(function(a){return function(c){a._trigger("update",c,this._uiHash(this))}}.call(this,this.containers[f]))}}for(f=this.containers.length-1;f>=0;f--){e||g.push(function(a){return function(c){a._trigger("deactivate",c,this._uiHash(this))}}.call(this,this.containers[f]));if(this.containers[f].containerCache.over){g.push(function(a){return function(c){a._trigger("out", +c,this._uiHash(this))}}.call(this,this.containers[f]));this.containers[f].containerCache.over=0}}this._storedCursor&&b("body").css("cursor",this._storedCursor);this._storedOpacity&&this.helper.css("opacity",this._storedOpacity);if(this._storedZIndex)this.helper.css("zIndex",this._storedZIndex=="auto"?"":this._storedZIndex);this.dragging=false;if(this.cancelHelperRemoval){if(!e){this._trigger("beforeStop",d,this._uiHash());for(f=0;f").addClass("ui-effects-wrapper").css({fontSize:"100%",background:"transparent", +border:"none",margin:0,padding:0});l.wrap(m);m=l.parent();if(l.css("position")=="static"){m.css({position:"relative"});l.css({position:"relative"})}else{b.extend(k,{position:l.css("position"),zIndex:l.css("z-index")});b.each(["top","left","bottom","right"],function(o,q){k[q]=l.css(q);if(isNaN(parseInt(k[q],10)))k[q]="auto"});l.css({position:"relative",top:0,left:0,right:"auto",bottom:"auto"})}return m.css(k).show()},removeWrapper:function(l){if(l.parent().is(".ui-effects-wrapper"))return l.parent().replaceWith(l); +return l},setTransition:function(l,k,m,o){o=o||{};b.each(k,function(q,s){unit=l.cssUnit(s);if(unit[0]>0)o[s]=unit[0]*m+unit[1]});return o}});b.fn.extend({effect:function(l){var k=h.apply(this,arguments),m={options:k[1],duration:k[2],callback:k[3]};k=m.options.mode;var o=b.effects[l];if(b.fx.off||!o)return k?this[k](m.duration,m.callback):this.each(function(){m.callback&&m.callback.call(this)});return o.call(this,m)},_show:b.fn.show,show:function(l){if(i(l))return this._show.apply(this,arguments); +else{var k=h.apply(this,arguments);k[1].mode="show";return this.effect.apply(this,k)}},_hide:b.fn.hide,hide:function(l){if(i(l))return this._hide.apply(this,arguments);else{var k=h.apply(this,arguments);k[1].mode="hide";return this.effect.apply(this,k)}},__toggle:b.fn.toggle,toggle:function(l){if(i(l)||typeof l==="boolean"||b.isFunction(l))return this.__toggle.apply(this,arguments);else{var k=h.apply(this,arguments);k[1].mode="toggle";return this.effect.apply(this,k)}},cssUnit:function(l){var k=this.css(l), +m=[];b.each(["em","px","%","pt"],function(o,q){if(k.indexOf(q)>0)m=[parseFloat(k),q]});return m}});b.easing.jswing=b.easing.swing;b.extend(b.easing,{def:"easeOutQuad",swing:function(l,k,m,o,q){return b.easing[b.easing.def](l,k,m,o,q)},easeInQuad:function(l,k,m,o,q){return o*(k/=q)*k+m},easeOutQuad:function(l,k,m,o,q){return-o*(k/=q)*(k-2)+m},easeInOutQuad:function(l,k,m,o,q){if((k/=q/2)<1)return o/2*k*k+m;return-o/2*(--k*(k-2)-1)+m},easeInCubic:function(l,k,m,o,q){return o*(k/=q)*k*k+m},easeOutCubic:function(l, +k,m,o,q){return o*((k=k/q-1)*k*k+1)+m},easeInOutCubic:function(l,k,m,o,q){if((k/=q/2)<1)return o/2*k*k*k+m;return o/2*((k-=2)*k*k+2)+m},easeInQuart:function(l,k,m,o,q){return o*(k/=q)*k*k*k+m},easeOutQuart:function(l,k,m,o,q){return-o*((k=k/q-1)*k*k*k-1)+m},easeInOutQuart:function(l,k,m,o,q){if((k/=q/2)<1)return o/2*k*k*k*k+m;return-o/2*((k-=2)*k*k*k-2)+m},easeInQuint:function(l,k,m,o,q){return o*(k/=q)*k*k*k*k+m},easeOutQuint:function(l,k,m,o,q){return o*((k=k/q-1)*k*k*k*k+1)+m},easeInOutQuint:function(l, +k,m,o,q){if((k/=q/2)<1)return o/2*k*k*k*k*k+m;return o/2*((k-=2)*k*k*k*k+2)+m},easeInSine:function(l,k,m,o,q){return-o*Math.cos(k/q*(Math.PI/2))+o+m},easeOutSine:function(l,k,m,o,q){return o*Math.sin(k/q*(Math.PI/2))+m},easeInOutSine:function(l,k,m,o,q){return-o/2*(Math.cos(Math.PI*k/q)-1)+m},easeInExpo:function(l,k,m,o,q){return k==0?m:o*Math.pow(2,10*(k/q-1))+m},easeOutExpo:function(l,k,m,o,q){return k==q?m+o:o*(-Math.pow(2,-10*k/q)+1)+m},easeInOutExpo:function(l,k,m,o,q){if(k==0)return m;if(k== +q)return m+o;if((k/=q/2)<1)return o/2*Math.pow(2,10*(k-1))+m;return o/2*(-Math.pow(2,-10*--k)+2)+m},easeInCirc:function(l,k,m,o,q){return-o*(Math.sqrt(1-(k/=q)*k)-1)+m},easeOutCirc:function(l,k,m,o,q){return o*Math.sqrt(1-(k=k/q-1)*k)+m},easeInOutCirc:function(l,k,m,o,q){if((k/=q/2)<1)return-o/2*(Math.sqrt(1-k*k)-1)+m;return o/2*(Math.sqrt(1-(k-=2)*k)+1)+m},easeInElastic:function(l,k,m,o,q){l=1.70158;var s=0,r=o;if(k==0)return m;if((k/=q)==1)return m+o;s||(s=q*0.3);if(r").css({position:"absolute",visibility:"visible",left:-j*(c/g),top:-i*(h/e)}).parent().addClass("ui-effects-explode").css({position:"absolute",overflow:"hidden",width:c/g,height:h/e,left:a.left+j*(c/g)+(d.options.mode=="show"?(j-Math.floor(g/2))*(c/g):0),top:a.top+i*(h/e)+(d.options.mode=="show"?(i-Math.floor(e/2))*(h/e):0),opacity:d.options.mode=="show"?0:1}).animate({left:a.left+j*(c/g)+(d.options.mode=="show"?0:(j-Math.floor(g/2))*(c/g)),top:a.top+ +i*(h/e)+(d.options.mode=="show"?0:(i-Math.floor(e/2))*(h/e)),opacity:d.options.mode=="show"?1:0},d.duration||500);setTimeout(function(){d.options.mode=="show"?f.css({visibility:"visible"}):f.css({visibility:"visible"}).hide();d.callback&&d.callback.apply(f[0]);f.dequeue();b("div.ui-effects-explode").remove()},d.duration||500)})}})(jQuery); +(function(b){b.effects.fade=function(d){return this.queue(function(){var e=b(this),g=b.effects.setMode(e,d.options.mode||"hide");e.animate({opacity:g},{queue:false,duration:d.duration,easing:d.options.easing,complete:function(){d.callback&&d.callback.apply(this,arguments);e.dequeue()}})})}})(jQuery); +(function(b){b.effects.fold=function(d){return this.queue(function(){var e=b(this),g=["position","top","bottom","left","right"],f=b.effects.setMode(e,d.options.mode||"hide"),a=d.options.size||15,c=!!d.options.horizFirst,h=d.duration?d.duration/2:b.fx.speeds._default/2;b.effects.save(e,g);e.show();var i=b.effects.createWrapper(e).css({overflow:"hidden"}),j=f=="show"!=c,n=j?["width","height"]:["height","width"];j=j?[i.width(),i.height()]:[i.height(),i.width()];var p=/([0-9]+)%/.exec(a);if(p)a=parseInt(p[1], +10)/100*j[f=="hide"?0:1];if(f=="show")i.css(c?{height:0,width:a}:{height:a,width:0});c={};p={};c[n[0]]=f=="show"?j[0]:a;p[n[1]]=f=="show"?j[1]:0;i.animate(c,h,d.options.easing).animate(p,h,d.options.easing,function(){f=="hide"&&e.hide();b.effects.restore(e,g);b.effects.removeWrapper(e);d.callback&&d.callback.apply(e[0],arguments);e.dequeue()})})}})(jQuery); +(function(b){b.effects.highlight=function(d){return this.queue(function(){var e=b(this),g=["backgroundImage","backgroundColor","opacity"],f=b.effects.setMode(e,d.options.mode||"show"),a={backgroundColor:e.css("backgroundColor")};if(f=="hide")a.opacity=0;b.effects.save(e,g);e.show().css({backgroundImage:"none",backgroundColor:d.options.color||"#ffff99"}).animate(a,{queue:false,duration:d.duration,easing:d.options.easing,complete:function(){f=="hide"&&e.hide();b.effects.restore(e,g);f=="show"&&!b.support.opacity&& +this.style.removeAttribute("filter");d.callback&&d.callback.apply(this,arguments);e.dequeue()}})})}})(jQuery); +(function(b){b.effects.pulsate=function(d){return this.queue(function(){var e=b(this),g=b.effects.setMode(e,d.options.mode||"show");times=(d.options.times||5)*2-1;duration=d.duration?d.duration/2:b.fx.speeds._default/2;isVisible=e.is(":visible");animateTo=0;if(!isVisible){e.css("opacity",0).show();animateTo=1}if(g=="hide"&&isVisible||g=="show"&&!isVisible)times--;for(g=0;g').appendTo(document.body).addClass(d.options.className).css({top:f.top,left:f.left,height:e.innerHeight(),width:e.innerWidth(),position:"absolute"}).animate(g,d.duration,d.options.easing,function(){a.remove();d.callback&&d.callback.apply(e[0],arguments); +e.dequeue()})})}})(jQuery); +(function(b){b.widget("ui.accordion",{options:{active:0,animated:"slide",autoHeight:true,clearStyle:false,collapsible:false,event:"click",fillSpace:false,header:"> li > :first-child,> :not(li):even",icons:{header:"ui-icon-triangle-1-e",headerSelected:"ui-icon-triangle-1-s"},navigation:false,navigationFilter:function(){return this.href.toLowerCase()===location.href.toLowerCase()}},_create:function(){var d=this,e=d.options;d.running=0;d.element.addClass("ui-accordion ui-widget ui-helper-reset").children("li").addClass("ui-accordion-li-fix");d.headers= +d.element.find(e.header).addClass("ui-accordion-header ui-helper-reset ui-state-default ui-corner-all").bind("mouseenter.accordion",function(){e.disabled||b(this).addClass("ui-state-hover")}).bind("mouseleave.accordion",function(){e.disabled||b(this).removeClass("ui-state-hover")}).bind("focus.accordion",function(){e.disabled||b(this).addClass("ui-state-focus")}).bind("blur.accordion",function(){e.disabled||b(this).removeClass("ui-state-focus")});d.headers.next().addClass("ui-accordion-content ui-helper-reset ui-widget-content ui-corner-bottom"); +if(e.navigation){var g=d.element.find("a").filter(e.navigationFilter).eq(0);if(g.length){var f=g.closest(".ui-accordion-header");d.active=f.length?f:g.closest(".ui-accordion-content").prev()}}d.active=d._findActive(d.active||e.active).addClass("ui-state-default ui-state-active").toggleClass("ui-corner-all").toggleClass("ui-corner-top");d.active.next().addClass("ui-accordion-content-active");d._createIcons();d.resize();d.element.attr("role","tablist");d.headers.attr("role","tab").bind("keydown.accordion", +function(a){return d._keydown(a)}).next().attr("role","tabpanel");d.headers.not(d.active||"").attr({"aria-expanded":"false","aria-selected":"false",tabIndex:-1}).next().hide();d.active.length?d.active.attr({"aria-expanded":"true","aria-selected":"true",tabIndex:0}):d.headers.eq(0).attr("tabIndex",0);b.browser.safari||d.headers.find("a").attr("tabIndex",-1);e.event&&d.headers.bind(e.event.split(" ").join(".accordion ")+".accordion",function(a){d._clickHandler.call(d,a,this);a.preventDefault()})},_createIcons:function(){var d= +this.options;if(d.icons){b("").addClass("ui-icon "+d.icons.header).prependTo(this.headers);this.active.children(".ui-icon").toggleClass(d.icons.header).toggleClass(d.icons.headerSelected);this.element.addClass("ui-accordion-icons")}},_destroyIcons:function(){this.headers.children(".ui-icon").remove();this.element.removeClass("ui-accordion-icons")},destroy:function(){var d=this.options;this.element.removeClass("ui-accordion ui-widget ui-helper-reset").removeAttr("role");this.headers.unbind(".accordion").removeClass("ui-accordion-header ui-accordion-disabled ui-helper-reset ui-state-default ui-corner-all ui-state-active ui-state-disabled ui-corner-top").removeAttr("role").removeAttr("aria-expanded").removeAttr("aria-selected").removeAttr("tabIndex"); +this.headers.find("a").removeAttr("tabIndex");this._destroyIcons();var e=this.headers.next().css("display","").removeAttr("role").removeClass("ui-helper-reset ui-widget-content ui-corner-bottom ui-accordion-content ui-accordion-content-active ui-accordion-disabled ui-state-disabled");if(d.autoHeight||d.fillHeight)e.css("height","");return b.Widget.prototype.destroy.call(this)},_setOption:function(d,e){b.Widget.prototype._setOption.apply(this,arguments);d=="active"&&this.activate(e);if(d=="icons"){this._destroyIcons(); +e&&this._createIcons()}if(d=="disabled")this.headers.add(this.headers.next())[e?"addClass":"removeClass"]("ui-accordion-disabled ui-state-disabled")},_keydown:function(d){if(!(this.options.disabled||d.altKey||d.ctrlKey)){var e=b.ui.keyCode,g=this.headers.length,f=this.headers.index(d.target),a=false;switch(d.keyCode){case e.RIGHT:case e.DOWN:a=this.headers[(f+1)%g];break;case e.LEFT:case e.UP:a=this.headers[(f-1+g)%g];break;case e.SPACE:case e.ENTER:this._clickHandler({target:d.target},d.target); +d.preventDefault()}if(a){b(d.target).attr("tabIndex",-1);b(a).attr("tabIndex",0);a.focus();return false}return true}},resize:function(){var d=this.options,e;if(d.fillSpace){if(b.browser.msie){var g=this.element.parent().css("overflow");this.element.parent().css("overflow","hidden")}e=this.element.parent().height();b.browser.msie&&this.element.parent().css("overflow",g);this.headers.each(function(){e-=b(this).outerHeight(true)});this.headers.next().each(function(){b(this).height(Math.max(0,e-b(this).innerHeight()+ +b(this).height()))}).css("overflow","auto")}else if(d.autoHeight){e=0;this.headers.next().each(function(){e=Math.max(e,b(this).height("").height())}).height(e)}return this},activate:function(d){this.options.active=d;d=this._findActive(d)[0];this._clickHandler({target:d},d);return this},_findActive:function(d){return d?typeof d==="number"?this.headers.filter(":eq("+d+")"):this.headers.not(this.headers.not(d)):d===false?b([]):this.headers.filter(":eq(0)")},_clickHandler:function(d,e){var g=this.options; +if(!g.disabled)if(d.target){d=b(d.currentTarget||e);e=d[0]===this.active[0];g.active=g.collapsible&&e?false:this.headers.index(d);if(!(this.running||!g.collapsible&&e)){var f=this.active;i=d.next();c=this.active.next();h={options:g,newHeader:e&&g.collapsible?b([]):d,oldHeader:this.active,newContent:e&&g.collapsible?b([]):i,oldContent:c};var a=this.headers.index(this.active[0])>this.headers.index(d[0]);this.active=e?b([]):d;this._toggle(i,c,h,e,a);f.removeClass("ui-state-active ui-corner-top").addClass("ui-state-default ui-corner-all").children(".ui-icon").removeClass(g.icons.headerSelected).addClass(g.icons.header); +if(!e){d.removeClass("ui-state-default ui-corner-all").addClass("ui-state-active ui-corner-top").children(".ui-icon").removeClass(g.icons.header).addClass(g.icons.headerSelected);d.next().addClass("ui-accordion-content-active")}}}else if(g.collapsible){this.active.removeClass("ui-state-active ui-corner-top").addClass("ui-state-default ui-corner-all").children(".ui-icon").removeClass(g.icons.headerSelected).addClass(g.icons.header);this.active.next().addClass("ui-accordion-content-active");var c=this.active.next(), +h={options:g,newHeader:b([]),oldHeader:g.active,newContent:b([]),oldContent:c},i=this.active=b([]);this._toggle(i,c,h)}},_toggle:function(d,e,g,f,a){var c=this,h=c.options;c.toShow=d;c.toHide=e;c.data=g;var i=function(){if(c)return c._completed.apply(c,arguments)};c._trigger("changestart",null,c.data);c.running=e.size()===0?d.size():e.size();if(h.animated){g={};g=h.collapsible&&f?{toShow:b([]),toHide:e,complete:i,down:a,autoHeight:h.autoHeight||h.fillSpace}:{toShow:d,toHide:e,complete:i,down:a,autoHeight:h.autoHeight|| +h.fillSpace};if(!h.proxied)h.proxied=h.animated;if(!h.proxiedDuration)h.proxiedDuration=h.duration;h.animated=b.isFunction(h.proxied)?h.proxied(g):h.proxied;h.duration=b.isFunction(h.proxiedDuration)?h.proxiedDuration(g):h.proxiedDuration;f=b.ui.accordion.animations;var j=h.duration,n=h.animated;if(n&&!f[n]&&!b.easing[n])n="slide";f[n]||(f[n]=function(p){this.slide(p,{easing:n,duration:j||700})});f[n](g)}else{if(h.collapsible&&f)d.toggle();else{e.hide();d.show()}i(true)}e.prev().attr({"aria-expanded":"false", +"aria-selected":"false",tabIndex:-1}).blur();d.prev().attr({"aria-expanded":"true","aria-selected":"true",tabIndex:0}).focus()},_completed:function(d){this.running=d?0:--this.running;if(!this.running){this.options.clearStyle&&this.toShow.add(this.toHide).css({height:"",overflow:""});this.toHide.removeClass("ui-accordion-content-active");if(this.toHide.length)this.toHide.parent()[0].className=this.toHide.parent()[0].className;this._trigger("change",null,this.data)}}});b.extend(b.ui.accordion,{version:"1.8.11", +animations:{slide:function(d,e){d=b.extend({easing:"swing",duration:300},d,e);if(d.toHide.size())if(d.toShow.size()){var g=d.toShow.css("overflow"),f=0,a={},c={},h;e=d.toShow;h=e[0].style.width;e.width(parseInt(e.parent().width(),10)-parseInt(e.css("paddingLeft"),10)-parseInt(e.css("paddingRight"),10)-(parseInt(e.css("borderLeftWidth"),10)||0)-(parseInt(e.css("borderRightWidth"),10)||0));b.each(["height","paddingTop","paddingBottom"],function(i,j){c[j]="hide";i=(""+b.css(d.toShow[0],j)).match(/^([\d+-.]+)(.*)$/); +a[j]={value:i[1],unit:i[2]||"px"}});d.toShow.css({height:0,overflow:"hidden"}).show();d.toHide.filter(":hidden").each(d.complete).end().filter(":visible").animate(c,{step:function(i,j){if(j.prop=="height")f=j.end-j.start===0?0:(j.now-j.start)/(j.end-j.start);d.toShow[0].style[j.prop]=f*a[j.prop].value+a[j.prop].unit},duration:d.duration,easing:d.easing,complete:function(){d.autoHeight||d.toShow.css("height","");d.toShow.css({width:h,overflow:g});d.complete()}})}else d.toHide.animate({height:"hide", +paddingTop:"hide",paddingBottom:"hide"},d);else d.toShow.animate({height:"show",paddingTop:"show",paddingBottom:"show"},d)},bounceslide:function(d){this.slide(d,{easing:d.down?"easeOutBounce":"swing",duration:d.down?1E3:200})}}})})(jQuery); +(function(b){var d=0;b.widget("ui.autocomplete",{options:{appendTo:"body",autoFocus:false,delay:300,minLength:1,position:{my:"left top",at:"left bottom",collision:"none"},source:null},pending:0,_create:function(){var e=this,g=this.element[0].ownerDocument,f;this.element.addClass("ui-autocomplete-input").attr("autocomplete","off").attr({role:"textbox","aria-autocomplete":"list","aria-haspopup":"true"}).bind("keydown.autocomplete",function(a){if(!(e.options.disabled||e.element.attr("readonly"))){f= +false;var c=b.ui.keyCode;switch(a.keyCode){case c.PAGE_UP:e._move("previousPage",a);break;case c.PAGE_DOWN:e._move("nextPage",a);break;case c.UP:e._move("previous",a);a.preventDefault();break;case c.DOWN:e._move("next",a);a.preventDefault();break;case c.ENTER:case c.NUMPAD_ENTER:if(e.menu.active){f=true;a.preventDefault()}case c.TAB:if(!e.menu.active)return;e.menu.select(a);break;case c.ESCAPE:e.element.val(e.term);e.close(a);break;default:clearTimeout(e.searching);e.searching=setTimeout(function(){if(e.term!= +e.element.val()){e.selectedItem=null;e.search(null,a)}},e.options.delay);break}}}).bind("keypress.autocomplete",function(a){if(f){f=false;a.preventDefault()}}).bind("focus.autocomplete",function(){if(!e.options.disabled){e.selectedItem=null;e.previous=e.element.val()}}).bind("blur.autocomplete",function(a){if(!e.options.disabled){clearTimeout(e.searching);e.closing=setTimeout(function(){e.close(a);e._change(a)},150)}});this._initSource();this.response=function(){return e._response.apply(e,arguments)}; +this.menu=b("
      ").addClass("ui-autocomplete").appendTo(b(this.options.appendTo||"body",g)[0]).mousedown(function(a){var c=e.menu.element[0];b(a.target).closest(".ui-menu-item").length||setTimeout(function(){b(document).one("mousedown",function(h){h.target!==e.element[0]&&h.target!==c&&!b.ui.contains(c,h.target)&&e.close()})},1);setTimeout(function(){clearTimeout(e.closing)},13)}).menu({focus:function(a,c){c=c.item.data("item.autocomplete");false!==e._trigger("focus",a,{item:c})&&/^key/.test(a.originalEvent.type)&& +e.element.val(c.value)},selected:function(a,c){var h=c.item.data("item.autocomplete"),i=e.previous;if(e.element[0]!==g.activeElement){e.element.focus();e.previous=i;setTimeout(function(){e.previous=i;e.selectedItem=h},1)}false!==e._trigger("select",a,{item:h})&&e.element.val(h.value);e.term=e.element.val();e.close(a);e.selectedItem=h},blur:function(){e.menu.element.is(":visible")&&e.element.val()!==e.term&&e.element.val(e.term)}}).zIndex(this.element.zIndex()+1).css({top:0,left:0}).hide().data("menu"); +b.fn.bgiframe&&this.menu.element.bgiframe()},destroy:function(){this.element.removeClass("ui-autocomplete-input").removeAttr("autocomplete").removeAttr("role").removeAttr("aria-autocomplete").removeAttr("aria-haspopup");this.menu.element.remove();b.Widget.prototype.destroy.call(this)},_setOption:function(e,g){b.Widget.prototype._setOption.apply(this,arguments);e==="source"&&this._initSource();if(e==="appendTo")this.menu.element.appendTo(b(g||"body",this.element[0].ownerDocument)[0]);e==="disabled"&& +g&&this.xhr&&this.xhr.abort()},_initSource:function(){var e=this,g,f;if(b.isArray(this.options.source)){g=this.options.source;this.source=function(a,c){c(b.ui.autocomplete.filter(g,a.term))}}else if(typeof this.options.source==="string"){f=this.options.source;this.source=function(a,c){e.xhr&&e.xhr.abort();e.xhr=b.ajax({url:f,data:a,dataType:"json",autocompleteRequest:++d,success:function(h){this.autocompleteRequest===d&&c(h)},error:function(){this.autocompleteRequest===d&&c([])}})}}else this.source= +this.options.source},search:function(e,g){e=e!=null?e:this.element.val();this.term=this.element.val();if(e.length").data("item.autocomplete",g).append(b("").text(g.label)).appendTo(e)},_move:function(e,g){if(this.menu.element.is(":visible"))if(this.menu.first()&&/^previous/.test(e)||this.menu.last()&&/^next/.test(e)){this.element.val(this.term);this.menu.deactivate()}else this.menu[e](g);else this.search(null,g)},widget:function(){return this.menu.element}});b.extend(b.ui.autocomplete,{escapeRegex:function(e){return e.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, +"\\$&")},filter:function(e,g){var f=new RegExp(b.ui.autocomplete.escapeRegex(g),"i");return b.grep(e,function(a){return f.test(a.label||a.value||a)})}})})(jQuery); +(function(b){b.widget("ui.menu",{_create:function(){var d=this;this.element.addClass("ui-menu ui-widget ui-widget-content ui-corner-all").attr({role:"listbox","aria-activedescendant":"ui-active-menuitem"}).click(function(e){if(b(e.target).closest(".ui-menu-item a").length){e.preventDefault();d.select(e)}});this.refresh()},refresh:function(){var d=this;this.element.children("li:not(.ui-menu-item):has(a)").addClass("ui-menu-item").attr("role","menuitem").children("a").addClass("ui-corner-all").attr("tabindex", +-1).mouseenter(function(e){d.activate(e,b(this).parent())}).mouseleave(function(){d.deactivate()})},activate:function(d,e){this.deactivate();if(this.hasScroll()){var g=e.offset().top-this.element.offset().top,f=this.element.attr("scrollTop"),a=this.element.height();if(g<0)this.element.attr("scrollTop",f+g);else g>=a&&this.element.attr("scrollTop",f+g-a+e.height())}this.active=e.eq(0).children("a").addClass("ui-state-hover").attr("id","ui-active-menuitem").end();this._trigger("focus",d,{item:e})}, +deactivate:function(){if(this.active){this.active.children("a").removeClass("ui-state-hover").removeAttr("id");this._trigger("blur");this.active=null}},next:function(d){this.move("next",".ui-menu-item:first",d)},previous:function(d){this.move("prev",".ui-menu-item:last",d)},first:function(){return this.active&&!this.active.prevAll(".ui-menu-item").length},last:function(){return this.active&&!this.active.nextAll(".ui-menu-item").length},move:function(d,e,g){if(this.active){d=this.active[d+"All"](".ui-menu-item").eq(0); +d.length?this.activate(g,d):this.activate(g,this.element.children(e))}else this.activate(g,this.element.children(e))},nextPage:function(d){if(this.hasScroll())if(!this.active||this.last())this.activate(d,this.element.children(".ui-menu-item:first"));else{var e=this.active.offset().top,g=this.element.height(),f=this.element.children(".ui-menu-item").filter(function(){var a=b(this).offset().top-e-g+b(this).height();return a<10&&a>-10});f.length||(f=this.element.children(".ui-menu-item:last"));this.activate(d, +f)}else this.activate(d,this.element.children(".ui-menu-item").filter(!this.active||this.last()?":first":":last"))},previousPage:function(d){if(this.hasScroll())if(!this.active||this.first())this.activate(d,this.element.children(".ui-menu-item:last"));else{var e=this.active.offset().top,g=this.element.height();result=this.element.children(".ui-menu-item").filter(function(){var f=b(this).offset().top-e+g-b(this).height();return f<10&&f>-10});result.length||(result=this.element.children(".ui-menu-item:first")); +this.activate(d,result)}else this.activate(d,this.element.children(".ui-menu-item").filter(!this.active||this.first()?":last":":first"))},hasScroll:function(){return this.element.height()").addClass("ui-button-text").html(this.options.label).appendTo(f.empty()).text(),c=this.options.icons,h=c.primary&&c.secondary,i=[];if(c.primary||c.secondary){if(this.options.text)i.push("ui-button-text-icon"+(h?"s":c.primary?"-primary":"-secondary"));c.primary&&f.prepend("");c.secondary&&f.append("");if(!this.options.text){i.push(h?"ui-button-icons-only": +"ui-button-icon-only");this.hasTitle||f.attr("title",a)}}else i.push("ui-button-text-only");f.addClass(i.join(" "))}}});b.widget("ui.buttonset",{options:{items:":button, :submit, :reset, :checkbox, :radio, a, :data(button)"},_create:function(){this.element.addClass("ui-buttonset")},_init:function(){this.refresh()},_setOption:function(f,a){f==="disabled"&&this.buttons.button("option",f,a);b.Widget.prototype._setOption.apply(this,arguments)},refresh:function(){this.buttons=this.element.find(this.options.items).filter(":ui-button").button("refresh").end().not(":ui-button").button().end().map(function(){return b(this).button("widget")[0]}).removeClass("ui-corner-all ui-corner-left ui-corner-right").filter(":first").addClass("ui-corner-left").end().filter(":last").addClass("ui-corner-right").end().end()}, +destroy:function(){this.element.removeClass("ui-buttonset");this.buttons.map(function(){return b(this).button("widget")[0]}).removeClass("ui-corner-left ui-corner-right").end().button("destroy");b.Widget.prototype.destroy.call(this)}})})(jQuery); +(function(b,d){function e(){this.debug=false;this._curInst=null;this._keyEvent=false;this._disabledInputs=[];this._inDialog=this._datepickerShowing=false;this._mainDivId="ui-datepicker-div";this._inlineClass="ui-datepicker-inline";this._appendClass="ui-datepicker-append";this._triggerClass="ui-datepicker-trigger";this._dialogClass="ui-datepicker-dialog";this._disableClass="ui-datepicker-disabled";this._unselectableClass="ui-datepicker-unselectable";this._currentClass="ui-datepicker-current-day";this._dayOverClass= +"ui-datepicker-days-cell-over";this.regional=[];this.regional[""]={closeText:"Done",prevText:"Prev",nextText:"Next",currentText:"Today",monthNames:["January","February","March","April","May","June","July","August","September","October","November","December"],monthNamesShort:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"],dayNames:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],dayNamesShort:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],dayNamesMin:["Su", +"Mo","Tu","We","Th","Fr","Sa"],weekHeader:"Wk",dateFormat:"mm/dd/yy",firstDay:0,isRTL:false,showMonthAfterYear:false,yearSuffix:""};this._defaults={showOn:"focus",showAnim:"fadeIn",showOptions:{},defaultDate:null,appendText:"",buttonText:"...",buttonImage:"",buttonImageOnly:false,hideIfNoPrevNext:false,navigationAsDateFormat:false,gotoCurrent:false,changeMonth:false,changeYear:false,yearRange:"c-10:c+10",showOtherMonths:false,selectOtherMonths:false,showWeek:false,calculateWeek:this.iso8601Week,shortYearCutoff:"+10", +minDate:null,maxDate:null,duration:"fast",beforeShowDay:null,beforeShow:null,onSelect:null,onChangeMonthYear:null,onClose:null,numberOfMonths:1,showCurrentAtPos:0,stepMonths:1,stepBigMonths:12,altField:"",altFormat:"",constrainInput:true,showButtonPanel:false,autoSize:false};b.extend(this._defaults,this.regional[""]);this.dpDiv=b('
      ')}function g(a,c){b.extend(a,c);for(var h in c)if(c[h]== +null||c[h]==d)a[h]=c[h];return a}b.extend(b.ui,{datepicker:{version:"1.8.11"}});var f=(new Date).getTime();b.extend(e.prototype,{markerClassName:"hasDatepicker",log:function(){this.debug&&console.log.apply("",arguments)},_widgetDatepicker:function(){return this.dpDiv},setDefaults:function(a){g(this._defaults,a||{});return this},_attachDatepicker:function(a,c){var h=null;for(var i in this._defaults){var j=a.getAttribute("date:"+i);if(j){h=h||{};try{h[i]=eval(j)}catch(n){h[i]=j}}}i=a.nodeName.toLowerCase(); +j=i=="div"||i=="span";if(!a.id){this.uuid+=1;a.id="dp"+this.uuid}var p=this._newInst(b(a),j);p.settings=b.extend({},c||{},h||{});if(i=="input")this._connectDatepicker(a,p);else j&&this._inlineDatepicker(a,p)},_newInst:function(a,c){return{id:a[0].id.replace(/([^A-Za-z0-9_-])/g,"\\\\$1"),input:a,selectedDay:0,selectedMonth:0,selectedYear:0,drawMonth:0,drawYear:0,inline:c,dpDiv:!c?this.dpDiv:b('
      ')}}, +_connectDatepicker:function(a,c){var h=b(a);c.append=b([]);c.trigger=b([]);if(!h.hasClass(this.markerClassName)){this._attachments(h,c);h.addClass(this.markerClassName).keydown(this._doKeyDown).keypress(this._doKeyPress).keyup(this._doKeyUp).bind("setData.datepicker",function(i,j,n){c.settings[j]=n}).bind("getData.datepicker",function(i,j){return this._get(c,j)});this._autoSize(c);b.data(a,"datepicker",c)}},_attachments:function(a,c){var h=this._get(c,"appendText"),i=this._get(c,"isRTL");c.append&& +c.append.remove();if(h){c.append=b(''+h+"");a[i?"before":"after"](c.append)}a.unbind("focus",this._showDatepicker);c.trigger&&c.trigger.remove();h=this._get(c,"showOn");if(h=="focus"||h=="both")a.focus(this._showDatepicker);if(h=="button"||h=="both"){h=this._get(c,"buttonText");var j=this._get(c,"buttonImage");c.trigger=b(this._get(c,"buttonImageOnly")?b("").addClass(this._triggerClass).attr({src:j,alt:h,title:h}):b('').addClass(this._triggerClass).html(j== +""?h:b("").attr({src:j,alt:h,title:h})));a[i?"before":"after"](c.trigger);c.trigger.click(function(){b.datepicker._datepickerShowing&&b.datepicker._lastInput==a[0]?b.datepicker._hideDatepicker():b.datepicker._showDatepicker(a[0]);return false})}},_autoSize:function(a){if(this._get(a,"autoSize")&&!a.inline){var c=new Date(2009,11,20),h=this._get(a,"dateFormat");if(h.match(/[DM]/)){var i=function(j){for(var n=0,p=0,l=0;ln){n=j[l].length;p=l}return p};c.setMonth(i(this._get(a, +h.match(/MM/)?"monthNames":"monthNamesShort")));c.setDate(i(this._get(a,h.match(/DD/)?"dayNames":"dayNamesShort"))+20-c.getDay())}a.input.attr("size",this._formatDate(a,c).length)}},_inlineDatepicker:function(a,c){var h=b(a);if(!h.hasClass(this.markerClassName)){h.addClass(this.markerClassName).append(c.dpDiv).bind("setData.datepicker",function(i,j,n){c.settings[j]=n}).bind("getData.datepicker",function(i,j){return this._get(c,j)});b.data(a,"datepicker",c);this._setDate(c,this._getDefaultDate(c), +true);this._updateDatepicker(c);this._updateAlternate(c);c.dpDiv.show()}},_dialogDatepicker:function(a,c,h,i,j){a=this._dialogInst;if(!a){this.uuid+=1;this._dialogInput=b('');this._dialogInput.keydown(this._doKeyDown);b("body").append(this._dialogInput);a=this._dialogInst=this._newInst(this._dialogInput,false);a.settings={};b.data(this._dialogInput[0],"datepicker",a)}g(a.settings,i||{}); +c=c&&c.constructor==Date?this._formatDate(a,c):c;this._dialogInput.val(c);this._pos=j?j.length?j:[j.pageX,j.pageY]:null;if(!this._pos)this._pos=[document.documentElement.clientWidth/2-100+(document.documentElement.scrollLeft||document.body.scrollLeft),document.documentElement.clientHeight/2-150+(document.documentElement.scrollTop||document.body.scrollTop)];this._dialogInput.css("left",this._pos[0]+20+"px").css("top",this._pos[1]+"px");a.settings.onSelect=h;this._inDialog=true;this.dpDiv.addClass(this._dialogClass); +this._showDatepicker(this._dialogInput[0]);b.blockUI&&b.blockUI(this.dpDiv);b.data(this._dialogInput[0],"datepicker",a);return this},_destroyDatepicker:function(a){var c=b(a),h=b.data(a,"datepicker");if(c.hasClass(this.markerClassName)){var i=a.nodeName.toLowerCase();b.removeData(a,"datepicker");if(i=="input"){h.append.remove();h.trigger.remove();c.removeClass(this.markerClassName).unbind("focus",this._showDatepicker).unbind("keydown",this._doKeyDown).unbind("keypress",this._doKeyPress).unbind("keyup", +this._doKeyUp)}else if(i=="div"||i=="span")c.removeClass(this.markerClassName).empty()}},_enableDatepicker:function(a){var c=b(a),h=b.data(a,"datepicker");if(c.hasClass(this.markerClassName)){var i=a.nodeName.toLowerCase();if(i=="input"){a.disabled=false;h.trigger.filter("button").each(function(){this.disabled=false}).end().filter("img").css({opacity:"1.0",cursor:""})}else if(i=="div"||i=="span")c.children("."+this._inlineClass).children().removeClass("ui-state-disabled");this._disabledInputs=b.map(this._disabledInputs, +function(j){return j==a?null:j})}},_disableDatepicker:function(a){var c=b(a),h=b.data(a,"datepicker");if(c.hasClass(this.markerClassName)){var i=a.nodeName.toLowerCase();if(i=="input"){a.disabled=true;h.trigger.filter("button").each(function(){this.disabled=true}).end().filter("img").css({opacity:"0.5",cursor:"default"})}else if(i=="div"||i=="span")c.children("."+this._inlineClass).children().addClass("ui-state-disabled");this._disabledInputs=b.map(this._disabledInputs,function(j){return j==a?null: +j});this._disabledInputs[this._disabledInputs.length]=a}},_isDisabledDatepicker:function(a){if(!a)return false;for(var c=0;c-1}},_doKeyUp:function(a){a=b.datepicker._getInst(a.target); +if(a.input.val()!=a.lastVal)try{if(b.datepicker.parseDate(b.datepicker._get(a,"dateFormat"),a.input?a.input.val():null,b.datepicker._getFormatConfig(a))){b.datepicker._setDateFromField(a);b.datepicker._updateAlternate(a);b.datepicker._updateDatepicker(a)}}catch(c){b.datepicker.log(c)}return true},_showDatepicker:function(a){a=a.target||a;if(a.nodeName.toLowerCase()!="input")a=b("input",a.parentNode)[0];if(!(b.datepicker._isDisabledDatepicker(a)||b.datepicker._lastInput==a)){var c=b.datepicker._getInst(a); +b.datepicker._curInst&&b.datepicker._curInst!=c&&b.datepicker._curInst.dpDiv.stop(true,true);var h=b.datepicker._get(c,"beforeShow");g(c.settings,h?h.apply(a,[a,c]):{});c.lastVal=null;b.datepicker._lastInput=a;b.datepicker._setDateFromField(c);if(b.datepicker._inDialog)a.value="";if(!b.datepicker._pos){b.datepicker._pos=b.datepicker._findPos(a);b.datepicker._pos[1]+=a.offsetHeight}var i=false;b(a).parents().each(function(){i|=b(this).css("position")=="fixed";return!i});if(i&&b.browser.opera){b.datepicker._pos[0]-= +document.documentElement.scrollLeft;b.datepicker._pos[1]-=document.documentElement.scrollTop}h={left:b.datepicker._pos[0],top:b.datepicker._pos[1]};b.datepicker._pos=null;c.dpDiv.empty();c.dpDiv.css({position:"absolute",display:"block",top:"-1000px"});b.datepicker._updateDatepicker(c);h=b.datepicker._checkOffset(c,h,i);c.dpDiv.css({position:b.datepicker._inDialog&&b.blockUI?"static":i?"fixed":"absolute",display:"none",left:h.left+"px",top:h.top+"px"});if(!c.inline){h=b.datepicker._get(c,"showAnim"); +var j=b.datepicker._get(c,"duration"),n=function(){b.datepicker._datepickerShowing=true;var p=c.dpDiv.find("iframe.ui-datepicker-cover");if(p.length){var l=b.datepicker._getBorders(c.dpDiv);p.css({left:-l[0],top:-l[1],width:c.dpDiv.outerWidth(),height:c.dpDiv.outerHeight()})}};c.dpDiv.zIndex(b(a).zIndex()+1);b.effects&&b.effects[h]?c.dpDiv.show(h,b.datepicker._get(c,"showOptions"),j,n):c.dpDiv[h||"show"](h?j:null,n);if(!h||!j)n();c.input.is(":visible")&&!c.input.is(":disabled")&&c.input.focus();b.datepicker._curInst= +c}}},_updateDatepicker:function(a){var c=this,h=b.datepicker._getBorders(a.dpDiv);a.dpDiv.empty().append(this._generateHTML(a));var i=a.dpDiv.find("iframe.ui-datepicker-cover");i.length&&i.css({left:-h[0],top:-h[1],width:a.dpDiv.outerWidth(),height:a.dpDiv.outerHeight()});a.dpDiv.find("button, .ui-datepicker-prev, .ui-datepicker-next, .ui-datepicker-calendar td a").bind("mouseout",function(){b(this).removeClass("ui-state-hover");this.className.indexOf("ui-datepicker-prev")!=-1&&b(this).removeClass("ui-datepicker-prev-hover"); +this.className.indexOf("ui-datepicker-next")!=-1&&b(this).removeClass("ui-datepicker-next-hover")}).bind("mouseover",function(){if(!c._isDisabledDatepicker(a.inline?a.dpDiv.parent()[0]:a.input[0])){b(this).parents(".ui-datepicker-calendar").find("a").removeClass("ui-state-hover");b(this).addClass("ui-state-hover");this.className.indexOf("ui-datepicker-prev")!=-1&&b(this).addClass("ui-datepicker-prev-hover");this.className.indexOf("ui-datepicker-next")!=-1&&b(this).addClass("ui-datepicker-next-hover")}}).end().find("."+ +this._dayOverClass+" a").trigger("mouseover").end();h=this._getNumberOfMonths(a);i=h[1];i>1?a.dpDiv.addClass("ui-datepicker-multi-"+i).css("width",17*i+"em"):a.dpDiv.removeClass("ui-datepicker-multi-2 ui-datepicker-multi-3 ui-datepicker-multi-4").width("");a.dpDiv[(h[0]!=1||h[1]!=1?"add":"remove")+"Class"]("ui-datepicker-multi");a.dpDiv[(this._get(a,"isRTL")?"add":"remove")+"Class"]("ui-datepicker-rtl");a==b.datepicker._curInst&&b.datepicker._datepickerShowing&&a.input&&a.input.is(":visible")&&!a.input.is(":disabled")&& +a.input[0]!=document.activeElement&&a.input.focus();if(a.yearshtml){var j=a.yearshtml;setTimeout(function(){j===a.yearshtml&&a.dpDiv.find("select.ui-datepicker-year:first").replaceWith(a.yearshtml);j=a.yearshtml=null},0)}},_getBorders:function(a){var c=function(h){return{thin:1,medium:2,thick:3}[h]||h};return[parseFloat(c(a.css("border-left-width"))),parseFloat(c(a.css("border-top-width")))]},_checkOffset:function(a,c,h){var i=a.dpDiv.outerWidth(),j=a.dpDiv.outerHeight(),n=a.input?a.input.outerWidth(): +0,p=a.input?a.input.outerHeight():0,l=document.documentElement.clientWidth+b(document).scrollLeft(),k=document.documentElement.clientHeight+b(document).scrollTop();c.left-=this._get(a,"isRTL")?i-n:0;c.left-=h&&c.left==a.input.offset().left?b(document).scrollLeft():0;c.top-=h&&c.top==a.input.offset().top+p?b(document).scrollTop():0;c.left-=Math.min(c.left,c.left+i>l&&l>i?Math.abs(c.left+i-l):0);c.top-=Math.min(c.top,c.top+j>k&&k>j?Math.abs(j+p):0);return c},_findPos:function(a){for(var c=this._get(this._getInst(a), +"isRTL");a&&(a.type=="hidden"||a.nodeType!=1||b.expr.filters.hidden(a));)a=a[c?"previousSibling":"nextSibling"];a=b(a).offset();return[a.left,a.top]},_hideDatepicker:function(a){var c=this._curInst;if(!(!c||a&&c!=b.data(a,"datepicker")))if(this._datepickerShowing){a=this._get(c,"showAnim");var h=this._get(c,"duration"),i=function(){b.datepicker._tidyDialog(c);this._curInst=null};b.effects&&b.effects[a]?c.dpDiv.hide(a,b.datepicker._get(c,"showOptions"),h,i):c.dpDiv[a=="slideDown"?"slideUp":a=="fadeIn"? +"fadeOut":"hide"](a?h:null,i);a||i();if(a=this._get(c,"onClose"))a.apply(c.input?c.input[0]:null,[c.input?c.input.val():"",c]);this._datepickerShowing=false;this._lastInput=null;if(this._inDialog){this._dialogInput.css({position:"absolute",left:"0",top:"-100px"});if(b.blockUI){b.unblockUI();b("body").append(this.dpDiv)}}this._inDialog=false}},_tidyDialog:function(a){a.dpDiv.removeClass(this._dialogClass).unbind(".ui-datepicker-calendar")},_checkExternalClick:function(a){if(b.datepicker._curInst){a= +b(a.target);a[0].id!=b.datepicker._mainDivId&&a.parents("#"+b.datepicker._mainDivId).length==0&&!a.hasClass(b.datepicker.markerClassName)&&!a.hasClass(b.datepicker._triggerClass)&&b.datepicker._datepickerShowing&&!(b.datepicker._inDialog&&b.blockUI)&&b.datepicker._hideDatepicker()}},_adjustDate:function(a,c,h){a=b(a);var i=this._getInst(a[0]);if(!this._isDisabledDatepicker(a[0])){this._adjustInstDate(i,c+(h=="M"?this._get(i,"showCurrentAtPos"):0),h);this._updateDatepicker(i)}},_gotoToday:function(a){a= +b(a);var c=this._getInst(a[0]);if(this._get(c,"gotoCurrent")&&c.currentDay){c.selectedDay=c.currentDay;c.drawMonth=c.selectedMonth=c.currentMonth;c.drawYear=c.selectedYear=c.currentYear}else{var h=new Date;c.selectedDay=h.getDate();c.drawMonth=c.selectedMonth=h.getMonth();c.drawYear=c.selectedYear=h.getFullYear()}this._notifyChange(c);this._adjustDate(a)},_selectMonthYear:function(a,c,h){a=b(a);var i=this._getInst(a[0]);i._selectingMonthYear=false;i["selected"+(h=="M"?"Month":"Year")]=i["draw"+(h== +"M"?"Month":"Year")]=parseInt(c.options[c.selectedIndex].value,10);this._notifyChange(i);this._adjustDate(a)},_clickMonthYear:function(a){var c=this._getInst(b(a)[0]);c.input&&c._selectingMonthYear&&setTimeout(function(){c.input.focus()},0);c._selectingMonthYear=!c._selectingMonthYear},_selectDay:function(a,c,h,i){var j=b(a);if(!(b(i).hasClass(this._unselectableClass)||this._isDisabledDatepicker(j[0]))){j=this._getInst(j[0]);j.selectedDay=j.currentDay=b("a",i).html();j.selectedMonth=j.currentMonth= +c;j.selectedYear=j.currentYear=h;this._selectDate(a,this._formatDate(j,j.currentDay,j.currentMonth,j.currentYear))}},_clearDate:function(a){a=b(a);this._getInst(a[0]);this._selectDate(a,"")},_selectDate:function(a,c){a=this._getInst(b(a)[0]);c=c!=null?c:this._formatDate(a);a.input&&a.input.val(c);this._updateAlternate(a);var h=this._get(a,"onSelect");if(h)h.apply(a.input?a.input[0]:null,[c,a]);else a.input&&a.input.trigger("change");if(a.inline)this._updateDatepicker(a);else{this._hideDatepicker(); +this._lastInput=a.input[0];typeof a.input[0]!="object"&&a.input.focus();this._lastInput=null}},_updateAlternate:function(a){var c=this._get(a,"altField");if(c){var h=this._get(a,"altFormat")||this._get(a,"dateFormat"),i=this._getDate(a),j=this.formatDate(h,i,this._getFormatConfig(a));b(c).each(function(){b(this).val(j)})}},noWeekends:function(a){a=a.getDay();return[a>0&&a<6,""]},iso8601Week:function(a){a=new Date(a.getTime());a.setDate(a.getDate()+4-(a.getDay()||7));var c=a.getTime();a.setMonth(0); +a.setDate(1);return Math.floor(Math.round((c-a)/864E5)/7)+1},parseDate:function(a,c,h){if(a==null||c==null)throw"Invalid arguments";c=typeof c=="object"?c.toString():c+"";if(c=="")return null;var i=(h?h.shortYearCutoff:null)||this._defaults.shortYearCutoff;i=typeof i!="string"?i:(new Date).getFullYear()%100+parseInt(i,10);for(var j=(h?h.dayNamesShort:null)||this._defaults.dayNamesShort,n=(h?h.dayNames:null)||this._defaults.dayNames,p=(h?h.monthNamesShort:null)||this._defaults.monthNamesShort,l=(h? +h.monthNames:null)||this._defaults.monthNames,k=h=-1,m=-1,o=-1,q=false,s=function(x){(x=y+1-1){k=1;m=o;do{i=this._getDaysInMonth(h,k-1);if(m<=i)break;k++;m-=i}while(1)}B=this._daylightSavingAdjust(new Date(h,k-1,m));if(B.getFullYear()!=h||B.getMonth()+1!=k||B.getDate()!=m)throw"Invalid date";return B},ATOM:"yy-mm-dd",COOKIE:"D, dd M yy",ISO_8601:"yy-mm-dd",RFC_822:"D, d M y",RFC_850:"DD, dd-M-y", +RFC_1036:"D, d M y",RFC_1123:"D, d M yy",RFC_2822:"D, d M yy",RSS:"D, d M y",TICKS:"!",TIMESTAMP:"@",W3C:"yy-mm-dd",_ticksTo1970:(718685+Math.floor(492.5)-Math.floor(19.7)+Math.floor(4.925))*24*60*60*1E7,formatDate:function(a,c,h){if(!c)return"";var i=(h?h.dayNamesShort:null)||this._defaults.dayNamesShort,j=(h?h.dayNames:null)||this._defaults.dayNames,n=(h?h.monthNamesShort:null)||this._defaults.monthNamesShort;h=(h?h.monthNames:null)||this._defaults.monthNames;var p=function(s){(s=q+112?a.getHours()+2:0);return a},_setDate:function(a,c,h){var i=!c,j=a.selectedMonth,n=a.selectedYear;c=this._restrictMinMax(a,this._determineDate(a,c,new Date));a.selectedDay= +a.currentDay=c.getDate();a.drawMonth=a.selectedMonth=a.currentMonth=c.getMonth();a.drawYear=a.selectedYear=a.currentYear=c.getFullYear();if((j!=a.selectedMonth||n!=a.selectedYear)&&!h)this._notifyChange(a);this._adjustInstDate(a);if(a.input)a.input.val(i?"":this._formatDate(a))},_getDate:function(a){return!a.currentYear||a.input&&a.input.val()==""?null:this._daylightSavingAdjust(new Date(a.currentYear,a.currentMonth,a.currentDay))},_generateHTML:function(a){var c=new Date;c=this._daylightSavingAdjust(new Date(c.getFullYear(), +c.getMonth(),c.getDate()));var h=this._get(a,"isRTL"),i=this._get(a,"showButtonPanel"),j=this._get(a,"hideIfNoPrevNext"),n=this._get(a,"navigationAsDateFormat"),p=this._getNumberOfMonths(a),l=this._get(a,"showCurrentAtPos"),k=this._get(a,"stepMonths"),m=p[0]!=1||p[1]!=1,o=this._daylightSavingAdjust(!a.currentDay?new Date(9999,9,9):new Date(a.currentYear,a.currentMonth,a.currentDay)),q=this._getMinMaxDate(a,"min"),s=this._getMinMaxDate(a,"max");l=a.drawMonth-l;var r=a.drawYear;if(l<0){l+=12;r--}if(s){var u= +this._daylightSavingAdjust(new Date(s.getFullYear(),s.getMonth()-p[0]*p[1]+1,s.getDate()));for(u=q&&uu;){l--;if(l<0){l=11;r--}}}a.drawMonth=l;a.drawYear=r;u=this._get(a,"prevText");u=!n?u:this.formatDate(u,this._daylightSavingAdjust(new Date(r,l-k,1)),this._getFormatConfig(a));u=this._canAdjustMonth(a,-1,r,l)?''+u+"":j?"":''+u+"";var v=this._get(a,"nextText");v=!n?v:this.formatDate(v,this._daylightSavingAdjust(new Date(r,l+k,1)),this._getFormatConfig(a));j=this._canAdjustMonth(a,+1,r,l)?''+v+"":j?"":''+v+"";k=this._get(a,"currentText");v=this._get(a,"gotoCurrent")&&a.currentDay?o:c;k=!n?k:this.formatDate(k,v,this._getFormatConfig(a));n=!a.inline?'":"";i=i?'
      '+(h?n:"")+(this._isInRange(a,v)?'":"")+(h?"":n)+"
      ":"";n=parseInt(this._get(a,"firstDay"),10);n=isNaN(n)?0:n;k=this._get(a,"showWeek");v=this._get(a,"dayNames");this._get(a,"dayNamesShort");var w=this._get(a,"dayNamesMin"),y= +this._get(a,"monthNames"),B=this._get(a,"monthNamesShort"),x=this._get(a,"beforeShowDay"),C=this._get(a,"showOtherMonths"),J=this._get(a,"selectOtherMonths");this._get(a,"calculateWeek");for(var M=this._getDefaultDate(a),K="",G=0;G1)switch(H){case 0:D+=" ui-datepicker-group-first";A=" ui-corner-"+(h?"right":"left");break;case p[1]- +1:D+=" ui-datepicker-group-last";A=" ui-corner-"+(h?"left":"right");break;default:D+=" ui-datepicker-group-middle";A="";break}D+='">'}D+='
      '+(/all|left/.test(A)&&G==0?h?j:u:"")+(/all|right/.test(A)&&G==0?h?u:j:"")+this._generateMonthYearHeader(a,l,r,q,s,G>0||H>0,y,B)+'
      ';var E=k?'":"";for(A=0;A<7;A++){var z= +(A+n)%7;E+="=5?' class="ui-datepicker-week-end"':"")+'>'+w[z]+""}D+=E+"";E=this._getDaysInMonth(r,l);if(r==a.selectedYear&&l==a.selectedMonth)a.selectedDay=Math.min(a.selectedDay,E);A=(this._getFirstDayOfMonth(r,l)-n+7)%7;E=m?6:Math.ceil((A+E)/7);z=this._daylightSavingAdjust(new Date(r,l,1-A));for(var P=0;P";var Q=!k?"":'";for(A=0;A<7;A++){var I= +x?x.apply(a.input?a.input[0]:null,[z]):[true,""],F=z.getMonth()!=l,L=F&&!J||!I[0]||q&&zs;Q+='";z.setDate(z.getDate()+1);z=this._daylightSavingAdjust(z)}D+= +Q+""}l++;if(l>11){l=0;r++}D+="
      '+this._get(a,"weekHeader")+"
      '+this._get(a,"calculateWeek")(z)+""+(F&&!C?" ":L?''+z.getDate()+"":''+z.getDate()+"")+"
      "+(m?""+(p[0]>0&&H==p[1]-1?'
      ':""):"");N+=D}K+=N}K+=i+(b.browser.msie&&parseInt(b.browser.version,10)<7&&!a.inline?'':"");a._keyEvent=false;return K},_generateMonthYearHeader:function(a,c,h,i,j,n,p,l){var k=this._get(a,"changeMonth"),m=this._get(a,"changeYear"),o=this._get(a,"showMonthAfterYear"),q='
      ', +s="";if(n||!k)s+=''+p[c]+"";else{p=i&&i.getFullYear()==h;var r=j&&j.getFullYear()==h;s+='"}o||(q+=s+(n||!(k&& +m)?" ":""));a.yearshtml="";if(n||!m)q+=''+h+"";else{l=this._get(a,"yearRange").split(":");var v=(new Date).getFullYear();p=function(w){w=w.match(/c[+-].*/)?h+parseInt(w.substring(1),10):w.match(/[+-].*/)?v+parseInt(w,10):parseInt(w,10);return isNaN(w)?v:w};c=p(l[0]);l=Math.max(c,p(l[1]||""));c=i?Math.max(c,i.getFullYear()):c;l=j?Math.min(l,j.getFullYear()):l;for(a.yearshtml+='";if(b.browser.mozilla)q+='";else{q+=a.yearshtml;a.yearshtml=null}}q+=this._get(a,"yearSuffix");if(o)q+=(n||!(k&&m)?" ":"")+s;q+="
      ";return q},_adjustInstDate:function(a,c,h){var i= +a.drawYear+(h=="Y"?c:0),j=a.drawMonth+(h=="M"?c:0);c=Math.min(a.selectedDay,this._getDaysInMonth(i,j))+(h=="D"?c:0);i=this._restrictMinMax(a,this._daylightSavingAdjust(new Date(i,j,c)));a.selectedDay=i.getDate();a.drawMonth=a.selectedMonth=i.getMonth();a.drawYear=a.selectedYear=i.getFullYear();if(h=="M"||h=="Y")this._notifyChange(a)},_restrictMinMax:function(a,c){var h=this._getMinMaxDate(a,"min");a=this._getMinMaxDate(a,"max");c=h&&ca?a:c},_notifyChange:function(a){var c=this._get(a, +"onChangeMonthYear");if(c)c.apply(a.input?a.input[0]:null,[a.selectedYear,a.selectedMonth+1,a])},_getNumberOfMonths:function(a){a=this._get(a,"numberOfMonths");return a==null?[1,1]:typeof a=="number"?[1,a]:a},_getMinMaxDate:function(a,c){return this._determineDate(a,this._get(a,c+"Date"),null)},_getDaysInMonth:function(a,c){return 32-this._daylightSavingAdjust(new Date(a,c,32)).getDate()},_getFirstDayOfMonth:function(a,c){return(new Date(a,c,1)).getDay()},_canAdjustMonth:function(a,c,h,i){var j=this._getNumberOfMonths(a); +h=this._daylightSavingAdjust(new Date(h,i+(c<0?c:j[0]*j[1]),1));c<0&&h.setDate(this._getDaysInMonth(h.getFullYear(),h.getMonth()));return this._isInRange(a,h)},_isInRange:function(a,c){var h=this._getMinMaxDate(a,"min");a=this._getMinMaxDate(a,"max");return(!h||c.getTime()>=h.getTime())&&(!a||c.getTime()<=a.getTime())},_getFormatConfig:function(a){var c=this._get(a,"shortYearCutoff");c=typeof c!="string"?c:(new Date).getFullYear()%100+parseInt(c,10);return{shortYearCutoff:c,dayNamesShort:this._get(a, +"dayNamesShort"),dayNames:this._get(a,"dayNames"),monthNamesShort:this._get(a,"monthNamesShort"),monthNames:this._get(a,"monthNames")}},_formatDate:function(a,c,h,i){if(!c){a.currentDay=a.selectedDay;a.currentMonth=a.selectedMonth;a.currentYear=a.selectedYear}c=c?typeof c=="object"?c:this._daylightSavingAdjust(new Date(i,h,c)):this._daylightSavingAdjust(new Date(a.currentYear,a.currentMonth,a.currentDay));return this.formatDate(this._get(a,"dateFormat"),c,this._getFormatConfig(a))}});b.fn.datepicker= +function(a){if(!this.length)return this;if(!b.datepicker.initialized){b(document).mousedown(b.datepicker._checkExternalClick).find("body").append(b.datepicker.dpDiv);b.datepicker.initialized=true}var c=Array.prototype.slice.call(arguments,1);if(typeof a=="string"&&(a=="isDisabled"||a=="getDate"||a=="widget"))return b.datepicker["_"+a+"Datepicker"].apply(b.datepicker,[this[0]].concat(c));if(a=="option"&&arguments.length==2&&typeof arguments[1]=="string")return b.datepicker["_"+a+"Datepicker"].apply(b.datepicker, +[this[0]].concat(c));return this.each(function(){typeof a=="string"?b.datepicker["_"+a+"Datepicker"].apply(b.datepicker,[this].concat(c)):b.datepicker._attachDatepicker(this,a)})};b.datepicker=new e;b.datepicker.initialized=false;b.datepicker.uuid=(new Date).getTime();b.datepicker.version="1.8.11";window["DP_jQuery_"+f]=b})(jQuery); +(function(b,d){var e={buttons:true,height:true,maxHeight:true,maxWidth:true,minHeight:true,minWidth:true,width:true},g={maxHeight:true,maxWidth:true,minHeight:true,minWidth:true};b.widget("ui.dialog",{options:{autoOpen:true,buttons:{},closeOnEscape:true,closeText:"close",dialogClass:"",draggable:true,hide:null,height:"auto",maxHeight:false,maxWidth:false,minHeight:150,minWidth:150,modal:false,position:{my:"center",at:"center",collision:"fit",using:function(f){var a=b(this).css(f).offset().top;a<0&& +b(this).css("top",f.top-a)}},resizable:true,show:null,stack:true,title:"",width:300,zIndex:1E3},_create:function(){this.originalTitle=this.element.attr("title");if(typeof this.originalTitle!=="string")this.originalTitle="";this.options.title=this.options.title||this.originalTitle;var f=this,a=f.options,c=a.title||" ",h=b.ui.dialog.getTitleId(f.element),i=(f.uiDialog=b("
      ")).appendTo(document.body).hide().addClass("ui-dialog ui-widget ui-widget-content ui-corner-all "+a.dialogClass).css({zIndex:a.zIndex}).attr("tabIndex", +-1).css("outline",0).keydown(function(p){if(a.closeOnEscape&&p.keyCode&&p.keyCode===b.ui.keyCode.ESCAPE){f.close(p);p.preventDefault()}}).attr({role:"dialog","aria-labelledby":h}).mousedown(function(p){f.moveToTop(false,p)});f.element.show().removeAttr("title").addClass("ui-dialog-content ui-widget-content").appendTo(i);var j=(f.uiDialogTitlebar=b("
      ")).addClass("ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix").prependTo(i),n=b('').addClass("ui-dialog-titlebar-close ui-corner-all").attr("role", +"button").hover(function(){n.addClass("ui-state-hover")},function(){n.removeClass("ui-state-hover")}).focus(function(){n.addClass("ui-state-focus")}).blur(function(){n.removeClass("ui-state-focus")}).click(function(p){f.close(p);return false}).appendTo(j);(f.uiDialogTitlebarCloseText=b("")).addClass("ui-icon ui-icon-closethick").text(a.closeText).appendTo(n);b("").addClass("ui-dialog-title").attr("id",h).html(c).prependTo(j);if(b.isFunction(a.beforeclose)&&!b.isFunction(a.beforeClose))a.beforeClose= +a.beforeclose;j.find("*").add(j).disableSelection();a.draggable&&b.fn.draggable&&f._makeDraggable();a.resizable&&b.fn.resizable&&f._makeResizable();f._createButtons(a.buttons);f._isOpen=false;b.fn.bgiframe&&i.bgiframe()},_init:function(){this.options.autoOpen&&this.open()},destroy:function(){var f=this;f.overlay&&f.overlay.destroy();f.uiDialog.hide();f.element.unbind(".dialog").removeData("dialog").removeClass("ui-dialog-content ui-widget-content").hide().appendTo("body");f.uiDialog.remove();f.originalTitle&& +f.element.attr("title",f.originalTitle);return f},widget:function(){return this.uiDialog},close:function(f){var a=this,c,h;if(false!==a._trigger("beforeClose",f)){a.overlay&&a.overlay.destroy();a.uiDialog.unbind("keypress.ui-dialog");a._isOpen=false;if(a.options.hide)a.uiDialog.hide(a.options.hide,function(){a._trigger("close",f)});else{a.uiDialog.hide();a._trigger("close",f)}b.ui.dialog.overlay.resize();if(a.options.modal){c=0;b(".ui-dialog").each(function(){if(this!==a.uiDialog[0]){h=b(this).css("z-index"); +isNaN(h)||(c=Math.max(c,h))}});b.ui.dialog.maxZ=c}return a}},isOpen:function(){return this._isOpen},moveToTop:function(f,a){var c=this,h=c.options;if(h.modal&&!f||!h.stack&&!h.modal)return c._trigger("focus",a);if(h.zIndex>b.ui.dialog.maxZ)b.ui.dialog.maxZ=h.zIndex;if(c.overlay){b.ui.dialog.maxZ+=1;c.overlay.$el.css("z-index",b.ui.dialog.overlay.maxZ=b.ui.dialog.maxZ)}f={scrollTop:c.element.attr("scrollTop"),scrollLeft:c.element.attr("scrollLeft")};b.ui.dialog.maxZ+=1;c.uiDialog.css("z-index",b.ui.dialog.maxZ); +c.element.attr(f);c._trigger("focus",a);return c},open:function(){if(!this._isOpen){var f=this,a=f.options,c=f.uiDialog;f.overlay=a.modal?new b.ui.dialog.overlay(f):null;f._size();f._position(a.position);c.show(a.show);f.moveToTop(true);a.modal&&c.bind("keypress.ui-dialog",function(h){if(h.keyCode===b.ui.keyCode.TAB){var i=b(":tabbable",this),j=i.filter(":first");i=i.filter(":last");if(h.target===i[0]&&!h.shiftKey){j.focus(1);return false}else if(h.target===j[0]&&h.shiftKey){i.focus(1);return false}}}); +b(f.element.find(":tabbable").get().concat(c.find(".ui-dialog-buttonpane :tabbable").get().concat(c.get()))).eq(0).focus();f._isOpen=true;f._trigger("open");return f}},_createButtons:function(f){var a=this,c=false,h=b("
      ").addClass("ui-dialog-buttonpane ui-widget-content ui-helper-clearfix"),i=b("
      ").addClass("ui-dialog-buttonset").appendTo(h);a.uiDialog.find(".ui-dialog-buttonpane").remove();typeof f==="object"&&f!==null&&b.each(f,function(){return!(c=true)});if(c){b.each(f,function(j, +n){n=b.isFunction(n)?{click:n,text:j}:n;j=b('').attr(n,true).unbind("click").click(function(){n.click.apply(a.element[0],arguments)}).appendTo(i);b.fn.button&&j.button()});h.appendTo(a.uiDialog)}},_makeDraggable:function(){function f(j){return{position:j.position,offset:j.offset}}var a=this,c=a.options,h=b(document),i;a.uiDialog.draggable({cancel:".ui-dialog-content, .ui-dialog-titlebar-close",handle:".ui-dialog-titlebar",containment:"document",start:function(j,n){i= +c.height==="auto"?"auto":b(this).height();b(this).height(b(this).height()).addClass("ui-dialog-dragging");a._trigger("dragStart",j,f(n))},drag:function(j,n){a._trigger("drag",j,f(n))},stop:function(j,n){c.position=[n.position.left-h.scrollLeft(),n.position.top-h.scrollTop()];b(this).removeClass("ui-dialog-dragging").height(i);a._trigger("dragStop",j,f(n));b.ui.dialog.overlay.resize()}})},_makeResizable:function(f){function a(j){return{originalPosition:j.originalPosition,originalSize:j.originalSize, +position:j.position,size:j.size}}f=f===d?this.options.resizable:f;var c=this,h=c.options,i=c.uiDialog.css("position");f=typeof f==="string"?f:"n,e,s,w,se,sw,ne,nw";c.uiDialog.resizable({cancel:".ui-dialog-content",containment:"document",alsoResize:c.element,maxWidth:h.maxWidth,maxHeight:h.maxHeight,minWidth:h.minWidth,minHeight:c._minHeight(),handles:f,start:function(j,n){b(this).addClass("ui-dialog-resizing");c._trigger("resizeStart",j,a(n))},resize:function(j,n){c._trigger("resize",j,a(n))},stop:function(j, +n){b(this).removeClass("ui-dialog-resizing");h.height=b(this).height();h.width=b(this).width();c._trigger("resizeStop",j,a(n));b.ui.dialog.overlay.resize()}}).css("position",i).find(".ui-resizable-se").addClass("ui-icon ui-icon-grip-diagonal-se")},_minHeight:function(){var f=this.options;return f.height==="auto"?f.minHeight:Math.min(f.minHeight,f.height)},_position:function(f){var a=[],c=[0,0],h;if(f){if(typeof f==="string"||typeof f==="object"&&"0"in f){a=f.split?f.split(" "):[f[0],f[1]];if(a.length=== +1)a[1]=a[0];b.each(["left","top"],function(i,j){if(+a[i]===a[i]){c[i]=a[i];a[i]=j}});f={my:a.join(" "),at:a.join(" "),offset:c.join(" ")}}f=b.extend({},b.ui.dialog.prototype.options.position,f)}else f=b.ui.dialog.prototype.options.position;(h=this.uiDialog.is(":visible"))||this.uiDialog.show();this.uiDialog.css({top:0,left:0}).position(b.extend({of:window},f));h||this.uiDialog.hide()},_setOptions:function(f){var a=this,c={},h=false;b.each(f,function(i,j){a._setOption(i,j);if(i in e)h=true;if(i in +g)c[i]=j});h&&this._size();this.uiDialog.is(":data(resizable)")&&this.uiDialog.resizable("option",c)},_setOption:function(f,a){var c=this,h=c.uiDialog;switch(f){case "beforeclose":f="beforeClose";break;case "buttons":c._createButtons(a);break;case "closeText":c.uiDialogTitlebarCloseText.text(""+a);break;case "dialogClass":h.removeClass(c.options.dialogClass).addClass("ui-dialog ui-widget ui-widget-content ui-corner-all "+a);break;case "disabled":a?h.addClass("ui-dialog-disabled"):h.removeClass("ui-dialog-disabled"); +break;case "draggable":var i=h.is(":data(draggable)");i&&!a&&h.draggable("destroy");!i&&a&&c._makeDraggable();break;case "position":c._position(a);break;case "resizable":(i=h.is(":data(resizable)"))&&!a&&h.resizable("destroy");i&&typeof a==="string"&&h.resizable("option","handles",a);!i&&a!==false&&c._makeResizable(a);break;case "title":b(".ui-dialog-title",c.uiDialogTitlebar).html(""+(a||" "));break}b.Widget.prototype._setOption.apply(c,arguments)},_size:function(){var f=this.options,a,c,h= +this.uiDialog.is(":visible");this.element.show().css({width:"auto",minHeight:0,height:0});if(f.minWidth>f.width)f.width=f.minWidth;a=this.uiDialog.css({height:"auto",width:f.width}).height();c=Math.max(0,f.minHeight-a);if(f.height==="auto")if(b.support.minHeight)this.element.css({minHeight:c,height:"auto"});else{this.uiDialog.show();f=this.element.css("height","auto").height();h||this.uiDialog.hide();this.element.height(Math.max(f,c))}else this.element.height(Math.max(f.height-a,0));this.uiDialog.is(":data(resizable)")&& +this.uiDialog.resizable("option","minHeight",this._minHeight())}});b.extend(b.ui.dialog,{version:"1.8.11",uuid:0,maxZ:0,getTitleId:function(f){f=f.attr("id");if(!f){this.uuid+=1;f=this.uuid}return"ui-dialog-title-"+f},overlay:function(f){this.$el=b.ui.dialog.overlay.create(f)}});b.extend(b.ui.dialog.overlay,{instances:[],oldInstances:[],maxZ:0,events:b.map("focus,mousedown,mouseup,keydown,keypress,click".split(","),function(f){return f+".dialog-overlay"}).join(" "),create:function(f){if(this.instances.length=== +0){setTimeout(function(){b.ui.dialog.overlay.instances.length&&b(document).bind(b.ui.dialog.overlay.events,function(c){if(b(c.target).zIndex()").addClass("ui-widget-overlay")).appendTo(document.body).css({width:this.width(), +height:this.height()});b.fn.bgiframe&&a.bgiframe();this.instances.push(a);return a},destroy:function(f){var a=b.inArray(f,this.instances);a!=-1&&this.oldInstances.push(this.instances.splice(a,1)[0]);this.instances.length===0&&b([document,window]).unbind(".dialog-overlay");f.remove();var c=0;b.each(this.instances,function(){c=Math.max(c,this.css("z-index"))});this.maxZ=c},height:function(){var f,a;if(b.browser.msie&&b.browser.version<7){f=Math.max(document.documentElement.scrollHeight,document.body.scrollHeight); +a=Math.max(document.documentElement.offsetHeight,document.body.offsetHeight);return f0?a.left-h:Math.max(a.left-c.collisionPosition.left,a.left)},top:function(a,c){var h=b(window);h=c.collisionPosition.top+c.collisionHeight-h.height()-h.scrollTop();a.top=h>0?a.top-h:Math.max(a.top-c.collisionPosition.top,a.top)}},flip:{left:function(a,c){if(c.at[0]!=="center"){var h=b(window);h=c.collisionPosition.left+c.collisionWidth-h.width()-h.scrollLeft();var i=c.my[0]==="left"?-c.elemWidth:c.my[0]==="right"?c.elemWidth:0,j=c.at[0]==="left"?c.targetWidth:-c.targetWidth,n=-2*c.offset[0];a.left+= +c.collisionPosition.left<0?i+j+n:h>0?i+j+n:0}},top:function(a,c){if(c.at[1]!=="center"){var h=b(window);h=c.collisionPosition.top+c.collisionHeight-h.height()-h.scrollTop();var i=c.my[1]==="top"?-c.elemHeight:c.my[1]==="bottom"?c.elemHeight:0,j=c.at[1]==="top"?c.targetHeight:-c.targetHeight,n=-2*c.offset[1];a.top+=c.collisionPosition.top<0?i+j+n:h>0?i+j+n:0}}}};if(!b.offset.setOffset){b.offset.setOffset=function(a,c){if(/static/.test(b.curCSS(a,"position")))a.style.position="relative";var h=b(a), +i=h.offset(),j=parseInt(b.curCSS(a,"top",true),10)||0,n=parseInt(b.curCSS(a,"left",true),10)||0;i={top:c.top-i.top+j,left:c.left-i.left+n};"using"in c?c.using.call(a,i):h.css(i)};b.fn.offset=function(a){var c=this[0];if(!c||!c.ownerDocument)return null;if(a)return this.each(function(){b.offset.setOffset(this,a)});return f.call(this)}}})(jQuery); +(function(b,d){b.widget("ui.progressbar",{options:{value:0,max:100},min:0,_create:function(){this.element.addClass("ui-progressbar ui-widget ui-widget-content ui-corner-all").attr({role:"progressbar","aria-valuemin":this.min,"aria-valuemax":this.options.max,"aria-valuenow":this._value()});this.valueDiv=b("
      ").appendTo(this.element);this.oldValue=this._value();this._refreshValue()},destroy:function(){this.element.removeClass("ui-progressbar ui-widget ui-widget-content ui-corner-all").removeAttr("role").removeAttr("aria-valuemin").removeAttr("aria-valuemax").removeAttr("aria-valuenow"); +this.valueDiv.remove();b.Widget.prototype.destroy.apply(this,arguments)},value:function(e){if(e===d)return this._value();this._setOption("value",e);return this},_setOption:function(e,g){if(e==="value"){this.options.value=g;this._refreshValue();this._value()===this.options.max&&this._trigger("complete")}b.Widget.prototype._setOption.apply(this,arguments)},_value:function(){var e=this.options.value;if(typeof e!=="number")e=0;return Math.min(this.options.max,Math.max(this.min,e))},_percentage:function(){return 100* +this._value()/this.options.max},_refreshValue:function(){var e=this.value(),g=this._percentage();if(this.oldValue!==e){this.oldValue=e;this._trigger("change")}this.valueDiv.toggleClass("ui-corner-right",e===this.options.max).width(g.toFixed(0)+"%");this.element.attr("aria-valuenow",e)}});b.extend(b.ui.progressbar,{version:"1.8.11"})})(jQuery); +(function(b){b.widget("ui.slider",b.ui.mouse,{widgetEventPrefix:"slide",options:{animate:false,distance:0,max:100,min:0,orientation:"horizontal",range:false,step:1,value:0,values:null},_create:function(){var d=this,e=this.options;this._mouseSliding=this._keySliding=false;this._animateOff=true;this._handleIndex=null;this._detectOrientation();this._mouseInit();this.element.addClass("ui-slider ui-slider-"+this.orientation+" ui-widget ui-widget-content ui-corner-all");e.disabled&&this.element.addClass("ui-slider-disabled ui-disabled"); +this.range=b([]);if(e.range){if(e.range===true){this.range=b("
      ");if(!e.values)e.values=[this._valueMin(),this._valueMin()];if(e.values.length&&e.values.length!==2)e.values=[e.values[0],e.values[0]]}else this.range=b("
      ");this.range.appendTo(this.element).addClass("ui-slider-range");if(e.range==="min"||e.range==="max")this.range.addClass("ui-slider-range-"+e.range);this.range.addClass("ui-widget-header")}b(".ui-slider-handle",this.element).length===0&&b("").appendTo(this.element).addClass("ui-slider-handle"); +if(e.values&&e.values.length)for(;b(".ui-slider-handle",this.element).length").appendTo(this.element).addClass("ui-slider-handle");this.handles=b(".ui-slider-handle",this.element).addClass("ui-state-default ui-corner-all");this.handle=this.handles.eq(0);this.handles.add(this.range).filter("a").click(function(g){g.preventDefault()}).hover(function(){e.disabled||b(this).addClass("ui-state-hover")},function(){b(this).removeClass("ui-state-hover")}).focus(function(){if(e.disabled)b(this).blur(); +else{b(".ui-slider .ui-state-focus").removeClass("ui-state-focus");b(this).addClass("ui-state-focus")}}).blur(function(){b(this).removeClass("ui-state-focus")});this.handles.each(function(g){b(this).data("index.ui-slider-handle",g)});this.handles.keydown(function(g){var f=true,a=b(this).data("index.ui-slider-handle"),c,h,i;if(!d.options.disabled){switch(g.keyCode){case b.ui.keyCode.HOME:case b.ui.keyCode.END:case b.ui.keyCode.PAGE_UP:case b.ui.keyCode.PAGE_DOWN:case b.ui.keyCode.UP:case b.ui.keyCode.RIGHT:case b.ui.keyCode.DOWN:case b.ui.keyCode.LEFT:f= +false;if(!d._keySliding){d._keySliding=true;b(this).addClass("ui-state-active");c=d._start(g,a);if(c===false)return}break}i=d.options.step;c=d.options.values&&d.options.values.length?(h=d.values(a)):(h=d.value());switch(g.keyCode){case b.ui.keyCode.HOME:h=d._valueMin();break;case b.ui.keyCode.END:h=d._valueMax();break;case b.ui.keyCode.PAGE_UP:h=d._trimAlignValue(c+(d._valueMax()-d._valueMin())/5);break;case b.ui.keyCode.PAGE_DOWN:h=d._trimAlignValue(c-(d._valueMax()-d._valueMin())/5);break;case b.ui.keyCode.UP:case b.ui.keyCode.RIGHT:if(c=== +d._valueMax())return;h=d._trimAlignValue(c+i);break;case b.ui.keyCode.DOWN:case b.ui.keyCode.LEFT:if(c===d._valueMin())return;h=d._trimAlignValue(c-i);break}d._slide(g,a,h);return f}}).keyup(function(g){var f=b(this).data("index.ui-slider-handle");if(d._keySliding){d._keySliding=false;d._stop(g,f);d._change(g,f);b(this).removeClass("ui-state-active")}});this._refreshValue();this._animateOff=false},destroy:function(){this.handles.remove();this.range.remove();this.element.removeClass("ui-slider ui-slider-horizontal ui-slider-vertical ui-slider-disabled ui-widget ui-widget-content ui-corner-all").removeData("slider").unbind(".slider"); +this._mouseDestroy();return this},_mouseCapture:function(d){var e=this.options,g,f,a,c,h;if(e.disabled)return false;this.elementSize={width:this.element.outerWidth(),height:this.element.outerHeight()};this.elementOffset=this.element.offset();g=this._normValueFromMouse({x:d.pageX,y:d.pageY});f=this._valueMax()-this._valueMin()+1;c=this;this.handles.each(function(i){var j=Math.abs(g-c.values(i));if(f>j){f=j;a=b(this);h=i}});if(e.range===true&&this.values(1)===e.min){h+=1;a=b(this.handles[h])}if(this._start(d, +h)===false)return false;this._mouseSliding=true;c._handleIndex=h;a.addClass("ui-state-active").focus();e=a.offset();this._clickOffset=!b(d.target).parents().andSelf().is(".ui-slider-handle")?{left:0,top:0}:{left:d.pageX-e.left-a.width()/2,top:d.pageY-e.top-a.height()/2-(parseInt(a.css("borderTopWidth"),10)||0)-(parseInt(a.css("borderBottomWidth"),10)||0)+(parseInt(a.css("marginTop"),10)||0)};this.handles.hasClass("ui-state-hover")||this._slide(d,h,g);return this._animateOff=true},_mouseStart:function(){return true}, +_mouseDrag:function(d){var e=this._normValueFromMouse({x:d.pageX,y:d.pageY});this._slide(d,this._handleIndex,e);return false},_mouseStop:function(d){this.handles.removeClass("ui-state-active");this._mouseSliding=false;this._stop(d,this._handleIndex);this._change(d,this._handleIndex);this._clickOffset=this._handleIndex=null;return this._animateOff=false},_detectOrientation:function(){this.orientation=this.options.orientation==="vertical"?"vertical":"horizontal"},_normValueFromMouse:function(d){var e; +if(this.orientation==="horizontal"){e=this.elementSize.width;d=d.x-this.elementOffset.left-(this._clickOffset?this._clickOffset.left:0)}else{e=this.elementSize.height;d=d.y-this.elementOffset.top-(this._clickOffset?this._clickOffset.top:0)}e=d/e;if(e>1)e=1;if(e<0)e=0;if(this.orientation==="vertical")e=1-e;d=this._valueMax()-this._valueMin();return this._trimAlignValue(this._valueMin()+e*d)},_start:function(d,e){var g={handle:this.handles[e],value:this.value()};if(this.options.values&&this.options.values.length){g.value= +this.values(e);g.values=this.values()}return this._trigger("start",d,g)},_slide:function(d,e,g){var f;if(this.options.values&&this.options.values.length){f=this.values(e?0:1);if(this.options.values.length===2&&this.options.range===true&&(e===0&&g>f||e===1&&g1){this.options.values[d]=this._trimAlignValue(e);this._refreshValue();this._change(null,d)}if(arguments.length)if(b.isArray(arguments[0])){g=this.options.values;f=arguments[0];for(a=0;a=this._valueMax())return this._valueMax();var e=this.options.step>0?this.options.step:1,g=(d-this._valueMin())%e;alignValue=d-g;if(Math.abs(g)*2>=e)alignValue+=g>0?e:-e;return parseFloat(alignValue.toFixed(5))},_valueMin:function(){return this.options.min},_valueMax:function(){return this.options.max}, +_refreshValue:function(){var d=this.options.range,e=this.options,g=this,f=!this._animateOff?e.animate:false,a,c={},h,i,j,n;if(this.options.values&&this.options.values.length)this.handles.each(function(p){a=(g.values(p)-g._valueMin())/(g._valueMax()-g._valueMin())*100;c[g.orientation==="horizontal"?"left":"bottom"]=a+"%";b(this).stop(1,1)[f?"animate":"css"](c,e.animate);if(g.options.range===true)if(g.orientation==="horizontal"){if(p===0)g.range.stop(1,1)[f?"animate":"css"]({left:a+"%"},e.animate); +if(p===1)g.range[f?"animate":"css"]({width:a-h+"%"},{queue:false,duration:e.animate})}else{if(p===0)g.range.stop(1,1)[f?"animate":"css"]({bottom:a+"%"},e.animate);if(p===1)g.range[f?"animate":"css"]({height:a-h+"%"},{queue:false,duration:e.animate})}h=a});else{i=this.value();j=this._valueMin();n=this._valueMax();a=n!==j?(i-j)/(n-j)*100:0;c[g.orientation==="horizontal"?"left":"bottom"]=a+"%";this.handle.stop(1,1)[f?"animate":"css"](c,e.animate);if(d==="min"&&this.orientation==="horizontal")this.range.stop(1, +1)[f?"animate":"css"]({width:a+"%"},e.animate);if(d==="max"&&this.orientation==="horizontal")this.range[f?"animate":"css"]({width:100-a+"%"},{queue:false,duration:e.animate});if(d==="min"&&this.orientation==="vertical")this.range.stop(1,1)[f?"animate":"css"]({height:a+"%"},e.animate);if(d==="max"&&this.orientation==="vertical")this.range[f?"animate":"css"]({height:100-a+"%"},{queue:false,duration:e.animate})}}});b.extend(b.ui.slider,{version:"1.8.11"})})(jQuery); +(function(b,d){function e(){return++f}function g(){return++a}var f=0,a=0;b.widget("ui.tabs",{options:{add:null,ajaxOptions:null,cache:false,cookie:null,collapsible:false,disable:null,disabled:[],enable:null,event:"click",fx:null,idPrefix:"ui-tabs-",load:null,panelTemplate:"
      ",remove:null,select:null,show:null,spinner:"Loading…",tabTemplate:"
    • #{label}
      • "},_create:function(){this._tabify(true)},_setOption:function(c,h){if(c=="selected")this.options.collapsible&& +h==this.options.selected||this.select(h);else{this.options[c]=h;this._tabify()}},_tabId:function(c){return c.title&&c.title.replace(/\s/g,"_").replace(/[^\w\u00c0-\uFFFF-]/g,"")||this.options.idPrefix+e()},_sanitizeSelector:function(c){return c.replace(/:/g,"\\:")},_cookie:function(){var c=this.cookie||(this.cookie=this.options.cookie.name||"ui-tabs-"+g());return b.cookie.apply(null,[c].concat(b.makeArray(arguments)))},_ui:function(c,h){return{tab:c,panel:h,index:this.anchors.index(c)}},_cleanup:function(){this.lis.filter(".ui-state-processing").removeClass("ui-state-processing").find("span:data(label.tabs)").each(function(){var c= +b(this);c.html(c.data("label.tabs")).removeData("label.tabs")})},_tabify:function(c){function h(r,u){r.css("display","");!b.support.opacity&&u.opacity&&r[0].style.removeAttribute("filter")}var i=this,j=this.options,n=/^#.+/;this.list=this.element.find("ol,ul").eq(0);this.lis=b(" > li:has(a[href])",this.list);this.anchors=this.lis.map(function(){return b("a",this)[0]});this.panels=b([]);this.anchors.each(function(r,u){var v=b(u).attr("href"),w=v.split("#")[0],y;if(w&&(w===location.toString().split("#")[0]|| +(y=b("base")[0])&&w===y.href)){v=u.hash;u.href=v}if(n.test(v))i.panels=i.panels.add(i.element.find(i._sanitizeSelector(v)));else if(v&&v!=="#"){b.data(u,"href.tabs",v);b.data(u,"load.tabs",v.replace(/#.*$/,""));v=i._tabId(u);u.href="#"+v;u=i.element.find("#"+v);if(!u.length){u=b(j.panelTemplate).attr("id",v).addClass("ui-tabs-panel ui-widget-content ui-corner-bottom").insertAfter(i.panels[r-1]||i.list);u.data("destroy.tabs",true)}i.panels=i.panels.add(u)}else j.disabled.push(r)});if(c){this.element.addClass("ui-tabs ui-widget ui-widget-content ui-corner-all"); +this.list.addClass("ui-tabs-nav ui-helper-reset ui-helper-clearfix ui-widget-header ui-corner-all");this.lis.addClass("ui-state-default ui-corner-top");this.panels.addClass("ui-tabs-panel ui-widget-content ui-corner-bottom");if(j.selected===d){location.hash&&this.anchors.each(function(r,u){if(u.hash==location.hash){j.selected=r;return false}});if(typeof j.selected!=="number"&&j.cookie)j.selected=parseInt(i._cookie(),10);if(typeof j.selected!=="number"&&this.lis.filter(".ui-tabs-selected").length)j.selected= +this.lis.index(this.lis.filter(".ui-tabs-selected"));j.selected=j.selected||(this.lis.length?0:-1)}else if(j.selected===null)j.selected=-1;j.selected=j.selected>=0&&this.anchors[j.selected]||j.selected<0?j.selected:0;j.disabled=b.unique(j.disabled.concat(b.map(this.lis.filter(".ui-state-disabled"),function(r){return i.lis.index(r)}))).sort();b.inArray(j.selected,j.disabled)!=-1&&j.disabled.splice(b.inArray(j.selected,j.disabled),1);this.panels.addClass("ui-tabs-hide");this.lis.removeClass("ui-tabs-selected ui-state-active"); +if(j.selected>=0&&this.anchors.length){i.element.find(i._sanitizeSelector(i.anchors[j.selected].hash)).removeClass("ui-tabs-hide");this.lis.eq(j.selected).addClass("ui-tabs-selected ui-state-active");i.element.queue("tabs",function(){i._trigger("show",null,i._ui(i.anchors[j.selected],i.element.find(i._sanitizeSelector(i.anchors[j.selected].hash))[0]))});this.load(j.selected)}b(window).bind("unload",function(){i.lis.add(i.anchors).unbind(".tabs");i.lis=i.anchors=i.panels=null})}else j.selected=this.lis.index(this.lis.filter(".ui-tabs-selected")); +this.element[j.collapsible?"addClass":"removeClass"]("ui-tabs-collapsible");j.cookie&&this._cookie(j.selected,j.cookie);c=0;for(var p;p=this.lis[c];c++)b(p)[b.inArray(c,j.disabled)!=-1&&!b(p).hasClass("ui-tabs-selected")?"addClass":"removeClass"]("ui-state-disabled");j.cache===false&&this.anchors.removeData("cache.tabs");this.lis.add(this.anchors).unbind(".tabs");if(j.event!=="mouseover"){var l=function(r,u){u.is(":not(.ui-state-disabled)")&&u.addClass("ui-state-"+r)},k=function(r,u){u.removeClass("ui-state-"+ +r)};this.lis.bind("mouseover.tabs",function(){l("hover",b(this))});this.lis.bind("mouseout.tabs",function(){k("hover",b(this))});this.anchors.bind("focus.tabs",function(){l("focus",b(this).closest("li"))});this.anchors.bind("blur.tabs",function(){k("focus",b(this).closest("li"))})}var m,o;if(j.fx)if(b.isArray(j.fx)){m=j.fx[0];o=j.fx[1]}else m=o=j.fx;var q=o?function(r,u){b(r).closest("li").addClass("ui-tabs-selected ui-state-active");u.hide().removeClass("ui-tabs-hide").animate(o,o.duration||"normal", +function(){h(u,o);i._trigger("show",null,i._ui(r,u[0]))})}:function(r,u){b(r).closest("li").addClass("ui-tabs-selected ui-state-active");u.removeClass("ui-tabs-hide");i._trigger("show",null,i._ui(r,u[0]))},s=m?function(r,u){u.animate(m,m.duration||"normal",function(){i.lis.removeClass("ui-tabs-selected ui-state-active");u.addClass("ui-tabs-hide");h(u,m);i.element.dequeue("tabs")})}:function(r,u){i.lis.removeClass("ui-tabs-selected ui-state-active");u.addClass("ui-tabs-hide");i.element.dequeue("tabs")}; +this.anchors.bind(j.event+".tabs",function(){var r=this,u=b(r).closest("li"),v=i.panels.filter(":not(.ui-tabs-hide)"),w=i.element.find(i._sanitizeSelector(r.hash));if(u.hasClass("ui-tabs-selected")&&!j.collapsible||u.hasClass("ui-state-disabled")||u.hasClass("ui-state-processing")||i.panels.filter(":animated").length||i._trigger("select",null,i._ui(this,w[0]))===false){this.blur();return false}j.selected=i.anchors.index(this);i.abort();if(j.collapsible)if(u.hasClass("ui-tabs-selected")){j.selected= +-1;j.cookie&&i._cookie(j.selected,j.cookie);i.element.queue("tabs",function(){s(r,v)}).dequeue("tabs");this.blur();return false}else if(!v.length){j.cookie&&i._cookie(j.selected,j.cookie);i.element.queue("tabs",function(){q(r,w)});i.load(i.anchors.index(this));this.blur();return false}j.cookie&&i._cookie(j.selected,j.cookie);if(w.length){v.length&&i.element.queue("tabs",function(){s(r,v)});i.element.queue("tabs",function(){q(r,w)});i.load(i.anchors.index(this))}else throw"jQuery UI Tabs: Mismatching fragment identifier."; +b.browser.msie&&this.blur()});this.anchors.bind("click.tabs",function(){return false})},_getIndex:function(c){if(typeof c=="string")c=this.anchors.index(this.anchors.filter("[href$="+c+"]"));return c},destroy:function(){var c=this.options;this.abort();this.element.unbind(".tabs").removeClass("ui-tabs ui-widget ui-widget-content ui-corner-all ui-tabs-collapsible").removeData("tabs");this.list.removeClass("ui-tabs-nav ui-helper-reset ui-helper-clearfix ui-widget-header ui-corner-all");this.anchors.each(function(){var h= +b.data(this,"href.tabs");if(h)this.href=h;var i=b(this).unbind(".tabs");b.each(["href","load","cache"],function(j,n){i.removeData(n+".tabs")})});this.lis.unbind(".tabs").add(this.panels).each(function(){b.data(this,"destroy.tabs")?b(this).remove():b(this).removeClass("ui-state-default ui-corner-top ui-tabs-selected ui-state-active ui-state-hover ui-state-focus ui-state-disabled ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide")});c.cookie&&this._cookie(null,c.cookie);return this},add:function(c, +h,i){if(i===d)i=this.anchors.length;var j=this,n=this.options;h=b(n.tabTemplate.replace(/#\{href\}/g,c).replace(/#\{label\}/g,h));c=!c.indexOf("#")?c.replace("#",""):this._tabId(b("a",h)[0]);h.addClass("ui-state-default ui-corner-top").data("destroy.tabs",true);var p=j.element.find("#"+c);p.length||(p=b(n.panelTemplate).attr("id",c).data("destroy.tabs",true));p.addClass("ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide");if(i>=this.lis.length){h.appendTo(this.list);p.appendTo(this.list[0].parentNode)}else{h.insertBefore(this.lis[i]); +p.insertBefore(this.panels[i])}n.disabled=b.map(n.disabled,function(l){return l>=i?++l:l});this._tabify();if(this.anchors.length==1){n.selected=0;h.addClass("ui-tabs-selected ui-state-active");p.removeClass("ui-tabs-hide");this.element.queue("tabs",function(){j._trigger("show",null,j._ui(j.anchors[0],j.panels[0]))});this.load(0)}this._trigger("add",null,this._ui(this.anchors[i],this.panels[i]));return this},remove:function(c){c=this._getIndex(c);var h=this.options,i=this.lis.eq(c).remove(),j=this.panels.eq(c).remove(); +if(i.hasClass("ui-tabs-selected")&&this.anchors.length>1)this.select(c+(c+1=c?--n:n});this._tabify();this._trigger("remove",null,this._ui(i.find("a")[0],j[0]));return this},enable:function(c){c=this._getIndex(c);var h=this.options;if(b.inArray(c,h.disabled)!=-1){this.lis.eq(c).removeClass("ui-state-disabled");h.disabled=b.grep(h.disabled,function(i){return i!=c});this._trigger("enable",null, +this._ui(this.anchors[c],this.panels[c]));return this}},disable:function(c){c=this._getIndex(c);var h=this.options;if(c!=h.selected){this.lis.eq(c).addClass("ui-state-disabled");h.disabled.push(c);h.disabled.sort();this._trigger("disable",null,this._ui(this.anchors[c],this.panels[c]))}return this},select:function(c){c=this._getIndex(c);if(c==-1)if(this.options.collapsible&&this.options.selected!=-1)c=this.options.selected;else return this;this.anchors.eq(c).trigger(this.options.event+".tabs");return this}, +load:function(c){c=this._getIndex(c);var h=this,i=this.options,j=this.anchors.eq(c)[0],n=b.data(j,"load.tabs");this.abort();if(!n||this.element.queue("tabs").length!==0&&b.data(j,"cache.tabs"))this.element.dequeue("tabs");else{this.lis.eq(c).addClass("ui-state-processing");if(i.spinner){var p=b("span",j);p.data("label.tabs",p.html()).html(i.spinner)}this.xhr=b.ajax(b.extend({},i.ajaxOptions,{url:n,success:function(l,k){h.element.find(h._sanitizeSelector(j.hash)).html(l);h._cleanup();i.cache&&b.data(j, +"cache.tabs",true);h._trigger("load",null,h._ui(h.anchors[c],h.panels[c]));try{i.ajaxOptions.success(l,k)}catch(m){}},error:function(l,k){h._cleanup();h._trigger("load",null,h._ui(h.anchors[c],h.panels[c]));try{i.ajaxOptions.error(l,k,c,j)}catch(m){}}}));h.element.dequeue("tabs");return this}},abort:function(){this.element.queue([]);this.panels.stop(false,true);this.element.queue("tabs",this.element.queue("tabs").splice(-2,2));if(this.xhr){this.xhr.abort();delete this.xhr}this._cleanup();return this}, +url:function(c,h){this.anchors.eq(c).removeData("cache.tabs").data("load.tabs",h);return this},length:function(){return this.anchors.length}});b.extend(b.ui.tabs,{version:"1.8.11"});b.extend(b.ui.tabs.prototype,{rotation:null,rotate:function(c,h){var i=this,j=this.options,n=i._rotate||(i._rotate=function(p){clearTimeout(i.rotation);i.rotation=setTimeout(function(){var l=j.selected;i.select(++l$1<\/strong>'); + } + + function Autocomplete(el, options) { + this.el = $(el); + this.el.attr('autocomplete', 'off'); + this.suggestions = []; + this.data = []; + this.badQueries = []; + this.selectedIndex = -1; + this.currentValue = this.el.val(); + this.intervalId = 0; + this.cachedResponse = []; + this.onChangeInterval = null; + this.ignoreValueChange = false; + this.serviceUrl = options.serviceUrl; + this.isLocal = false; + this.options = { + autoSubmit: false, + minChars: 1, + maxHeight: 300, + deferRequestBy: 0, + width: 0, + highlight: true, + params: {}, + fnFormatResult: fnFormatResult, + delimiter: null, + zIndex: 9999 + }; + this.initialize(); + this.setOptions(options); + } + + $.fn.autocomplete = function(options) { + return new Autocomplete(this.get(0)||$(''), options); + }; + + + Autocomplete.prototype = { + + killerFn: null, + + initialize: function() { + + var me, uid, autocompleteElId; + me = this; + uid = Math.floor(Math.random()*0x100000).toString(16); + autocompleteElId = 'Autocomplete_' + uid; + + this.killerFn = function(e) { + if ($(e.target).parents('.autocomplete').size() === 0) { + me.killSuggestions(); + me.disableKillerFn(); + } + }; + + if (!this.options.width) { this.options.width = this.el.width(); } + this.mainContainerId = 'AutocompleteContainter_' + uid; + + $('
      ').appendTo('body'); + + this.container = $('#' + autocompleteElId); + this.fixPosition(); + if (window.opera) { + this.el.keypress(function(e) { me.onKeyPress(e); }); + } else { + this.el.keydown(function(e) { me.onKeyPress(e); }); + } + this.el.keyup(function(e) { me.onKeyUp(e); }); + this.el.blur(function() { me.enableKillerFn(); }); + this.el.focus(function() { me.fixPosition(); }); + }, + + setOptions: function(options){ + var o = this.options; + $.extend(o, options); + if(o.lookup){ + this.isLocal = true; + if($.isArray(o.lookup)){ o.lookup = { suggestions:o.lookup, data:[] }; } + } + $('#'+this.mainContainerId).css({ zIndex:o.zIndex }); + this.container.css({ maxHeight: o.maxHeight + 'px', width:o.width }); + }, + + clearCache: function(){ + this.cachedResponse = []; + this.badQueries = []; + }, + + disable: function(){ + this.disabled = true; + }, + + enable: function(){ + this.disabled = false; + }, + + fixPosition: function() { + var offset = this.el.offset(); + $('#' + this.mainContainerId).css({ top: (offset.top + this.el.innerHeight()) + 'px', left: offset.left + 'px' }); + }, + + enableKillerFn: function() { + var me = this; + $(document).bind('click', me.killerFn); + }, + + disableKillerFn: function() { + var me = this; + $(document).unbind('click', me.killerFn); + }, + + killSuggestions: function() { + var me = this; + this.stopKillSuggestions(); + this.intervalId = window.setInterval(function() { me.hide(); me.stopKillSuggestions(); }, 300); + }, + + stopKillSuggestions: function() { + window.clearInterval(this.intervalId); + }, + + onKeyPress: function(e) { + if (this.disabled || !this.enabled) { return; } + // return will exit the function + // and event will not be prevented + switch (e.keyCode) { + case 27: //KEY_ESC: + this.el.val(this.currentValue); + this.hide(); + break; + case 9: //KEY_TAB: + case 13: //KEY_RETURN: + if (this.selectedIndex === -1) { + this.hide(); + return; + } + this.select(this.selectedIndex); + if(e.keyCode === 9){ return; } + if(e.keyCode === 13){ this.el.parents("form").submit(); } // ADDED BY DBUTHAY + break; + case 38: //KEY_UP: + this.moveUp(); + break; + case 40: //KEY_DOWN: + this.moveDown(); + break; + default: + return; + } + e.stopImmediatePropagation(); + e.preventDefault(); + }, + + onKeyUp: function(e) { + if(this.disabled){ return; } + switch (e.keyCode) { + case 38: //KEY_UP: + case 40: //KEY_DOWN: + return; + } + clearInterval(this.onChangeInterval); + if (this.currentValue !== this.el.val()) { + if (this.options.deferRequestBy > 0) { + // Defer lookup in case when value changes very quickly: + var me = this; + this.onChangeInterval = setInterval(function() { me.onValueChange(); }, this.options.deferRequestBy); + } else { + this.onValueChange(); + } + } + }, + + onValueChange: function() { + clearInterval(this.onChangeInterval); + this.currentValue = this.el.val(); + var q = this.getQuery(this.currentValue); + this.selectedIndex = -1; + if (this.ignoreValueChange) { + this.ignoreValueChange = false; + return; + } + if (q === '' || q.length < this.options.minChars) { + this.hide(); + } else { + this.getSuggestions(q); + } + }, + + getQuery: function(val) { + var d, arr; + d = this.options.delimiter; + if (!d) { return $.trim(val); } + arr = val.split(d); + return $.trim(arr[arr.length - 1]); + }, + + getSuggestionsLocal: function(q) { + var ret, arr, len, val, i; + arr = this.options.lookup; + len = arr.suggestions.length; + ret = { suggestions:[], data:[] }; + q = q.toLowerCase(); + for(i=0; i< len; i++){ + val = arr.suggestions[i]; + if(val.toLowerCase().indexOf(q) === 0){ + ret.suggestions.push(val); + ret.data.push(arr.data[i]); + } + } + return ret; + }, + + getSuggestions: function(q) { + var cr, me; + cr = this.isLocal ? this.getSuggestionsLocal(q) : this.cachedResponse[q]; + if (cr && $.isArray(cr.suggestions)) { + this.suggestions = cr.suggestions; + this.data = cr.data; + this.suggest(); + } else if (!this.isBadQuery(q)) { + me = this; + me.options.params.query = q; + $.ajax( { + url: this.serviceUrl, + data: me.options.params, + success: function(txt) { me.processResponse(txt); }, + dataType: 'jsonp' + + } ); + } + }, + + isBadQuery: function(q) { + var i = this.badQueries.length; + while (i--) { + if (q.indexOf(this.badQueries[i]) === 0) { return true; } + } + return false; + }, + + hide: function() { + this.enabled = false; + this.selectedIndex = -1; + this.container.hide(); + }, + + suggest: function() { + if (this.suggestions.length === 0) { + this.hide(); + return; + } + + var me, len, div, f, v, i, s, mOver, mClick; + me = this; + len = this.suggestions.length; + f = this.options.fnFormatResult; + v = this.getQuery(this.currentValue); + mOver = function(xi) { return function() { me.activate(xi); }; }; + mClick = function(xi) { return function() { me.select(xi); }; }; + this.container.hide().empty(); + for (i = 0; i < len; i++) { + s = this.suggestions[i]; + div = $((me.selectedIndex === i ? '
      ' + f(s, this.data[i], v) + '
      '); + div.mouseover(mOver(i)); + div.click(mClick(i)); + this.container.append(div); + } + this.enabled = true; + this.container.show(); + }, + + processResponse: function(text) { + var response; + try { +// response = eval('(' + text + ')'); + response = text; // HACK + } catch (err) { return; } + if (!$.isArray(response.data)) { response.data = []; } + if(!this.options.noCache){ + this.cachedResponse[response.query] = response; + if (response.suggestions.length === 0) { this.badQueries.push(response.query); } + } + if (response.query === this.getQuery(this.currentValue)) { + this.suggestions = response.suggestions; + this.data = response.data; + this.suggest(); + } + }, + + activate: function(index) { + var divs, activeItem; + divs = this.container.children(); + // Clear previous selection: + if (this.selectedIndex !== -1 && divs.length > this.selectedIndex) { + $(divs.get(this.selectedIndex)).removeClass(); + } + this.selectedIndex = index; + if (this.selectedIndex !== -1 && divs.length > this.selectedIndex) { + activeItem = divs.get(this.selectedIndex); + $(activeItem).addClass('selected'); + } + return activeItem; + }, + + deactivate: function(div, index) { + div.className = ''; + if (this.selectedIndex === index) { this.selectedIndex = -1; } + }, + + select: function(i) { + var selectedValue, f; + selectedValue = this.suggestions[i]; + if (selectedValue) { + this.el.val(selectedValue); + if (this.options.autoSubmit) { + f = this.el.parents('form'); + if (f.length > 0) { f.get(0).submit(); } + } + this.ignoreValueChange = true; + this.hide(); + this.onSelect(i); + } + }, + + moveUp: function() { + if (this.selectedIndex === -1) { return; } + if (this.selectedIndex === 0) { + this.container.children().get(0).className = ''; + this.selectedIndex = -1; + this.el.val(this.currentValue); + return; + } + this.adjustScroll(this.selectedIndex - 1); + }, + + moveDown: function() { + if (this.selectedIndex === (this.suggestions.length - 1)) { return; } + this.adjustScroll(this.selectedIndex + 1); + }, + + adjustScroll: function(i) { + var activeItem, offsetTop, upperBound, lowerBound; + activeItem = this.activate(i); + offsetTop = activeItem.offsetTop; + upperBound = this.container.scrollTop(); + lowerBound = upperBound + this.options.maxHeight - 25; + if (offsetTop < upperBound) { + this.container.scrollTop(offsetTop); + } else if (offsetTop > lowerBound) { + this.container.scrollTop(offsetTop - this.options.maxHeight + 25); + } + this.el.val(this.getValue(this.suggestions[i])); + }, + + onSelect: function(i) { + var me, fn, s, d; + me = this; + fn = me.options.onSelect; + s = me.suggestions[i]; + d = me.data[i]; + me.el.val(me.getValue(s)); + if ($.isFunction(fn)) { fn(s, d, me.el); } + }, + + getValue: function(value){ + var del, currVal, arr, me; + me = this; + del = me.options.delimiter; + if (!del) { return value; } + currVal = me.currentValue; + arr = currVal.split(del); + if (arr.length === 1) { return value; } + return currVal.substr(0, currVal.length - arr[arr.length - 1].length) + value; + } + + }; + +}(jQuery)); diff --git a/storefront/static/js/jquery.cycle.lite.min.js b/storefront/static/js/jquery.cycle.lite.min.js new file mode 100644 index 0000000..3488ff3 --- /dev/null +++ b/storefront/static/js/jquery.cycle.lite.min.js @@ -0,0 +1,11 @@ +/* + * jQuery Cycle Lite Plugin + * http://malsup.com/jquery/cycle/lite/ + * Copyright (c) 2008 M. Alsup + * Version: 1.0 (06/08/2008) + * Dual licensed under the MIT and GPL licenses: + * http://www.opensource.org/licenses/mit-license.php + * http://www.gnu.org/licenses/gpl.html + * Requires: jQuery v1.2.3 or later + */ +;(function(D){var A="Lite-1.0";D.fn.cycle=function(E){return this.each(function(){E=E||{};if(this.cycleTimeout){clearTimeout(this.cycleTimeout)}this.cycleTimeout=0;this.cyclePause=0;var I=D(this);var J=E.slideExpr?D(E.slideExpr,this):I.children();var G=J.get();if(G.length<2){if(window.console&&window.console.log){window.console.log("terminating; too few slides: "+G.length)}return }var H=D.extend({},D.fn.cycle.defaults,E||{},D.metadata?I.metadata():D.meta?I.data():{});H.before=H.before?[H.before]:[];H.after=H.after?[H.after]:[];H.after.unshift(function(){H.busy=0});var F=this.className;H.width=parseInt((F.match(/w:(\d+)/)||[])[1])||H.width;H.height=parseInt((F.match(/h:(\d+)/)||[])[1])||H.height;H.timeout=parseInt((F.match(/t:(\d+)/)||[])[1])||H.timeout;if(I.css("position")=="static"){I.css("position","relative")}if(H.width){I.width(H.width)}if(H.height&&H.height!="auto"){I.height(H.height)}var K=0;J.css({position:"absolute",top:0,left:0}).hide().each(function(M){D(this).css("z-index",G.length-M)});D(G[K]).css("opacity",1).show();if(D.browser.msie){G[K].style.removeAttribute("filter")}if(H.fit&&H.width){J.width(H.width)}if(H.fit&&H.height&&H.height!="auto"){J.height(H.height)}if(H.pause){I.hover(function(){this.cyclePause=1},function(){this.cyclePause=0})}D.fn.cycle.transitions.fade(I,J,H);J.each(function(){var M=D(this);this.cycleH=(H.fit&&H.height)?H.height:M.height();this.cycleW=(H.fit&&H.width)?H.width:M.width()});J.not(":eq("+K+")").css({opacity:0});if(H.cssFirst){D(J[K]).css(H.cssFirst)}if(H.timeout){if(H.speed.constructor==String){H.speed={slow:600,fast:200}[H.speed]||400}if(!H.sync){H.speed=H.speed/2}while((H.timeout-H.speed)<250){H.timeout+=H.speed}}H.speedIn=H.speed;H.speedOut=H.speed;H.slideCount=G.length;H.currSlide=K;H.nextSlide=1;var L=J[K];if(H.before.length){H.before[0].apply(L,[L,L,H,true])}if(H.after.length>1){H.after[1].apply(L,[L,L,H,true])}if(H.click&&!H.next){H.next=H.click}if(H.next){D(H.next).bind("click",function(){return C(G,H,H.rev?-1:1)})}if(H.prev){D(H.prev).bind("click",function(){return C(G,H,H.rev?1:-1)})}if(H.timeout){this.cycleTimeout=setTimeout(function(){B(G,H,0,!H.rev)},H.timeout+(H.delay||0))}})};function B(J,E,I,K){if(E.busy){return }var H=J[0].parentNode,M=J[E.currSlide],L=J[E.nextSlide];if(H.cycleTimeout===0&&!I){return }if(I||!H.cyclePause){if(E.before.length){D.each(E.before,function(N,O){O.apply(L,[M,L,E,K])})}var F=function(){if(D.browser.msie){this.style.removeAttribute("filter")}D.each(E.after,function(N,O){O.apply(L,[M,L,E,K])})};if(E.nextSlide!=E.currSlide){E.busy=1;D.fn.cycle.custom(M,L,E,F)}var G=(E.nextSlide+1)==J.length;E.nextSlide=G?0:E.nextSlide+1;E.currSlide=G?J.length-1:E.nextSlide-1}if(E.timeout){H.cycleTimeout=setTimeout(function(){B(J,E,0,!E.rev)},E.timeout)}}function C(E,F,I){var H=E[0].parentNode,G=H.cycleTimeout;if(G){clearTimeout(G);H.cycleTimeout=0}F.nextSlide=F.currSlide+I;if(F.nextSlide<0){F.nextSlide=E.length-1}else{if(F.nextSlide>=E.length){F.nextSlide=0}}B(E,F,1,I>=0);return false}D.fn.cycle.custom=function(K,H,I,E){var J=D(K),G=D(H);G.css({opacity:0});var F=function(){G.animate({opacity:1},I.speedIn,I.easeIn,E)};J.animate({opacity:0},I.speedOut,I.easeOut,function(){J.css({display:"none"});if(!I.sync){F()}});if(I.sync){F()}};D.fn.cycle.transitions={fade:function(F,G,E){G.not(":eq(0)").css("opacity",0);E.before.push(function(){D(this).show()})}};D.fn.cycle.ver=function(){return A};D.fn.cycle.defaults={timeout:4000,speed:1000,next:null,prev:null,before:null,after:null,height:"auto",sync:1,fit:0,pause:0,delay:0,slideExpr:null}})(jQuery); diff --git a/storefront/static/js/jquery.flot.js b/storefront/static/js/jquery.flot.js new file mode 100644 index 0000000..6534a46 --- /dev/null +++ b/storefront/static/js/jquery.flot.js @@ -0,0 +1,2119 @@ +/* Javascript plotting library for jQuery, v. 0.6. + * + * Released under the MIT license by IOLA, December 2007. + * + */ + +// first an inline dependency, jquery.colorhelpers.js, we inline it here +// for convenience + +/* Plugin for jQuery for working with colors. + * + * Version 1.0. + * + * Inspiration from jQuery color animation plugin by John Resig. + * + * Released under the MIT license by Ole Laursen, October 2009. + * + * Examples: + * + * $.color.parse("#fff").scale('rgb', 0.25).add('a', -0.5).toString() + * var c = $.color.extract($("#mydiv"), 'background-color'); + * console.log(c.r, c.g, c.b, c.a); + * $.color.make(100, 50, 25, 0.4).toString() // returns "rgba(100,50,25,0.4)" + * + * Note that .scale() and .add() work in-place instead of returning + * new objects. + */ +(function(){jQuery.color={};jQuery.color.make=function(E,D,B,C){var F={};F.r=E||0;F.g=D||0;F.b=B||0;F.a=C!=null?C:1;F.add=function(I,H){for(var G=0;G=1){return"rgb("+[F.r,F.g,F.b].join(",")+")"}else{return"rgba("+[F.r,F.g,F.b,F.a].join(",")+")"}};F.normalize=function(){function G(I,J,H){return JH?H:J)}F.r=G(0,parseInt(F.r),255);F.g=G(0,parseInt(F.g),255);F.b=G(0,parseInt(F.b),255);F.a=G(0,F.a,1);return F};F.clone=function(){return jQuery.color.make(F.r,F.b,F.g,F.a)};return F.normalize()};jQuery.color.extract=function(C,B){var D;do{D=C.css(B).toLowerCase();if(D!=""&&D!="transparent"){break}C=C.parent()}while(!jQuery.nodeName(C.get(0),"body"));if(D=="rgba(0, 0, 0, 0)"){D="transparent"}return jQuery.color.parse(D)};jQuery.color.parse=function(E){var D,B=jQuery.color.make;if(D=/rgb\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*\)/.exec(E)){return B(parseInt(D[1],10),parseInt(D[2],10),parseInt(D[3],10))}if(D=/rgba\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]+(?:\.[0-9]+)?)\s*\)/.exec(E)){return B(parseInt(D[1],10),parseInt(D[2],10),parseInt(D[3],10),parseFloat(D[4]))}if(D=/rgb\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*\)/.exec(E)){return B(parseFloat(D[1])*2.55,parseFloat(D[2])*2.55,parseFloat(D[3])*2.55)}if(D=/rgba\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\s*\)/.exec(E)){return B(parseFloat(D[1])*2.55,parseFloat(D[2])*2.55,parseFloat(D[3])*2.55,parseFloat(D[4]))}if(D=/#([a-fA-F0-9]{2})([a-fA-F0-9]{2})([a-fA-F0-9]{2})/.exec(E)){return B(parseInt(D[1],16),parseInt(D[2],16),parseInt(D[3],16))}if(D=/#([a-fA-F0-9])([a-fA-F0-9])([a-fA-F0-9])/.exec(E)){return B(parseInt(D[1]+D[1],16),parseInt(D[2]+D[2],16),parseInt(D[3]+D[3],16))}var C=jQuery.trim(E).toLowerCase();if(C=="transparent"){return B(255,255,255,0)}else{D=A[C];return B(D[0],D[1],D[2])}};var A={aqua:[0,255,255],azure:[240,255,255],beige:[245,245,220],black:[0,0,0],blue:[0,0,255],brown:[165,42,42],cyan:[0,255,255],darkblue:[0,0,139],darkcyan:[0,139,139],darkgrey:[169,169,169],darkgreen:[0,100,0],darkkhaki:[189,183,107],darkmagenta:[139,0,139],darkolivegreen:[85,107,47],darkorange:[255,140,0],darkorchid:[153,50,204],darkred:[139,0,0],darksalmon:[233,150,122],darkviolet:[148,0,211],fuchsia:[255,0,255],gold:[255,215,0],green:[0,128,0],indigo:[75,0,130],khaki:[240,230,140],lightblue:[173,216,230],lightcyan:[224,255,255],lightgreen:[144,238,144],lightgrey:[211,211,211],lightpink:[255,182,193],lightyellow:[255,255,224],lime:[0,255,0],magenta:[255,0,255],maroon:[128,0,0],navy:[0,0,128],olive:[128,128,0],orange:[255,165,0],pink:[255,192,203],purple:[128,0,128],violet:[128,0,128],red:[255,0,0],silver:[192,192,192],white:[255,255,255],yellow:[255,255,0]}})(); + +// the actual Flot code +(function($) { + function Plot(placeholder, data_, options_, plugins) { + // data is on the form: + // [ series1, series2 ... ] + // where series is either just the data as [ [x1, y1], [x2, y2], ... ] + // or { data: [ [x1, y1], [x2, y2], ... ], label: "some label", ... } + + var series = [], + options = { + // the color theme used for graphs + colors: ["#edc240", "#afd8f8", "#cb4b4b", "#4da74d", "#9440ed"], + legend: { + show: true, + noColumns: 1, // number of colums in legend table + labelFormatter: null, // fn: string -> string + labelBoxBorderColor: "#ccc", // border color for the little label boxes + container: null, // container (as jQuery object) to put legend in, null means default on top of graph + position: "ne", // position of default legend container within plot + margin: 5, // distance from grid edge to default legend container within plot + backgroundColor: null, // null means auto-detect + backgroundOpacity: 0.85 // set to 0 to avoid background + }, + xaxis: { + mode: null, // null or "time" + transform: null, // null or f: number -> number to transform axis + inverseTransform: null, // if transform is set, this should be the inverse function + min: null, // min. value to show, null means set automatically + max: null, // max. value to show, null means set automatically + autoscaleMargin: null, // margin in % to add if auto-setting min/max + ticks: null, // either [1, 3] or [[1, "a"], 3] or (fn: axis info -> ticks) or app. number of ticks for auto-ticks + tickFormatter: null, // fn: number -> string + labelWidth: null, // size of tick labels in pixels + labelHeight: null, + + // mode specific options + tickDecimals: null, // no. of decimals, null means auto + tickSize: null, // number or [number, "unit"] + minTickSize: null, // number or [number, "unit"] + monthNames: null, // list of names of months + timeformat: null, // format string to use + twelveHourClock: false // 12 or 24 time in time mode + }, + yaxis: { + autoscaleMargin: 0.02 + }, + x2axis: { + autoscaleMargin: null + }, + y2axis: { + autoscaleMargin: 0.02 + }, + series: { + points: { + show: false, + radius: 3, + lineWidth: 2, // in pixels + fill: true, + fillColor: "#ffffff" + }, + lines: { + // we don't put in show: false so we can see + // whether lines were actively disabled + lineWidth: 2, // in pixels + fill: false, + fillColor: null, + steps: false + }, + bars: { + show: false, + lineWidth: 2, // in pixels + barWidth: 1, // in units of the x axis + fill: true, + fillColor: null, + align: "left", // or "center" + horizontal: false // when horizontal, left is now top + }, + shadowSize: 3 + }, + grid: { + show: true, + aboveData: false, + color: "#545454", // primary color used for outline and labels + backgroundColor: null, // null for transparent, else color + tickColor: "rgba(0,0,0,0.15)", // color used for the ticks + labelMargin: 5, // in pixels + borderWidth: 2, // in pixels + borderColor: null, // set if different from the grid color + markings: null, // array of ranges or fn: axes -> array of ranges + markingsColor: "#f4f4f4", + markingsLineWidth: 2, + // interactive stuff + clickable: false, + hoverable: false, + autoHighlight: true, // highlight in case mouse is near + mouseActiveRadius: 10 // how far the mouse can be away to activate an item + }, + hooks: {} + }, + canvas = null, // the canvas for the plot itself + overlay = null, // canvas for interactive stuff on top of plot + eventHolder = null, // jQuery object that events should be bound to + ctx = null, octx = null, + axes = { xaxis: {}, yaxis: {}, x2axis: {}, y2axis: {} }, + plotOffset = { left: 0, right: 0, top: 0, bottom: 0}, + canvasWidth = 0, canvasHeight = 0, + plotWidth = 0, plotHeight = 0, + hooks = { + processOptions: [], + processRawData: [], + processDatapoints: [], + draw: [], + bindEvents: [], + drawOverlay: [] + }, + plot = this; + + // public functions + plot.setData = setData; + plot.setupGrid = setupGrid; + plot.draw = draw; + plot.getPlaceholder = function() { return placeholder; }; + plot.getCanvas = function() { return canvas; }; + plot.getPlotOffset = function() { return plotOffset; }; + plot.width = function () { return plotWidth; }; + plot.height = function () { return plotHeight; }; + plot.offset = function () { + var o = eventHolder.offset(); + o.left += plotOffset.left; + o.top += plotOffset.top; + return o; + }; + plot.getData = function() { return series; }; + plot.getAxes = function() { return axes; }; + plot.getOptions = function() { return options; }; + plot.highlight = highlight; + plot.unhighlight = unhighlight; + plot.triggerRedrawOverlay = triggerRedrawOverlay; + plot.pointOffset = function(point) { + return { left: parseInt(axisSpecToRealAxis(point, "xaxis").p2c(+point.x) + plotOffset.left), + top: parseInt(axisSpecToRealAxis(point, "yaxis").p2c(+point.y) + plotOffset.top) }; + }; + + + // public attributes + plot.hooks = hooks; + + // initialize + initPlugins(plot); + parseOptions(options_); + constructCanvas(); + setData(data_); + setupGrid(); + draw(); + bindEvents(); + + + function executeHooks(hook, args) { + args = [plot].concat(args); + for (var i = 0; i < hook.length; ++i) + hook[i].apply(this, args); + } + + function initPlugins() { + for (var i = 0; i < plugins.length; ++i) { + var p = plugins[i]; + p.init(plot); + if (p.options) + $.extend(true, options, p.options); + } + } + + function parseOptions(opts) { + $.extend(true, options, opts); + if (options.grid.borderColor == null) + options.grid.borderColor = options.grid.color; + // backwards compatibility, to be removed in future + if (options.xaxis.noTicks && options.xaxis.ticks == null) + options.xaxis.ticks = options.xaxis.noTicks; + if (options.yaxis.noTicks && options.yaxis.ticks == null) + options.yaxis.ticks = options.yaxis.noTicks; + if (options.grid.coloredAreas) + options.grid.markings = options.grid.coloredAreas; + if (options.grid.coloredAreasColor) + options.grid.markingsColor = options.grid.coloredAreasColor; + if (options.lines) + $.extend(true, options.series.lines, options.lines); + if (options.points) + $.extend(true, options.series.points, options.points); + if (options.bars) + $.extend(true, options.series.bars, options.bars); + if (options.shadowSize) + options.series.shadowSize = options.shadowSize; + + for (var n in hooks) + if (options.hooks[n] && options.hooks[n].length) + hooks[n] = hooks[n].concat(options.hooks[n]); + + executeHooks(hooks.processOptions, [options]); + } + + function setData(d) { + series = parseData(d); + fillInSeriesOptions(); + processData(); + } + + function parseData(d) { + var res = []; + for (var i = 0; i < d.length; ++i) { + var s = $.extend(true, {}, options.series); + + if (d[i].data) { + s.data = d[i].data; // move the data instead of deep-copy + delete d[i].data; + + $.extend(true, s, d[i]); + + d[i].data = s.data; + } + else + s.data = d[i]; + res.push(s); + } + + return res; + } + + function axisSpecToRealAxis(obj, attr) { + var a = obj[attr]; + if (!a || a == 1) + return axes[attr]; + if (typeof a == "number") + return axes[attr.charAt(0) + a + attr.slice(1)]; + return a; // assume it's OK + } + + function fillInSeriesOptions() { + var i; + + // collect what we already got of colors + var neededColors = series.length, + usedColors = [], + assignedColors = []; + for (i = 0; i < series.length; ++i) { + var sc = series[i].color; + if (sc != null) { + --neededColors; + if (typeof sc == "number") + assignedColors.push(sc); + else + usedColors.push($.color.parse(series[i].color)); + } + } + + // we might need to generate more colors if higher indices + // are assigned + for (i = 0; i < assignedColors.length; ++i) { + neededColors = Math.max(neededColors, assignedColors[i] + 1); + } + + // produce colors as needed + var colors = [], variation = 0; + i = 0; + while (colors.length < neededColors) { + var c; + if (options.colors.length == i) // check degenerate case + c = $.color.make(100, 100, 100); + else + c = $.color.parse(options.colors[i]); + + // vary color if needed + var sign = variation % 2 == 1 ? -1 : 1; + c.scale('rgb', 1 + sign * Math.ceil(variation / 2) * 0.2) + + // FIXME: if we're getting to close to something else, + // we should probably skip this one + colors.push(c); + + ++i; + if (i >= options.colors.length) { + i = 0; + ++variation; + } + } + + // fill in the options + var colori = 0, s; + for (i = 0; i < series.length; ++i) { + s = series[i]; + + // assign colors + if (s.color == null) { + s.color = colors[colori].toString(); + ++colori; + } + else if (typeof s.color == "number") + s.color = colors[s.color].toString(); + + // turn on lines automatically in case nothing is set + if (s.lines.show == null) { + var v, show = true; + for (v in s) + if (s[v].show) { + show = false; + break; + } + if (show) + s.lines.show = true; + } + + // setup axes + s.xaxis = axisSpecToRealAxis(s, "xaxis"); + s.yaxis = axisSpecToRealAxis(s, "yaxis"); + } + } + + function processData() { + var topSentry = Number.POSITIVE_INFINITY, + bottomSentry = Number.NEGATIVE_INFINITY, + i, j, k, m, length, + s, points, ps, x, y, axis, val, f, p; + + for (axis in axes) { + axes[axis].datamin = topSentry; + axes[axis].datamax = bottomSentry; + axes[axis].used = false; + } + + function updateAxis(axis, min, max) { + if (min < axis.datamin) + axis.datamin = min; + if (max > axis.datamax) + axis.datamax = max; + } + + for (i = 0; i < series.length; ++i) { + s = series[i]; + s.datapoints = { points: [] }; + + executeHooks(hooks.processRawData, [ s, s.data, s.datapoints ]); + } + + // first pass: clean and copy data + for (i = 0; i < series.length; ++i) { + s = series[i]; + + var data = s.data, format = s.datapoints.format; + + if (!format) { + format = []; + // find out how to copy + format.push({ x: true, number: true, required: true }); + format.push({ y: true, number: true, required: true }); + + if (s.bars.show) + format.push({ y: true, number: true, required: false, defaultValue: 0 }); + + s.datapoints.format = format; + } + + if (s.datapoints.pointsize != null) + continue; // already filled in + + if (s.datapoints.pointsize == null) + s.datapoints.pointsize = format.length; + + ps = s.datapoints.pointsize; + points = s.datapoints.points; + + insertSteps = s.lines.show && s.lines.steps; + s.xaxis.used = s.yaxis.used = true; + + for (j = k = 0; j < data.length; ++j, k += ps) { + p = data[j]; + + var nullify = p == null; + if (!nullify) { + for (m = 0; m < ps; ++m) { + val = p[m]; + f = format[m]; + + if (f) { + if (f.number && val != null) { + val = +val; // convert to number + if (isNaN(val)) + val = null; + } + + if (val == null) { + if (f.required) + nullify = true; + + if (f.defaultValue != null) + val = f.defaultValue; + } + } + + points[k + m] = val; + } + } + + if (nullify) { + for (m = 0; m < ps; ++m) { + val = points[k + m]; + if (val != null) { + f = format[m]; + // extract min/max info + if (f.x) + updateAxis(s.xaxis, val, val); + if (f.y) + updateAxis(s.yaxis, val, val); + } + points[k + m] = null; + } + } + else { + // a little bit of line specific stuff that + // perhaps shouldn't be here, but lacking + // better means... + if (insertSteps && k > 0 + && points[k - ps] != null + && points[k - ps] != points[k] + && points[k - ps + 1] != points[k + 1]) { + // copy the point to make room for a middle point + for (m = 0; m < ps; ++m) + points[k + ps + m] = points[k + m]; + + // middle point has same y + points[k + 1] = points[k - ps + 1]; + + // we've added a point, better reflect that + k += ps; + } + } + } + } + + // give the hooks a chance to run + for (i = 0; i < series.length; ++i) { + s = series[i]; + + executeHooks(hooks.processDatapoints, [ s, s.datapoints]); + } + + // second pass: find datamax/datamin for auto-scaling + for (i = 0; i < series.length; ++i) { + s = series[i]; + points = s.datapoints.points, + ps = s.datapoints.pointsize; + + var xmin = topSentry, ymin = topSentry, + xmax = bottomSentry, ymax = bottomSentry; + + for (j = 0; j < points.length; j += ps) { + if (points[j] == null) + continue; + + for (m = 0; m < ps; ++m) { + val = points[j + m]; + f = format[m]; + if (!f) + continue; + + if (f.x) { + if (val < xmin) + xmin = val; + if (val > xmax) + xmax = val; + } + if (f.y) { + if (val < ymin) + ymin = val; + if (val > ymax) + ymax = val; + } + } + } + + if (s.bars.show) { + // make sure we got room for the bar on the dancing floor + var delta = s.bars.align == "left" ? 0 : -s.bars.barWidth/2; + if (s.bars.horizontal) { + ymin += delta; + ymax += delta + s.bars.barWidth; + } + else { + xmin += delta; + xmax += delta + s.bars.barWidth; + } + } + + updateAxis(s.xaxis, xmin, xmax); + updateAxis(s.yaxis, ymin, ymax); + } + + for (axis in axes) { + if (axes[axis].datamin == topSentry) + axes[axis].datamin = null; + if (axes[axis].datamax == bottomSentry) + axes[axis].datamax = null; + } + } + + function constructCanvas() { + function makeCanvas(width, height) { + var c = document.createElement('canvas'); + c.width = width; + c.height = height; + if ($.browser.msie) // excanvas hack + c = window.G_vmlCanvasManager.initElement(c); + return c; + } + + canvasWidth = placeholder.width(); + canvasHeight = placeholder.height(); + placeholder.html(""); // clear placeholder + if (placeholder.css("position") == 'static') + placeholder.css("position", "relative"); // for positioning labels and overlay + + if (canvasWidth <= 0 || canvasHeight <= 0) + throw "Invalid dimensions for plot, width = " + canvasWidth + ", height = " + canvasHeight; + + if ($.browser.msie) // excanvas hack + window.G_vmlCanvasManager.init_(document); // make sure everything is setup + + // the canvas + canvas = $(makeCanvas(canvasWidth, canvasHeight)).appendTo(placeholder).get(0); + ctx = canvas.getContext("2d"); + + // overlay canvas for interactive features + overlay = $(makeCanvas(canvasWidth, canvasHeight)).css({ position: 'absolute', left: 0, top: 0 }).appendTo(placeholder).get(0); + octx = overlay.getContext("2d"); + octx.stroke(); + } + + function bindEvents() { + // we include the canvas in the event holder too, because IE 7 + // sometimes has trouble with the stacking order + eventHolder = $([overlay, canvas]); + + // bind events + if (options.grid.hoverable) + eventHolder.mousemove(onMouseMove); + + if (options.grid.clickable) + eventHolder.click(onClick); + + executeHooks(hooks.bindEvents, [eventHolder]); + } + + function setupGrid() { + function setTransformationHelpers(axis, o) { + function identity(x) { return x; } + + var s, m, t = o.transform || identity, + it = o.inverseTransform; + + // add transformation helpers + if (axis == axes.xaxis || axis == axes.x2axis) { + // precompute how much the axis is scaling a point + // in canvas space + s = axis.scale = plotWidth / (t(axis.max) - t(axis.min)); + m = t(axis.min); + + // data point to canvas coordinate + if (t == identity) // slight optimization + axis.p2c = function (p) { return (p - m) * s; }; + else + axis.p2c = function (p) { return (t(p) - m) * s; }; + // canvas coordinate to data point + if (!it) + axis.c2p = function (c) { return m + c / s; }; + else + axis.c2p = function (c) { return it(m + c / s); }; + } + else { + s = axis.scale = plotHeight / (t(axis.max) - t(axis.min)); + m = t(axis.max); + + if (t == identity) + axis.p2c = function (p) { return (m - p) * s; }; + else + axis.p2c = function (p) { return (m - t(p)) * s; }; + if (!it) + axis.c2p = function (c) { return m - c / s; }; + else + axis.c2p = function (c) { return it(m - c / s); }; + } + } + + function measureLabels(axis, axisOptions) { + var i, labels = [], l; + + axis.labelWidth = axisOptions.labelWidth; + axis.labelHeight = axisOptions.labelHeight; + + if (axis == axes.xaxis || axis == axes.x2axis) { + // to avoid measuring the widths of the labels, we + // construct fixed-size boxes and put the labels inside + // them, we don't need the exact figures and the + // fixed-size box content is easy to center + if (axis.labelWidth == null) + axis.labelWidth = canvasWidth / (axis.ticks.length > 0 ? axis.ticks.length : 1); + + // measure x label heights + if (axis.labelHeight == null) { + labels = []; + for (i = 0; i < axis.ticks.length; ++i) { + l = axis.ticks[i].label; + if (l) + labels.push('
      ' + l + '
      '); + } + + if (labels.length > 0) { + var dummyDiv = $('
      ' + + labels.join("") + '
      ').appendTo(placeholder); + axis.labelHeight = dummyDiv.height(); + dummyDiv.remove(); + } + } + } + else if (axis.labelWidth == null || axis.labelHeight == null) { + // calculate y label dimensions + for (i = 0; i < axis.ticks.length; ++i) { + l = axis.ticks[i].label; + if (l) + labels.push('
      ' + l + '
      '); + } + + if (labels.length > 0) { + var dummyDiv = $('
      ' + + labels.join("") + '
      ').appendTo(placeholder); + if (axis.labelWidth == null) + axis.labelWidth = dummyDiv.width(); + if (axis.labelHeight == null) + axis.labelHeight = dummyDiv.find("div").height(); + dummyDiv.remove(); + } + + } + + if (axis.labelWidth == null) + axis.labelWidth = 0; + if (axis.labelHeight == null) + axis.labelHeight = 0; + } + + function setGridSpacing() { + // get the most space needed around the grid for things + // that may stick out + var maxOutset = options.grid.borderWidth; + for (i = 0; i < series.length; ++i) + maxOutset = Math.max(maxOutset, 2 * (series[i].points.radius + series[i].points.lineWidth/2)); + + plotOffset.left = plotOffset.right = plotOffset.top = plotOffset.bottom = maxOutset; + + var margin = options.grid.labelMargin + options.grid.borderWidth; + + if (axes.xaxis.labelHeight > 0) + plotOffset.bottom = Math.max(maxOutset, axes.xaxis.labelHeight + margin); + if (axes.yaxis.labelWidth > 0) + plotOffset.left = Math.max(maxOutset, axes.yaxis.labelWidth + margin); + if (axes.x2axis.labelHeight > 0) + plotOffset.top = Math.max(maxOutset, axes.x2axis.labelHeight + margin); + if (axes.y2axis.labelWidth > 0) + plotOffset.right = Math.max(maxOutset, axes.y2axis.labelWidth + margin); + + plotWidth = canvasWidth - plotOffset.left - plotOffset.right; + plotHeight = canvasHeight - plotOffset.bottom - plotOffset.top; + } + + var axis; + for (axis in axes) + setRange(axes[axis], options[axis]); + + if (options.grid.show) { + for (axis in axes) { + prepareTickGeneration(axes[axis], options[axis]); + setTicks(axes[axis], options[axis]); + measureLabels(axes[axis], options[axis]); + } + + setGridSpacing(); + } + else { + plotOffset.left = plotOffset.right = plotOffset.top = plotOffset.bottom = 0; + plotWidth = canvasWidth; + plotHeight = canvasHeight; + } + + for (axis in axes) + setTransformationHelpers(axes[axis], options[axis]); + + if (options.grid.show) + insertLabels(); + + insertLegend(); + } + + function setRange(axis, axisOptions) { + var min = +(axisOptions.min != null ? axisOptions.min : axis.datamin), + max = +(axisOptions.max != null ? axisOptions.max : axis.datamax), + delta = max - min; + + if (delta == 0.0) { + // degenerate case + var widen = max == 0 ? 1 : 0.01; + + if (axisOptions.min == null) + min -= widen; + // alway widen max if we couldn't widen min to ensure we + // don't fall into min == max which doesn't work + if (axisOptions.max == null || axisOptions.min != null) + max += widen; + } + else { + // consider autoscaling + var margin = axisOptions.autoscaleMargin; + if (margin != null) { + if (axisOptions.min == null) { + min -= delta * margin; + // make sure we don't go below zero if all values + // are positive + if (min < 0 && axis.datamin != null && axis.datamin >= 0) + min = 0; + } + if (axisOptions.max == null) { + max += delta * margin; + if (max > 0 && axis.datamax != null && axis.datamax <= 0) + max = 0; + } + } + } + axis.min = min; + axis.max = max; + } + + function prepareTickGeneration(axis, axisOptions) { + // estimate number of ticks + var noTicks; + if (typeof axisOptions.ticks == "number" && axisOptions.ticks > 0) + noTicks = axisOptions.ticks; + else if (axis == axes.xaxis || axis == axes.x2axis) + // heuristic based on the model a*sqrt(x) fitted to + // some reasonable data points + noTicks = 0.3 * Math.sqrt(canvasWidth); + else + noTicks = 0.3 * Math.sqrt(canvasHeight); + + var delta = (axis.max - axis.min) / noTicks, + size, generator, unit, formatter, i, magn, norm; + + if (axisOptions.mode == "time") { + // pretty handling of time + + // map of app. size of time units in milliseconds + var timeUnitSize = { + "second": 1000, + "minute": 60 * 1000, + "hour": 60 * 60 * 1000, + "day": 24 * 60 * 60 * 1000, + "month": 30 * 24 * 60 * 60 * 1000, + "year": 365.2425 * 24 * 60 * 60 * 1000 + }; + + + // the allowed tick sizes, after 1 year we use + // an integer algorithm + var spec = [ + [1, "second"], [2, "second"], [5, "second"], [10, "second"], + [30, "second"], + [1, "minute"], [2, "minute"], [5, "minute"], [10, "minute"], + [30, "minute"], + [1, "hour"], [2, "hour"], [4, "hour"], + [8, "hour"], [12, "hour"], + [1, "day"], [2, "day"], [3, "day"], + [0.25, "month"], [0.5, "month"], [1, "month"], + [2, "month"], [3, "month"], [6, "month"], + [1, "year"] + ]; + + var minSize = 0; + if (axisOptions.minTickSize != null) { + if (typeof axisOptions.tickSize == "number") + minSize = axisOptions.tickSize; + else + minSize = axisOptions.minTickSize[0] * timeUnitSize[axisOptions.minTickSize[1]]; + } + + for (i = 0; i < spec.length - 1; ++i) + if (delta < (spec[i][0] * timeUnitSize[spec[i][1]] + + spec[i + 1][0] * timeUnitSize[spec[i + 1][1]]) / 2 + && spec[i][0] * timeUnitSize[spec[i][1]] >= minSize) + break; + size = spec[i][0]; + unit = spec[i][1]; + + // special-case the possibility of several years + if (unit == "year") { + magn = Math.pow(10, Math.floor(Math.log(delta / timeUnitSize.year) / Math.LN10)); + norm = (delta / timeUnitSize.year) / magn; + if (norm < 1.5) + size = 1; + else if (norm < 3) + size = 2; + else if (norm < 7.5) + size = 5; + else + size = 10; + + size *= magn; + } + + if (axisOptions.tickSize) { + size = axisOptions.tickSize[0]; + unit = axisOptions.tickSize[1]; + } + + generator = function(axis) { + var ticks = [], + tickSize = axis.tickSize[0], unit = axis.tickSize[1], + d = new Date(axis.min); + + var step = tickSize * timeUnitSize[unit]; + + if (unit == "second") + d.setUTCSeconds(floorInBase(d.getUTCSeconds(), tickSize)); + if (unit == "minute") + d.setUTCMinutes(floorInBase(d.getUTCMinutes(), tickSize)); + if (unit == "hour") + d.setUTCHours(floorInBase(d.getUTCHours(), tickSize)); + if (unit == "month") + d.setUTCMonth(floorInBase(d.getUTCMonth(), tickSize)); + if (unit == "year") + d.setUTCFullYear(floorInBase(d.getUTCFullYear(), tickSize)); + + // reset smaller components + d.setUTCMilliseconds(0); + if (step >= timeUnitSize.minute) + d.setUTCSeconds(0); + if (step >= timeUnitSize.hour) + d.setUTCMinutes(0); + if (step >= timeUnitSize.day) + d.setUTCHours(0); + if (step >= timeUnitSize.day * 4) + d.setUTCDate(1); + if (step >= timeUnitSize.year) + d.setUTCMonth(0); + + + var carry = 0, v = Number.NaN, prev; + do { + prev = v; + v = d.getTime(); + ticks.push({ v: v, label: axis.tickFormatter(v, axis) }); + if (unit == "month") { + if (tickSize < 1) { + // a bit complicated - we'll divide the month + // up but we need to take care of fractions + // so we don't end up in the middle of a day + d.setUTCDate(1); + var start = d.getTime(); + d.setUTCMonth(d.getUTCMonth() + 1); + var end = d.getTime(); + d.setTime(v + carry * timeUnitSize.hour + (end - start) * tickSize); + carry = d.getUTCHours(); + d.setUTCHours(0); + } + else + d.setUTCMonth(d.getUTCMonth() + tickSize); + } + else if (unit == "year") { + d.setUTCFullYear(d.getUTCFullYear() + tickSize); + } + else + d.setTime(v + step); + } while (v < axis.max && v != prev); + + return ticks; + }; + + formatter = function (v, axis) { + var d = new Date(v); + + // first check global format + if (axisOptions.timeformat != null) + return $.plot.formatDate(d, axisOptions.timeformat, axisOptions.monthNames); + + var t = axis.tickSize[0] * timeUnitSize[axis.tickSize[1]]; + var span = axis.max - axis.min; + var suffix = (axisOptions.twelveHourClock) ? " %p" : ""; + + if (t < timeUnitSize.minute) + fmt = "%h:%M:%S" + suffix; + else if (t < timeUnitSize.day) { + if (span < 2 * timeUnitSize.day) + fmt = "%h:%M" + suffix; + else + fmt = "%b %d %h:%M" + suffix; + } + else if (t < timeUnitSize.month) + fmt = "%b %d"; + else if (t < timeUnitSize.year) { + if (span < timeUnitSize.year) + fmt = "%b"; + else + fmt = "%b %y"; + } + else + fmt = "%y"; + + return $.plot.formatDate(d, fmt, axisOptions.monthNames); + }; + } + else { + // pretty rounding of base-10 numbers + var maxDec = axisOptions.tickDecimals; + var dec = -Math.floor(Math.log(delta) / Math.LN10); + if (maxDec != null && dec > maxDec) + dec = maxDec; + + magn = Math.pow(10, -dec); + norm = delta / magn; // norm is between 1.0 and 10.0 + + if (norm < 1.5) + size = 1; + else if (norm < 3) { + size = 2; + // special case for 2.5, requires an extra decimal + if (norm > 2.25 && (maxDec == null || dec + 1 <= maxDec)) { + size = 2.5; + ++dec; + } + } + else if (norm < 7.5) + size = 5; + else + size = 10; + + size *= magn; + + if (axisOptions.minTickSize != null && size < axisOptions.minTickSize) + size = axisOptions.minTickSize; + + if (axisOptions.tickSize != null) + size = axisOptions.tickSize; + + axis.tickDecimals = Math.max(0, (maxDec != null) ? maxDec : dec); + + generator = function (axis) { + var ticks = []; + + // spew out all possible ticks + var start = floorInBase(axis.min, axis.tickSize), + i = 0, v = Number.NaN, prev; + do { + prev = v; + v = start + i * axis.tickSize; + ticks.push({ v: v, label: axis.tickFormatter(v, axis) }); + ++i; + } while (v < axis.max && v != prev); + return ticks; + }; + + formatter = function (v, axis) { + return v.toFixed(axis.tickDecimals); + }; + } + + axis.tickSize = unit ? [size, unit] : size; + axis.tickGenerator = generator; + if ($.isFunction(axisOptions.tickFormatter)) + axis.tickFormatter = function (v, axis) { return "" + axisOptions.tickFormatter(v, axis); }; + else + axis.tickFormatter = formatter; + } + + function setTicks(axis, axisOptions) { + axis.ticks = []; + + if (!axis.used) + return; + + if (axisOptions.ticks == null) + axis.ticks = axis.tickGenerator(axis); + else if (typeof axisOptions.ticks == "number") { + if (axisOptions.ticks > 0) + axis.ticks = axis.tickGenerator(axis); + } + else if (axisOptions.ticks) { + var ticks = axisOptions.ticks; + + if ($.isFunction(ticks)) + // generate the ticks + ticks = ticks({ min: axis.min, max: axis.max }); + + // clean up the user-supplied ticks, copy them over + var i, v; + for (i = 0; i < ticks.length; ++i) { + var label = null; + var t = ticks[i]; + if (typeof t == "object") { + v = t[0]; + if (t.length > 1) + label = t[1]; + } + else + v = t; + if (label == null) + label = axis.tickFormatter(v, axis); + axis.ticks[i] = { v: v, label: label }; + } + } + + if (axisOptions.autoscaleMargin != null && axis.ticks.length > 0) { + // snap to ticks + if (axisOptions.min == null) + axis.min = Math.min(axis.min, axis.ticks[0].v); + if (axisOptions.max == null && axis.ticks.length > 1) + axis.max = Math.max(axis.max, axis.ticks[axis.ticks.length - 1].v); + } + } + + function draw() { + ctx.clearRect(0, 0, canvasWidth, canvasHeight); + + var grid = options.grid; + + if (grid.show && !grid.aboveData) + drawGrid(); + + for (var i = 0; i < series.length; ++i) + drawSeries(series[i]); + + executeHooks(hooks.draw, [ctx]); + + if (grid.show && grid.aboveData) + drawGrid(); + } + + function extractRange(ranges, coord) { + var firstAxis = coord + "axis", + secondaryAxis = coord + "2axis", + axis, from, to, reverse; + + if (ranges[firstAxis]) { + axis = axes[firstAxis]; + from = ranges[firstAxis].from; + to = ranges[firstAxis].to; + } + else if (ranges[secondaryAxis]) { + axis = axes[secondaryAxis]; + from = ranges[secondaryAxis].from; + to = ranges[secondaryAxis].to; + } + else { + // backwards-compat stuff - to be removed in future + axis = axes[firstAxis]; + from = ranges[coord + "1"]; + to = ranges[coord + "2"]; + } + + // auto-reverse as an added bonus + if (from != null && to != null && from > to) + return { from: to, to: from, axis: axis }; + + return { from: from, to: to, axis: axis }; + } + + function drawGrid() { + var i; + + ctx.save(); + ctx.translate(plotOffset.left, plotOffset.top); + + // draw background, if any + if (options.grid.backgroundColor) { + ctx.fillStyle = getColorOrGradient(options.grid.backgroundColor, plotHeight, 0, "rgba(255, 255, 255, 0)"); + ctx.fillRect(0, 0, plotWidth, plotHeight); + } + + // draw markings + var markings = options.grid.markings; + if (markings) { + if ($.isFunction(markings)) + // xmin etc. are backwards-compatible, to be removed in future + markings = markings({ xmin: axes.xaxis.min, xmax: axes.xaxis.max, ymin: axes.yaxis.min, ymax: axes.yaxis.max, xaxis: axes.xaxis, yaxis: axes.yaxis, x2axis: axes.x2axis, y2axis: axes.y2axis }); + + for (i = 0; i < markings.length; ++i) { + var m = markings[i], + xrange = extractRange(m, "x"), + yrange = extractRange(m, "y"); + + // fill in missing + if (xrange.from == null) + xrange.from = xrange.axis.min; + if (xrange.to == null) + xrange.to = xrange.axis.max; + if (yrange.from == null) + yrange.from = yrange.axis.min; + if (yrange.to == null) + yrange.to = yrange.axis.max; + + // clip + if (xrange.to < xrange.axis.min || xrange.from > xrange.axis.max || + yrange.to < yrange.axis.min || yrange.from > yrange.axis.max) + continue; + + xrange.from = Math.max(xrange.from, xrange.axis.min); + xrange.to = Math.min(xrange.to, xrange.axis.max); + yrange.from = Math.max(yrange.from, yrange.axis.min); + yrange.to = Math.min(yrange.to, yrange.axis.max); + + if (xrange.from == xrange.to && yrange.from == yrange.to) + continue; + + // then draw + xrange.from = xrange.axis.p2c(xrange.from); + xrange.to = xrange.axis.p2c(xrange.to); + yrange.from = yrange.axis.p2c(yrange.from); + yrange.to = yrange.axis.p2c(yrange.to); + + if (xrange.from == xrange.to || yrange.from == yrange.to) { + // draw line + ctx.beginPath(); + ctx.strokeStyle = m.color || options.grid.markingsColor; + ctx.lineWidth = m.lineWidth || options.grid.markingsLineWidth; + //ctx.moveTo(Math.floor(xrange.from), yrange.from); + //ctx.lineTo(Math.floor(xrange.to), yrange.to); + ctx.moveTo(xrange.from, yrange.from); + ctx.lineTo(xrange.to, yrange.to); + ctx.stroke(); + } + else { + // fill area + ctx.fillStyle = m.color || options.grid.markingsColor; + ctx.fillRect(xrange.from, yrange.to, + xrange.to - xrange.from, + yrange.from - yrange.to); + } + } + } + + // draw the inner grid + ctx.lineWidth = 1; + ctx.strokeStyle = options.grid.tickColor; + ctx.beginPath(); + var v, axis = axes.xaxis; + for (i = 0; i < axis.ticks.length; ++i) { + v = axis.ticks[i].v; + if (v <= axis.min || v >= axes.xaxis.max) + continue; // skip those lying on the axes + + ctx.moveTo(Math.floor(axis.p2c(v)) + ctx.lineWidth/2, 0); + ctx.lineTo(Math.floor(axis.p2c(v)) + ctx.lineWidth/2, plotHeight); + } + + axis = axes.yaxis; + for (i = 0; i < axis.ticks.length; ++i) { + v = axis.ticks[i].v; + if (v <= axis.min || v >= axis.max) + continue; + + ctx.moveTo(0, Math.floor(axis.p2c(v)) + ctx.lineWidth/2); + ctx.lineTo(plotWidth, Math.floor(axis.p2c(v)) + ctx.lineWidth/2); + } + + axis = axes.x2axis; + for (i = 0; i < axis.ticks.length; ++i) { + v = axis.ticks[i].v; + if (v <= axis.min || v >= axis.max) + continue; + + ctx.moveTo(Math.floor(axis.p2c(v)) + ctx.lineWidth/2, -5); + ctx.lineTo(Math.floor(axis.p2c(v)) + ctx.lineWidth/2, 5); + } + + axis = axes.y2axis; + for (i = 0; i < axis.ticks.length; ++i) { + v = axis.ticks[i].v; + if (v <= axis.min || v >= axis.max) + continue; + + ctx.moveTo(plotWidth-5, Math.floor(axis.p2c(v)) + ctx.lineWidth/2); + ctx.lineTo(plotWidth+5, Math.floor(axis.p2c(v)) + ctx.lineWidth/2); + } + + ctx.stroke(); + + if (options.grid.borderWidth) { + // draw border + var bw = options.grid.borderWidth; + ctx.lineWidth = bw; + ctx.strokeStyle = options.grid.borderColor; + ctx.strokeRect(-bw/2, -bw/2, plotWidth + bw, plotHeight + bw); + } + + ctx.restore(); + } + + function insertLabels() { + placeholder.find(".tickLabels").remove(); + + var html = ['
      ']; + + function addLabels(axis, labelGenerator) { + for (var i = 0; i < axis.ticks.length; ++i) { + var tick = axis.ticks[i]; + if (!tick.label || tick.v < axis.min || tick.v > axis.max) + continue; + html.push(labelGenerator(tick, axis)); + } + } + + var margin = options.grid.labelMargin + options.grid.borderWidth; + + addLabels(axes.xaxis, function (tick, axis) { + return '
      ' + tick.label + "
      "; + }); + + + addLabels(axes.yaxis, function (tick, axis) { + return '
      ' + tick.label + "
      "; + }); + + addLabels(axes.x2axis, function (tick, axis) { + return '
      ' + tick.label + "
      "; + }); + + addLabels(axes.y2axis, function (tick, axis) { + return '
      ' + tick.label + "
      "; + }); + + html.push('
      '); + + placeholder.append(html.join("")); + } + + function drawSeries(series) { + if (series.lines.show) + drawSeriesLines(series); + if (series.bars.show) + drawSeriesBars(series); + if (series.points.show) + drawSeriesPoints(series); + } + + function drawSeriesLines(series) { + function plotLine(datapoints, xoffset, yoffset, axisx, axisy) { + var points = datapoints.points, + ps = datapoints.pointsize, + prevx = null, prevy = null; + + ctx.beginPath(); + for (var i = ps; i < points.length; i += ps) { + var x1 = points[i - ps], y1 = points[i - ps + 1], + x2 = points[i], y2 = points[i + 1]; + + if (x1 == null || x2 == null) + continue; + + // clip with ymin + if (y1 <= y2 && y1 < axisy.min) { + if (y2 < axisy.min) + continue; // line segment is outside + // compute new intersection point + x1 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1; + y1 = axisy.min; + } + else if (y2 <= y1 && y2 < axisy.min) { + if (y1 < axisy.min) + continue; + x2 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1; + y2 = axisy.min; + } + + // clip with ymax + if (y1 >= y2 && y1 > axisy.max) { + if (y2 > axisy.max) + continue; + x1 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1; + y1 = axisy.max; + } + else if (y2 >= y1 && y2 > axisy.max) { + if (y1 > axisy.max) + continue; + x2 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1; + y2 = axisy.max; + } + + // clip with xmin + if (x1 <= x2 && x1 < axisx.min) { + if (x2 < axisx.min) + continue; + y1 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1; + x1 = axisx.min; + } + else if (x2 <= x1 && x2 < axisx.min) { + if (x1 < axisx.min) + continue; + y2 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1; + x2 = axisx.min; + } + + // clip with xmax + if (x1 >= x2 && x1 > axisx.max) { + if (x2 > axisx.max) + continue; + y1 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1; + x1 = axisx.max; + } + else if (x2 >= x1 && x2 > axisx.max) { + if (x1 > axisx.max) + continue; + y2 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1; + x2 = axisx.max; + } + + if (x1 != prevx || y1 != prevy) + ctx.moveTo(axisx.p2c(x1) + xoffset, axisy.p2c(y1) + yoffset); + + prevx = x2; + prevy = y2; + ctx.lineTo(axisx.p2c(x2) + xoffset, axisy.p2c(y2) + yoffset); + } + ctx.stroke(); + } + + function plotLineArea(datapoints, axisx, axisy) { + var points = datapoints.points, + ps = datapoints.pointsize, + bottom = Math.min(Math.max(0, axisy.min), axisy.max), + top, lastX = 0, areaOpen = false; + + for (var i = ps; i < points.length; i += ps) { + var x1 = points[i - ps], y1 = points[i - ps + 1], + x2 = points[i], y2 = points[i + 1]; + + if (areaOpen && x1 != null && x2 == null) { + // close area + ctx.lineTo(axisx.p2c(lastX), axisy.p2c(bottom)); + ctx.fill(); + areaOpen = false; + continue; + } + + if (x1 == null || x2 == null) + continue; + + // clip x values + + // clip with xmin + if (x1 <= x2 && x1 < axisx.min) { + if (x2 < axisx.min) + continue; + y1 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1; + x1 = axisx.min; + } + else if (x2 <= x1 && x2 < axisx.min) { + if (x1 < axisx.min) + continue; + y2 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1; + x2 = axisx.min; + } + + // clip with xmax + if (x1 >= x2 && x1 > axisx.max) { + if (x2 > axisx.max) + continue; + y1 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1; + x1 = axisx.max; + } + else if (x2 >= x1 && x2 > axisx.max) { + if (x1 > axisx.max) + continue; + y2 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1; + x2 = axisx.max; + } + + if (!areaOpen) { + // open area + ctx.beginPath(); + ctx.moveTo(axisx.p2c(x1), axisy.p2c(bottom)); + areaOpen = true; + } + + // now first check the case where both is outside + if (y1 >= axisy.max && y2 >= axisy.max) { + ctx.lineTo(axisx.p2c(x1), axisy.p2c(axisy.max)); + ctx.lineTo(axisx.p2c(x2), axisy.p2c(axisy.max)); + lastX = x2; + continue; + } + else if (y1 <= axisy.min && y2 <= axisy.min) { + ctx.lineTo(axisx.p2c(x1), axisy.p2c(axisy.min)); + ctx.lineTo(axisx.p2c(x2), axisy.p2c(axisy.min)); + lastX = x2; + continue; + } + + // else it's a bit more complicated, there might + // be two rectangles and two triangles we need to fill + // in; to find these keep track of the current x values + var x1old = x1, x2old = x2; + + // and clip the y values, without shortcutting + + // clip with ymin + if (y1 <= y2 && y1 < axisy.min && y2 >= axisy.min) { + x1 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1; + y1 = axisy.min; + } + else if (y2 <= y1 && y2 < axisy.min && y1 >= axisy.min) { + x2 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1; + y2 = axisy.min; + } + + // clip with ymax + if (y1 >= y2 && y1 > axisy.max && y2 <= axisy.max) { + x1 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1; + y1 = axisy.max; + } + else if (y2 >= y1 && y2 > axisy.max && y1 <= axisy.max) { + x2 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1; + y2 = axisy.max; + } + + + // if the x value was changed we got a rectangle + // to fill + if (x1 != x1old) { + if (y1 <= axisy.min) + top = axisy.min; + else + top = axisy.max; + + ctx.lineTo(axisx.p2c(x1old), axisy.p2c(top)); + ctx.lineTo(axisx.p2c(x1), axisy.p2c(top)); + } + + // fill the triangles + ctx.lineTo(axisx.p2c(x1), axisy.p2c(y1)); + ctx.lineTo(axisx.p2c(x2), axisy.p2c(y2)); + + // fill the other rectangle if it's there + if (x2 != x2old) { + if (y2 <= axisy.min) + top = axisy.min; + else + top = axisy.max; + + ctx.lineTo(axisx.p2c(x2), axisy.p2c(top)); + ctx.lineTo(axisx.p2c(x2old), axisy.p2c(top)); + } + + lastX = Math.max(x2, x2old); + } + + if (areaOpen) { + ctx.lineTo(axisx.p2c(lastX), axisy.p2c(bottom)); + ctx.fill(); + } + } + + ctx.save(); + ctx.translate(plotOffset.left, plotOffset.top); + ctx.lineJoin = "round"; + + var lw = series.lines.lineWidth, + sw = series.shadowSize; + // FIXME: consider another form of shadow when filling is turned on + if (lw > 0 && sw > 0) { + // draw shadow as a thick and thin line with transparency + ctx.lineWidth = sw; + ctx.strokeStyle = "rgba(0,0,0,0.1)"; + // position shadow at angle from the mid of line + var angle = Math.PI/18; + plotLine(series.datapoints, Math.sin(angle) * (lw/2 + sw/2), Math.cos(angle) * (lw/2 + sw/2), series.xaxis, series.yaxis); + ctx.lineWidth = sw/2; + plotLine(series.datapoints, Math.sin(angle) * (lw/2 + sw/4), Math.cos(angle) * (lw/2 + sw/4), series.xaxis, series.yaxis); + } + + ctx.lineWidth = lw; + ctx.strokeStyle = series.color; + var fillStyle = getFillStyle(series.lines, series.color, 0, plotHeight); + if (fillStyle) { + ctx.fillStyle = fillStyle; + plotLineArea(series.datapoints, series.xaxis, series.yaxis); + } + + if (lw > 0) + plotLine(series.datapoints, 0, 0, series.xaxis, series.yaxis); + ctx.restore(); + } + + function drawSeriesPoints(series) { + function plotPoints(datapoints, radius, fillStyle, offset, circumference, axisx, axisy) { + var points = datapoints.points, ps = datapoints.pointsize; + + for (var i = 0; i < points.length; i += ps) { + var x = points[i], y = points[i + 1]; + if (x == null || x < axisx.min || x > axisx.max || y < axisy.min || y > axisy.max) + continue; + + ctx.beginPath(); + ctx.arc(axisx.p2c(x), axisy.p2c(y) + offset, radius, 0, circumference, false); + if (fillStyle) { + ctx.fillStyle = fillStyle; + ctx.fill(); + } + ctx.stroke(); + } + } + + ctx.save(); + ctx.translate(plotOffset.left, plotOffset.top); + + var lw = series.lines.lineWidth, + sw = series.shadowSize, + radius = series.points.radius; + if (lw > 0 && sw > 0) { + // draw shadow in two steps + var w = sw / 2; + ctx.lineWidth = w; + ctx.strokeStyle = "rgba(0,0,0,0.1)"; + plotPoints(series.datapoints, radius, null, w + w/2, Math.PI, + series.xaxis, series.yaxis); + + ctx.strokeStyle = "rgba(0,0,0,0.2)"; + plotPoints(series.datapoints, radius, null, w/2, Math.PI, + series.xaxis, series.yaxis); + } + + ctx.lineWidth = lw; + ctx.strokeStyle = series.color; + plotPoints(series.datapoints, radius, + getFillStyle(series.points, series.color), 0, 2 * Math.PI, + series.xaxis, series.yaxis); + ctx.restore(); + } + + function drawBar(x, y, b, barLeft, barRight, offset, fillStyleCallback, axisx, axisy, c, horizontal) { + var left, right, bottom, top, + drawLeft, drawRight, drawTop, drawBottom, + tmp; + + if (horizontal) { + drawBottom = drawRight = drawTop = true; + drawLeft = false; + left = b; + right = x; + top = y + barLeft; + bottom = y + barRight; + + // account for negative bars + if (right < left) { + tmp = right; + right = left; + left = tmp; + drawLeft = true; + drawRight = false; + } + } + else { + drawLeft = drawRight = drawTop = true; + drawBottom = false; + left = x + barLeft; + right = x + barRight; + bottom = b; + top = y; + + // account for negative bars + if (top < bottom) { + tmp = top; + top = bottom; + bottom = tmp; + drawBottom = true; + drawTop = false; + } + } + + // clip + if (right < axisx.min || left > axisx.max || + top < axisy.min || bottom > axisy.max) + return; + + if (left < axisx.min) { + left = axisx.min; + drawLeft = false; + } + + if (right > axisx.max) { + right = axisx.max; + drawRight = false; + } + + if (bottom < axisy.min) { + bottom = axisy.min; + drawBottom = false; + } + + if (top > axisy.max) { + top = axisy.max; + drawTop = false; + } + + left = axisx.p2c(left); + bottom = axisy.p2c(bottom); + right = axisx.p2c(right); + top = axisy.p2c(top); + + // fill the bar + if (fillStyleCallback) { + c.beginPath(); + c.moveTo(left, bottom); + c.lineTo(left, top); + c.lineTo(right, top); + c.lineTo(right, bottom); + c.fillStyle = fillStyleCallback(bottom, top); + c.fill(); + } + + // draw outline + if (drawLeft || drawRight || drawTop || drawBottom) { + c.beginPath(); + + // FIXME: inline moveTo is buggy with excanvas + c.moveTo(left, bottom + offset); + if (drawLeft) + c.lineTo(left, top + offset); + else + c.moveTo(left, top + offset); + if (drawTop) + c.lineTo(right, top + offset); + else + c.moveTo(right, top + offset); + if (drawRight) + c.lineTo(right, bottom + offset); + else + c.moveTo(right, bottom + offset); + if (drawBottom) + c.lineTo(left, bottom + offset); + else + c.moveTo(left, bottom + offset); + c.stroke(); + } + } + + function drawSeriesBars(series) { + function plotBars(datapoints, barLeft, barRight, offset, fillStyleCallback, axisx, axisy) { + var points = datapoints.points, ps = datapoints.pointsize; + + for (var i = 0; i < points.length; i += ps) { + if (points[i] == null) + continue; + drawBar(points[i], points[i + 1], points[i + 2], barLeft, barRight, offset, fillStyleCallback, axisx, axisy, ctx, series.bars.horizontal); + } + } + + ctx.save(); + ctx.translate(plotOffset.left, plotOffset.top); + + // FIXME: figure out a way to add shadows (for instance along the right edge) + ctx.lineWidth = series.bars.lineWidth; + ctx.strokeStyle = series.color; + var barLeft = series.bars.align == "left" ? 0 : -series.bars.barWidth/2; + var fillStyleCallback = series.bars.fill ? function (bottom, top) { return getFillStyle(series.bars, series.color, bottom, top); } : null; + plotBars(series.datapoints, barLeft, barLeft + series.bars.barWidth, 0, fillStyleCallback, series.xaxis, series.yaxis); + ctx.restore(); + } + + function getFillStyle(filloptions, seriesColor, bottom, top) { + var fill = filloptions.fill; + if (!fill) + return null; + + if (filloptions.fillColor) + return getColorOrGradient(filloptions.fillColor, bottom, top, seriesColor); + + var c = $.color.parse(seriesColor); + c.a = typeof fill == "number" ? fill : 0.4; + c.normalize(); + return c.toString(); + } + + function insertLegend() { + placeholder.find(".legend").remove(); + + if (!options.legend.show) + return; + + var fragments = [], rowStarted = false, + lf = options.legend.labelFormatter, s, label; + for (i = 0; i < series.length; ++i) { + s = series[i]; + label = s.label; + if (!label) + continue; + + if (i % options.legend.noColumns == 0) { + if (rowStarted) + fragments.push(''); + fragments.push(''); + rowStarted = true; + } + + if (lf) + label = lf(label, s); + + fragments.push( + '
      ' + + '' + label + ''); + } + if (rowStarted) + fragments.push(''); + + if (fragments.length == 0) + return; + + var table = '' + fragments.join("") + '
      '; + if (options.legend.container != null) + $(options.legend.container).html(table); + else { + var pos = "", + p = options.legend.position, + m = options.legend.margin; + if (m[0] == null) + m = [m, m]; + if (p.charAt(0) == "n") + pos += 'top:' + (m[1] + plotOffset.top) + 'px;'; + else if (p.charAt(0) == "s") + pos += 'bottom:' + (m[1] + plotOffset.bottom) + 'px;'; + if (p.charAt(1) == "e") + pos += 'right:' + (m[0] + plotOffset.right) + 'px;'; + else if (p.charAt(1) == "w") + pos += 'left:' + (m[0] + plotOffset.left) + 'px;'; + var legend = $('
      ' + table.replace('style="', 'style="position:absolute;' + pos +';') + '
      ').appendTo(placeholder); + if (options.legend.backgroundOpacity != 0.0) { + // put in the transparent background + // separately to avoid blended labels and + // label boxes + var c = options.legend.backgroundColor; + if (c == null) { + c = options.grid.backgroundColor; + if (c && typeof c == "string") + c = $.color.parse(c); + else + c = $.color.extract(legend, 'background-color'); + c.a = 1; + c = c.toString(); + } + var div = legend.children(); + $('
      ').prependTo(legend).css('opacity', options.legend.backgroundOpacity); + } + } + } + + + // interactive features + + var highlights = [], + redrawTimeout = null; + + // returns the data item the mouse is over, or null if none is found + function findNearbyItem(mouseX, mouseY, seriesFilter) { + var maxDistance = options.grid.mouseActiveRadius, + smallestDistance = maxDistance * maxDistance + 1, + item = null, foundPoint = false, i, j; + + for (i = 0; i < series.length; ++i) { + if (!seriesFilter(series[i])) + continue; + + var s = series[i], + axisx = s.xaxis, + axisy = s.yaxis, + points = s.datapoints.points, + ps = s.datapoints.pointsize, + mx = axisx.c2p(mouseX), // precompute some stuff to make the loop faster + my = axisy.c2p(mouseY), + maxx = maxDistance / axisx.scale, + maxy = maxDistance / axisy.scale; + + if (s.lines.show || s.points.show) { + for (j = 0; j < points.length; j += ps) { + var x = points[j], y = points[j + 1]; + if (x == null) + continue; + + // For points and lines, the cursor must be within a + // certain distance to the data point + if (x - mx > maxx || x - mx < -maxx || + y - my > maxy || y - my < -maxy) + continue; + + // We have to calculate distances in pixels, not in + // data units, because the scales of the axes may be different + var dx = Math.abs(axisx.p2c(x) - mouseX), + dy = Math.abs(axisy.p2c(y) - mouseY), + dist = dx * dx + dy * dy; // we save the sqrt + + // use <= to ensure last point takes precedence + // (last generally means on top of) + if (dist <= smallestDistance) { + smallestDistance = dist; + item = [i, j / ps]; + } + } + } + + if (s.bars.show && !item) { // no other point can be nearby + var barLeft = s.bars.align == "left" ? 0 : -s.bars.barWidth/2, + barRight = barLeft + s.bars.barWidth; + + for (j = 0; j < points.length; j += ps) { + var x = points[j], y = points[j + 1], b = points[j + 2]; + if (x == null) + continue; + + // for a bar graph, the cursor must be inside the bar + if (series[i].bars.horizontal ? + (mx <= Math.max(b, x) && mx >= Math.min(b, x) && + my >= y + barLeft && my <= y + barRight) : + (mx >= x + barLeft && mx <= x + barRight && + my >= Math.min(b, y) && my <= Math.max(b, y))) + item = [i, j / ps]; + } + } + } + + if (item) { + i = item[0]; + j = item[1]; + ps = series[i].datapoints.pointsize; + + return { datapoint: series[i].datapoints.points.slice(j * ps, (j + 1) * ps), + dataIndex: j, + series: series[i], + seriesIndex: i }; + } + + return null; + } + + function onMouseMove(e) { + if (options.grid.hoverable) + triggerClickHoverEvent("plothover", e, + function (s) { return s["hoverable"] != false; }); + } + + function onClick(e) { + triggerClickHoverEvent("plotclick", e, + function (s) { return s["clickable"] != false; }); + } + + // trigger click or hover event (they send the same parameters + // so we share their code) + function triggerClickHoverEvent(eventname, event, seriesFilter) { + var offset = eventHolder.offset(), + pos = { pageX: event.pageX, pageY: event.pageY }, + canvasX = event.pageX - offset.left - plotOffset.left, + canvasY = event.pageY - offset.top - plotOffset.top; + + if (axes.xaxis.used) + pos.x = axes.xaxis.c2p(canvasX); + if (axes.yaxis.used) + pos.y = axes.yaxis.c2p(canvasY); + if (axes.x2axis.used) + pos.x2 = axes.x2axis.c2p(canvasX); + if (axes.y2axis.used) + pos.y2 = axes.y2axis.c2p(canvasY); + + var item = findNearbyItem(canvasX, canvasY, seriesFilter); + + if (item) { + // fill in mouse pos for any listeners out there + item.pageX = parseInt(item.series.xaxis.p2c(item.datapoint[0]) + offset.left + plotOffset.left); + item.pageY = parseInt(item.series.yaxis.p2c(item.datapoint[1]) + offset.top + plotOffset.top); + } + + if (options.grid.autoHighlight) { + // clear auto-highlights + for (var i = 0; i < highlights.length; ++i) { + var h = highlights[i]; + if (h.auto == eventname && + !(item && h.series == item.series && h.point == item.datapoint)) + unhighlight(h.series, h.point); + } + + if (item) + highlight(item.series, item.datapoint, eventname); + } + + placeholder.trigger(eventname, [ pos, item ]); + } + + function triggerRedrawOverlay() { + if (!redrawTimeout) + redrawTimeout = setTimeout(drawOverlay, 30); + } + + function drawOverlay() { + redrawTimeout = null; + + // draw highlights + octx.save(); + octx.clearRect(0, 0, canvasWidth, canvasHeight); + octx.translate(plotOffset.left, plotOffset.top); + + var i, hi; + for (i = 0; i < highlights.length; ++i) { + hi = highlights[i]; + + if (hi.series.bars.show) + drawBarHighlight(hi.series, hi.point); + else + drawPointHighlight(hi.series, hi.point); + } + octx.restore(); + + executeHooks(hooks.drawOverlay, [octx]); + } + + function highlight(s, point, auto) { + if (typeof s == "number") + s = series[s]; + + if (typeof point == "number") + point = s.data[point]; + + var i = indexOfHighlight(s, point); + if (i == -1) { + highlights.push({ series: s, point: point, auto: auto }); + + triggerRedrawOverlay(); + } + else if (!auto) + highlights[i].auto = false; + } + + function unhighlight(s, point) { + if (s == null && point == null) { + highlights = []; + triggerRedrawOverlay(); + } + + if (typeof s == "number") + s = series[s]; + + if (typeof point == "number") + point = s.data[point]; + + var i = indexOfHighlight(s, point); + if (i != -1) { + highlights.splice(i, 1); + + triggerRedrawOverlay(); + } + } + + function indexOfHighlight(s, p) { + for (var i = 0; i < highlights.length; ++i) { + var h = highlights[i]; + if (h.series == s && h.point[0] == p[0] + && h.point[1] == p[1]) + return i; + } + return -1; + } + + function drawPointHighlight(series, point) { + var x = point[0], y = point[1], + axisx = series.xaxis, axisy = series.yaxis; + + if (x < axisx.min || x > axisx.max || y < axisy.min || y > axisy.max) + return; + + var pointRadius = series.points.radius + series.points.lineWidth / 2; + octx.lineWidth = pointRadius; + octx.strokeStyle = $.color.parse(series.color).scale('a', 0.5).toString(); + var radius = 1.5 * pointRadius; + octx.beginPath(); + octx.arc(axisx.p2c(x), axisy.p2c(y), radius, 0, 2 * Math.PI, false); + octx.stroke(); + } + + function drawBarHighlight(series, point) { + octx.lineWidth = series.bars.lineWidth; + octx.strokeStyle = $.color.parse(series.color).scale('a', 0.5).toString(); + var fillStyle = $.color.parse(series.color).scale('a', 0.5).toString(); + var barLeft = series.bars.align == "left" ? 0 : -series.bars.barWidth/2; + drawBar(point[0], point[1], point[2] || 0, barLeft, barLeft + series.bars.barWidth, + 0, function () { return fillStyle; }, series.xaxis, series.yaxis, octx, series.bars.horizontal); + } + + function getColorOrGradient(spec, bottom, top, defaultColor) { + if (typeof spec == "string") + return spec; + else { + // assume this is a gradient spec; IE currently only + // supports a simple vertical gradient properly, so that's + // what we support too + var gradient = ctx.createLinearGradient(0, top, 0, bottom); + + for (var i = 0, l = spec.colors.length; i < l; ++i) { + var c = spec.colors[i]; + if (typeof c != "string") { + c = $.color.parse(defaultColor).scale('rgb', c.brightness); + c.a *= c.opacity; + c = c.toString(); + } + gradient.addColorStop(i / (l - 1), c); + } + + return gradient; + } + } + } + + $.plot = function(placeholder, data, options) { + var plot = new Plot($(placeholder), data, options, $.plot.plugins); + /*var t0 = new Date(); + var t1 = new Date(); + var tstr = "time used (msecs): " + (t1.getTime() - t0.getTime()) + if (window.console) + console.log(tstr); + else + alert(tstr);*/ + return plot; + }; + + $.plot.plugins = []; + + // returns a string with the date d formatted according to fmt + $.plot.formatDate = function(d, fmt, monthNames) { + var leftPad = function(n) { + n = "" + n; + return n.length == 1 ? "0" + n : n; + }; + + var r = []; + var escape = false; + var hours = d.getUTCHours(); + var isAM = hours < 12; + if (monthNames == null) + monthNames = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + + if (fmt.search(/%p|%P/) != -1) { + if (hours > 12) { + hours = hours - 12; + } else if (hours == 0) { + hours = 12; + } + } + for (var i = 0; i < fmt.length; ++i) { + var c = fmt.charAt(i); + + if (escape) { + switch (c) { + case 'h': c = "" + hours; break; + case 'H': c = leftPad(hours); break; + case 'M': c = leftPad(d.getUTCMinutes()); break; + case 'S': c = leftPad(d.getUTCSeconds()); break; + case 'd': c = "" + d.getUTCDate(); break; + case 'm': c = "" + (d.getUTCMonth() + 1); break; + case 'y': c = "" + d.getUTCFullYear(); break; + case 'b': c = "" + monthNames[d.getUTCMonth()]; break; + case 'p': c = (isAM) ? ("" + "am") : ("" + "pm"); break; + case 'P': c = (isAM) ? ("" + "AM") : ("" + "PM"); break; + } + r.push(c); + escape = false; + } + else { + if (c == "%") + escape = true; + else + r.push(c); + } + } + return r.join(""); + }; + + // round to nearby lower multiple of base + function floorInBase(n, base) { + return base * Math.floor(n / base); + } + +})(jQuery); diff --git a/storefront/static/js/jquery.indextank-autocomplete.js b/storefront/static/js/jquery.indextank-autocomplete.js new file mode 100644 index 0000000..0c77bdd --- /dev/null +++ b/storefront/static/js/jquery.indextank-autocomplete.js @@ -0,0 +1,52 @@ +/** + * Indextank autocomplete. + * + * @param url: your Indextank PUBLIC API url. Required. + * @param indexName: the name of the index to show automcomplete for. Required. + * @param options: a hash to override default settings. Optional. + * + * @author Diego Buthay + * @version 0.1 + */ + +(function( $ ){ + + $.fn.autocompleteWithIndextank = function( url, indexName, options ) { + + var settings = { + selectCallback: function( event, ui ) { + event.target.value = ui.item.value; + event.target.form.submit(); + }, // select callback + sourceCallback: function( request, responseCallback ) { + $.ajax( { + url: url + "/v1/indexes/" + indexName + "/autocomplete", + dataType: "jsonp", + data: { query: request.term }, + success: function( data ) { responseCallback( data.suggestions ); } + } ); + }, // source callback + delay: 100, + minLength: 2 + } + + return this.each(function() { + + var $this = $(this); + // If options exist, lets merge them + // with our default settings + if ( options ) { + $.extend( settings, options ); + } + + $this.autocomplete( { + source: settings.sourceCallback, + delay: settings.delay, + minLength: settings.minLength, + select: settings.selectCallback + }); + + }); + + }; +})( jQuery ); diff --git a/storefront/static/js/jquery.indextank.ajaxsearch.js b/storefront/static/js/jquery.indextank.ajaxsearch.js new file mode 100644 index 0000000..c1a86cd --- /dev/null +++ b/storefront/static/js/jquery.indextank.ajaxsearch.js @@ -0,0 +1,145 @@ +/** XXX XXX THIS IS A MODIFIED VERSION OF INDEXTANK-JQUERY .. DON'T UPDATE IT IF NOT SURE ABOUT WHAT YOU ARE DOING .. */ + +(function($){ + if(!$.Indextank){ + $.Indextank = new Object(); + }; + + $.Indextank.AjaxSearch = function(el, options){ + // To avoid scope issues, use 'base' instead of 'this' + // to reference this class from internal events and functions. + var base = this; + + // Access to jQuery and DOM versions of element + base.$el = $(el); + base.el = el; + + // Add a reverse reference to the DOM object + base.$el.data("Indextank.AjaxSearch", base); + + base.init = function(){ + + base.options = $.extend({},$.Indextank.AjaxSearch.defaultOptions, options); + base.xhr = undefined; + + + // TODO: make sure ize is an Indextank.Ize element somehow + base.ize = $(base.el.form).data("Indextank.Ize"); + base.ize.$el.bind("submit", base.hijackFormSubmit); + + + // make it possible for other to trigger an ajax search + base.$el.bind( "Indextank.AjaxSearch.runQuery", base.runQuery ); + }; + + // Sample Function, Uncomment to use + // base.functionName = function(paramaters){ + // + // }; + + base.runQuery = function( event, term, start, rsLength ) { + // don't run a query twice + var query = base.options.rewriteQuery( term || base.el.value ); + start = start || base.options.start; + rsLength = rsLength || base.options.rsLength; + + if (base.query == query && base.start == start && base.rsLength == rsLength ) { + return; + } + + // if we are running a query, an old one makes no sense. + if (base.xhr != undefined ) { + base.xhr.abort(); + } + + + // remember the current running query + base.query = query; + base.start = start; + base.rsLength = rsLength; + + base.options.listeners.trigger("Indextank.AjaxSearch.searching"); + base.$el.trigger("Indextank.AjaxSearch.searching"); + + + // run the query, with ajax + base.xhr = $.ajax( { + url: base.ize.apiurl + "/v1/indexes/" + base.ize.indexName + "/search", + dataType: "jsonp", + timeout: 10000, + data: { + "q": query, + "fetch": base.options.fields, + "snippet": base.options.snippets, + "function": base.options.scoringFunction, + "start": start, + "len": rsLength, + "fetch_variables": base.options.fetchVariables, + "fetch_categories": base.options.fetchCategories + + }, + success: function( data ) { + // Indextank API does not send the query, nor start or rsLength + // I'll save the current query inside 'data', + // so our listeners can use it. + data.query = query; + data.start = start; + data.rsLength = rsLength; + base.options.listeners.trigger("Indextank.AjaxSearch.success", data); + }, + error: function( jqXHR, textStatus, errorThrown) { + base.options.listeners.trigger("Indextank.AjaxSearch.failure"); + } + } ); + } + + base.hijackFormSubmit = function(event) { + // make sure the form is not submitted + event.preventDefault(); + base.runQuery(); + }; + + + // unbind everything + base.destroy = function() { + base.$el.unbind("Indextank.AjaxSearch.runQuery", base.runQuery); + base.ize.$el.unbind("submit", base.hijackFormSubmit); + base.$el.removeData("Indextank.AjaxSearch"); + }; + + + // Run initializer + base.init(); + }; + + $.Indextank.AjaxSearch.defaultOptions = { + // first result to fetch .. it can be overrided at query-time, + // but we need a default. 99.95% of the times you'll want to keep the default + start : 0, + // how many results to fetch on every query? + // it can be overriden at query-time. + rsLength : 10, + // default fields to fetch .. + fields : "name,title,image,url,link", + // fields to make snippets for + snippets : "text", + // no one listening .. sad + listeners: $([]), + // scoring function to use + scoringFunction: 0, + // fetch variables ? + fetchVariables: true, + // fetch categories ? + fetchCategories: true, + // the default query re-writer is identity + rewriteQuery: function(q) {return q} + + }; + + $.fn.indextank_AjaxSearch = function(options){ + return this.each(function(){ + (new $.Indextank.AjaxSearch(this, options)); + }); + }; + +})(jQuery); diff --git a/storefront/static/js/jquery.indextank.autocomplete.js b/storefront/static/js/jquery.indextank.autocomplete.js new file mode 100644 index 0000000..1e5ada6 --- /dev/null +++ b/storefront/static/js/jquery.indextank.autocomplete.js @@ -0,0 +1,82 @@ +(function($){ + if(!$.Indextank){ + $.Indextank = new Object(); + }; + + $.Indextank.Autocomplete = function(el, options){ + // To avoid scope issues, use 'base' instead of 'this' + // to reference this class from internal events and functions. + var base = this; + + // Access to jQuery and DOM versions of element + base.$el = $(el); + base.el = el; + + // Add a reverse reference to the DOM object + base.$el.data("Indextank.Autocomplete", base); + + base.init = function(){ + base.options = $.extend({},$.Indextank.Autocomplete.defaultOptions, options); + + // Put your initialization code here + var ize = $(base.el.form).data("Indextank.Ize"); + + base.$el.autocomplete({ + select: function( event, ui ) { + event.target.value = ui.item.value; + // wrap form into a jQuery object, so submit honors onsubmit. + $(event.target.form).submit(); + }, + source: function ( request, responseCallback ) { + $.ajax( { + url: ize.apiurl + "/v1/indexes/" + ize.indexName + "/autocomplete", + dataType: "jsonp", + data: { query: request.term, field: base.options.fieldName }, + success: function( data ) { responseCallback(data.suggestions); base.$el.trigger("Indextank.Autocomplete.success", data.suggestions); } + } ); + }, + minLength: base.options.minLength, + delay: base.options.delay + }); + + // make sure autocomplete closes when IndextankIzed form submits + ize.$el.submit(function(e){ + base.$el.data("autocomplete").close(); + }); + + // and also disable it when Indextank.AjaxSearch is searching .. + base.$el.bind("Indextank.AjaxSearch.searching", function(e) { + // hacky way to abort a request on jquery.ui.autocomplete. + //base.$el.data("autocomplete").disable(); + //window.setTimeout(function(){base.$el.data("autocomplete").enable();}, 1000); + }); + }; + + // Sample Function, Uncomment to use + // base.functionName = function(paramaters){ + // + // }; + + // Run initializer + base.init(); + }; + + $.Indextank.Autocomplete.defaultOptions = { + fieldName: "text", + minLength: 2, + delay: 100 + }; + + $.fn.indextank_Autocomplete = function(options){ + return this.each(function(){ + (new $.Indextank.Autocomplete(this, options)); + }); + }; + + // This function breaks the chain, but returns + // the Indextank.autocomplete if it has been attached to the object. + $.fn.getIndextank_Autocomplete = function(){ + this.data("Indextank.Autocomplete"); + }; + +})(jQuery); diff --git a/storefront/static/js/jquery.indextank.basic.js b/storefront/static/js/jquery.indextank.basic.js new file mode 100644 index 0000000..65ece18 --- /dev/null +++ b/storefront/static/js/jquery.indextank.basic.js @@ -0,0 +1,64 @@ +(function($){ + if(!$.Indextank){ + $.Indextank = new Object(); + }; + + $.Indextank.Basic = function(el, apiurl, indexName, options){ + // To avoid scope issues, use 'base' instead of 'this' + // to reference this class from internal events and functions. + var base = this; + + // Access to jQuery and DOM versions of element + base.$el = $(el); + base.el = el; + + // Add a reverse reference to the DOM object + base.$el.data("Indextank.Basic", base); + + base.init = function(){ + base.apiurl = apiurl; + base.indexName = indexName; + + base.options = $.extend({},$.Indextank.Basic.defaultOptions, options); + + // create a form + base.form = $("
      "); + base.form.indextank_Ize(apiurl, indexName); + base.form.attr("id","IndextankBasicForm"); + base.form.appendTo(base.$el); + + // create an input + base.queryInput = $(""); + base.queryInput.appendTo(base.form); + base.queryInput.indextank_Autocomplete(); + + + // create a result div + base.resultDiv = $("
      "); + base.resultDiv.attr("id","IndextankBasicResults"); + base.resultDiv.indextank_Renderer(); + base.resultDiv.appendTo(base.$el); + + // make queryInput send its results to the renderer + base.queryInput.indextank_AjaxSearch({listeners: base.resultDiv}); + }; + + // Sample Function, Uncomment to use + // base.functionName = function(paramaters){ + // + // }; + + // Run initializer + base.init(); + }; + + $.Indextank.Basic.defaultOptions = { + }; + + $.fn.indextank_Basic = function(apiurl, indexName, options){ + return this.each(function(){ + (new $.Indextank.Basic(this, apiurl, indexName, options)); + }); + }; + +})(jQuery); diff --git a/storefront/static/js/jquery.indextank.dashboardrenderer.js b/storefront/static/js/jquery.indextank.dashboardrenderer.js new file mode 100644 index 0000000..4eed904 --- /dev/null +++ b/storefront/static/js/jquery.indextank.dashboardrenderer.js @@ -0,0 +1,67 @@ +(function($){ + if(!$.Indextank){ + $.Indextank = new Object(); + }; + + $.Indextank.DashboardRenderer = function(el, options){ + // To avoid scope issues, use 'base' instead of 'this' + // to reference this class from internal events and functions. + var base = this; + + // Access to jQuery and DOM versions of element + base.$el = $(el); + base.el = el; + + // Add a reverse reference to the DOM object + base.$el.data("Indextank.DashboardRenderer", base); + + base.init = function(){ + base.options = $.extend({},$.Indextank.Renderer.defaultOptions, options); + + // Put your initialization code here + base.$el.bind("Indextank.AjaxSearch.searching", function(e) { + base.$el.css({opacity: 0.5}); + }); + + base.$el.bind("Indextank.AjaxSearch.success", function(e, data) { + // hacky clean up + $("td.result", base.$el).parent().detach(); + $("td.fullresult", base.$el.parent()).hide(); + + $(data.results).each( function (i, item) { + var r = base.options.format(item); + r.appendTo(base.$el); + }); + base.$el.css({opacity: 1}); + }); + base.$el.bind("Indextank.AjaxSearch.failure", function(e) { + base.$el.css({opacity: 1}); + }); + + }; + + // Sample Function, Uncomment to use + // base.functionName = function(paramaters){ + // + // }; + + // Run initializer + base.init(); + }; + + $.Indextank.DashboardRenderer.defaultOptions = { + format: function(item){ + return $("
      ") + .addClass("result") + .append( $("").attr("href", item.link || item.url ).text(item.title || item.name) ) + .append( $("").addClass("description").html(item.snippet_text || item.text) ); + } + }; + + $.fn.indextank_DashboardRenderer = function(options){ + return this.each(function(){ + (new $.Indextank.DashboardRenderer(this, options)); + }); + }; + +})(jQuery); diff --git a/storefront/static/js/jquery.indextank.instantlinks.js b/storefront/static/js/jquery.indextank.instantlinks.js new file mode 100644 index 0000000..44d5558 --- /dev/null +++ b/storefront/static/js/jquery.indextank.instantlinks.js @@ -0,0 +1,115 @@ +(function($){ + if(!$.Indextank){ + $.Indextank = new Object(); + }; + + $.Indextank.InstantLinks = function(el, options){ + // To avoid scope issues, use 'base' instead of 'this' + // to reference this class from internal events and functions. + var base = this; + + // Access to jQuery and DOM versions of element + base.$el = $(el); + base.el = el; + + // Add a reverse reference to the DOM object + base.$el.data("Indextank.InstantLinks", base); + + base.options = $.extend({},$.Indextank.InstantLinks.defaultOptions, options); + + base.init = function(){ + // Put your initialization code here + var ize = $(base.el.form).data("Indextank.Ize"); + + base.$el.autocomplete({ + select: function( event, ui ) { + window.location.href = ui.item[base.options.url]; + }, + source: function ( request, responseCallback ) { + $.ajax( { + url: ize.apiurl + "/v1/indexes/" + ize.indexName + "/instantlinks", + dataType: "jsonp", + data: { query: request.term, field: base.options.name, fetch: base.options.fields }, + success: function( data ) { + // augment results, so that they contain the matched query + var results = $.map(data.results, function(r) { + r.queryTerm = request.term; + return r; + }); + responseCallback(results); + } + } ); + }, + minLength: base.options.minLength, + delay: base.options.delay + }) + .data( "autocomplete" )._renderItem = function( ul, item) { + // create the list entry + var $li = $("
    • ").addClass("result").data("item", item); + + // append a formatted item. + $li.append(base.options.format(item, base.options)); + + // put the li back on the ul + return $li.appendTo( ul ); + }; + }; + + // Run initializer + base.init(); + }; + + $.Indextank.InstantLinks.defaultOptions = { + name: "name", + url: "url", + thumbnail: "thumbnail", + description: "description", + fields: "name,url,thumbnail,description", + minLength: 2, + delay: 100, + format: function( item , options ) { + + function hl(text, query){ + rx = new RegExp(query,'ig'); + bolds = $.map(text.match(rx) || [], function(i) { return ""+i+"";}); + regulars = $( $.map(text.split(rx), function(i){ return $("").addClass("regular").text(i).get(0);})); + + return regulars.append(function(i, h) { + return bolds[i] || ""; + }); + }; + + + var name = item[options.name]; + var highlightedName = hl(name, item.queryTerm); + + + var l = $("").attr("href", item[options.url]); + + // only display images for those documents that have one + if (item[options.thumbnail]) { + l.addClass("with-thumbnail"); + l.append( $("") + .attr("src", item[options.thumbnail]) + .css( { "max-width": "50px", "max-height": "50px"} ) ) ; + } + + l.append( $("").addClass("name").append(highlightedName) ); + + // only add description for those documents that have one + if (item[options.description]) { + l.addClass("with-description"); + l.append( $("").addClass("description").text(item[options.description])); + } + + return l; + } + }; + + $.fn.indextank_InstantLinks = function(options){ + return this.each(function(){ + (new $.Indextank.InstantLinks(this, options)); + }); + }; + +})(jQuery); diff --git a/storefront/static/js/jquery.indextank.instantsearch.js b/storefront/static/js/jquery.indextank.instantsearch.js new file mode 100644 index 0000000..2e6394b --- /dev/null +++ b/storefront/static/js/jquery.indextank.instantsearch.js @@ -0,0 +1,59 @@ +(function($){ + if(!$.Indextank){ + $.Indextank = new Object(); + }; + + $.Indextank.InstantSearch = function(el, options){ + // To avoid scope issues, use 'base' instead of 'this' + // to reference this class from internal events and functions. + var base = this; + + // Access to jQuery and DOM versions of element + base.$el = $(el); + base.el = el; + + // Add a reverse reference to the DOM object + base.$el.data("Indextank.InstantSearch", base); + + base.init = function(){ + base.options = $.extend({},$.Indextank.InstantSearch.defaultOptions, options); + + // make autocomplete trigger a query when suggestions appear + base.$el.bind( "Indextank.Autocomplete.success", function (event, suggestions ) { + base.$el.trigger( "Indextank.AjaxSearch.runQuery", suggestions ); + }); + + // make autocomplete focus trigger an AjaxSearch, only if requested + if (base.options.focusTriggersSearch) { + base.$el.bind( "autocompletefocus", function (event, ui) { + base.$el.trigger( "Indextank.AjaxSearch.runQuery", ui.item.value ); + }); + } + + }; + + // Sample Function, Uncomment to use + // base.functionName = function(paramaters){ + // + // }; + + // Run initializer + base.init(); + }; + + $.Indextank.InstantSearch.defaultOptions = { + // trigger a query whenever an option on the autocomplete box is + // focused. Either by keyboard or mouse hover. + // Note that setting this to true can be annoying if your search box is + // above the result set, as moving the mouse over the suggestions will + // change the result set. + focusTriggersSearch : false + }; + + $.fn.indextank_InstantSearch = function(options){ + return this.each(function(){ + (new $.Indextank.InstantSearch(this, options)); + }); + }; + +})(jQuery); diff --git a/storefront/static/js/jquery.indextank.ize.js b/storefront/static/js/jquery.indextank.ize.js new file mode 100644 index 0000000..b1a34a5 --- /dev/null +++ b/storefront/static/js/jquery.indextank.ize.js @@ -0,0 +1,56 @@ +(function($){ + if(!$.Indextank){ + $.Indextank = new Object(); + }; + + $.Indextank.Ize = function(el, apiurl, indexName, options){ + // To avoid scope issues, use 'base' instead of 'this' + // to reference this class from internal events and functions. + var base = this; + + // Access to jQuery and DOM versions of element + base.$el = $(el); + base.el = el; + + // some parameter validation + var urlrx = /http(s)?:\/\/[a-z0-9]+.api.indextank.com/ + //if (!urlrx.test(apiurl)) throw("invalid api url!"); + if (indexName == undefined) throw("index name is not defined!"); + + // Add a reverse reference to the DOM object + base.$el.data("Indextank.Ize", base); + + base.init = function(){ + base.apiurl = apiurl; + base.indexName = indexName; + + base.options = $.extend({},$.Indextank.Ize.defaultOptions, options); + + // Put your initialization code here + }; + + // Sample Function, Uncomment to use + // base.functionName = function(paramaters){ + // + // }; + + // Run initializer + base.init(); + }; + + $.Indextank.Ize.defaultOptions = { + }; + + $.fn.indextank_Ize = function(apiurl, indexName, options){ + return this.each(function(){ + (new $.Indextank.Ize(this, apiurl, indexName, options)); + }); + }; + + // This function breaks the chain, but returns + // the Indextank.Ize if it has been attached to the object. + $.fn.getIndextank_Ize = function(){ + this.data("Indextank.Ize"); + }; + +})(jQuery); diff --git a/storefront/static/js/jquery.indextank.renderer.js b/storefront/static/js/jquery.indextank.renderer.js new file mode 100644 index 0000000..d35a95f --- /dev/null +++ b/storefront/static/js/jquery.indextank.renderer.js @@ -0,0 +1,65 @@ +(function($){ + if(!$.Indextank){ + $.Indextank = new Object(); + }; + + $.Indextank.Renderer = function(el, options){ + // To avoid scope issues, use 'base' instead of 'this' + // to reference this class from internal events and functions. + var base = this; + + // Access to jQuery and DOM versions of element + base.$el = $(el); + base.el = el; + + // Add a reverse reference to the DOM object + base.$el.data("Indextank.Renderer", base); + + base.init = function(){ + base.options = $.extend({},$.Indextank.Renderer.defaultOptions, options); + + // Put your initialization code here + base.$el.bind("Indextank.AjaxSearch.searching", function(e) { + base.$el.css({opacity: 0.5}); + }); + + base.$el.bind("Indextank.AjaxSearch.success", function(e, data) { + base.$el.html(""); + + $(data.results).each( function (i, item) { + var r = base.options.format(item); + r.appendTo(base.$el); + }); + base.$el.css({opacity: 1}); + }); + base.$el.bind("Indextank.AjaxSearch.failure", function(e) { + base.$el.css({opacity: 1}); + }); + + }; + + // Sample Function, Uncomment to use + // base.functionName = function(paramaters){ + // + // }; + + // Run initializer + base.init(); + }; + + $.Indextank.Renderer.defaultOptions = { + format: function(item){ + return $("
      ") + .addClass("result") + .append( $("").attr("href", item.link || item.url ).text(item.title || item.name) ) + .append( $("").addClass("description").html(item.snippet_text || item.text) ); + } + }; + + $.fn.indextank_Renderer = function(options){ + return this.each(function(){ + (new $.Indextank.Renderer(this, options)); + }); + }; + +})(jQuery); diff --git a/storefront/static/js/jquery.indextank.statsrenderer.js b/storefront/static/js/jquery.indextank.statsrenderer.js new file mode 100644 index 0000000..8b99dc2 --- /dev/null +++ b/storefront/static/js/jquery.indextank.statsrenderer.js @@ -0,0 +1,56 @@ +(function($){ + if(!$.Indextank){ + $.Indextank = new Object(); + }; + + $.Indextank.StatsRenderer = function(el, options){ + // To avoid scope issues, use 'base' instead of 'this' + // to reference this class from internal events and functions. + var base = this; + + // Access to jQuery and DOM versions of element + base.$el = $(el); + base.el = el; + + // Add a reverse reference to the DOM object + base.$el.data("Indextank.StatsRenderer", base); + + base.init = function(){ + base.options = $.extend({},$.Indextank.StatsRenderer.defaultOptions, options); + + + base.$el.bind( "Indextank.AjaxSearch.success", function (event, data) { + base.$el.show(); + base.$el.html(""); + + var stats = base.options.format(data); + stats.appendTo(base.$el); + }); + }; + + + // Run initializer + base.init(); + }; + + $.Indextank.StatsRenderer.defaultOptions = { + format: function (data) { + var r = $("
      ") + .append( $("").text(data.matches) ) + .append( $("").text(" " + (data.matches == 1 ? "result":"results" )+ " for ") ) + .append( $("").text(data.query) ) + .append( $("").text(" in ") ) + .append( $("").text(data.search_time) ) + .append( $("").text(" seconds.") ); + + return r; + } + }; + + $.fn.indextank_StatsRenderer = function(options){ + return this.each(function(){ + (new $.Indextank.StatsRenderer(this, options)); + }); + }; + +})(jQuery); diff --git a/storefront/static/mysql_import.py.gz b/storefront/static/mysql_import.py.gz new file mode 100644 index 0000000..18e12df Binary files /dev/null and b/storefront/static/mysql_import.py.gz differ diff --git a/storefront/static/mysql_import.zip b/storefront/static/mysql_import.zip new file mode 100644 index 0000000..19b5f57 Binary files /dev/null and b/storefront/static/mysql_import.zip differ diff --git a/storefront/static/old-images/accept.png b/storefront/static/old-images/accept.png new file mode 100644 index 0000000..4723b92 Binary files /dev/null and b/storefront/static/old-images/accept.png differ diff --git a/storefront/static/old-images/btn3.png b/storefront/static/old-images/btn3.png new file mode 100644 index 0000000..3647454 Binary files /dev/null and b/storefront/static/old-images/btn3.png differ diff --git a/storefront/static/old-images/clarin.gif b/storefront/static/old-images/clarin.gif new file mode 100644 index 0000000..23ed3fb Binary files /dev/null and b/storefront/static/old-images/clarin.gif differ diff --git a/storefront/static/old-images/clarin.png b/storefront/static/old-images/clarin.png new file mode 100644 index 0000000..6957f6a Binary files /dev/null and b/storefront/static/old-images/clarin.png differ diff --git a/storefront/static/old-images/clock.png b/storefront/static/old-images/clock.png new file mode 100644 index 0000000..1e087e6 Binary files /dev/null and b/storefront/static/old-images/clock.png differ diff --git a/storefront/static/old-images/clock_2.png b/storefront/static/old-images/clock_2.png new file mode 100644 index 0000000..055abf8 Binary files /dev/null and b/storefront/static/old-images/clock_2.png differ diff --git a/storefront/static/old-images/database_add.png b/storefront/static/old-images/database_add.png new file mode 100644 index 0000000..878699b Binary files /dev/null and b/storefront/static/old-images/database_add.png differ diff --git a/storefront/static/old-images/database_add_2.png b/storefront/static/old-images/database_add_2.png new file mode 100644 index 0000000..65000f1 Binary files /dev/null and b/storefront/static/old-images/database_add_2.png differ diff --git a/storefront/static/old-images/database_search.png b/storefront/static/old-images/database_search.png new file mode 100644 index 0000000..26c1f2f Binary files /dev/null and b/storefront/static/old-images/database_search.png differ diff --git a/storefront/static/old-images/database_search_2.png b/storefront/static/old-images/database_search_2.png new file mode 100644 index 0000000..54b114a Binary files /dev/null and b/storefront/static/old-images/database_search_2.png differ diff --git a/storefront/static/old-images/dollar_currency_sign.png b/storefront/static/old-images/dollar_currency_sign.png new file mode 100644 index 0000000..85cbcab Binary files /dev/null and b/storefront/static/old-images/dollar_currency_sign.png differ diff --git a/storefront/static/old-images/graph_up.png b/storefront/static/old-images/graph_up.png new file mode 100644 index 0000000..8bd08b4 Binary files /dev/null and b/storefront/static/old-images/graph_up.png differ diff --git a/storefront/static/old-images/img03.gif b/storefront/static/old-images/img03.gif new file mode 100644 index 0000000..2bc7a72 Binary files /dev/null and b/storefront/static/old-images/img03.gif differ diff --git a/storefront/static/old-images/indextank copy.png b/storefront/static/old-images/indextank copy.png new file mode 100644 index 0000000..687e8d2 Binary files /dev/null and b/storefront/static/old-images/indextank copy.png differ diff --git a/storefront/static/old-images/indextank.png b/storefront/static/old-images/indextank.png new file mode 100644 index 0000000..4abddfc Binary files /dev/null and b/storefront/static/old-images/indextank.png differ diff --git a/storefront/static/old-images/logo.png b/storefront/static/old-images/logo.png new file mode 100644 index 0000000..5aafc83 Binary files /dev/null and b/storefront/static/old-images/logo.png differ diff --git a/storefront/static/old-images/male_female_users.png b/storefront/static/old-images/male_female_users.png new file mode 100644 index 0000000..2b1f9cf Binary files /dev/null and b/storefront/static/old-images/male_female_users.png differ diff --git a/storefront/static/old-images/mylife.gif b/storefront/static/old-images/mylife.gif new file mode 100644 index 0000000..c98fc0f Binary files /dev/null and b/storefront/static/old-images/mylife.gif differ diff --git a/storefront/static/old-images/mylife.png b/storefront/static/old-images/mylife.png new file mode 100644 index 0000000..2c18588 Binary files /dev/null and b/storefront/static/old-images/mylife.png differ diff --git a/storefront/static/old-images/new-badge.png b/storefront/static/old-images/new-badge.png new file mode 100644 index 0000000..59fe307 Binary files /dev/null and b/storefront/static/old-images/new-badge.png differ diff --git a/storefront/static/old-images/process.png b/storefront/static/old-images/process.png new file mode 100644 index 0000000..72acb87 Binary files /dev/null and b/storefront/static/old-images/process.png differ diff --git a/storefront/static/old-images/process_2.png b/storefront/static/old-images/process_2.png new file mode 100644 index 0000000..2b902fd Binary files /dev/null and b/storefront/static/old-images/process_2.png differ diff --git a/storefront/static/old-images/refresh.png b/storefront/static/old-images/refresh.png new file mode 100644 index 0000000..b57cf87 Binary files /dev/null and b/storefront/static/old-images/refresh.png differ diff --git a/storefront/static/old-images/security.png b/storefront/static/old-images/security.png new file mode 100644 index 0000000..ffc9316 Binary files /dev/null and b/storefront/static/old-images/security.png differ diff --git a/storefront/static/old-images/staff.jpg b/storefront/static/old-images/staff.jpg new file mode 100644 index 0000000..00b3511 Binary files /dev/null and b/storefront/static/old-images/staff.jpg differ diff --git a/storefront/static/old-images/tableheader.png b/storefront/static/old-images/tableheader.png new file mode 100644 index 0000000..3a66423 Binary files /dev/null and b/storefront/static/old-images/tableheader.png differ diff --git a/storefront/static/old-images/target.png b/storefront/static/old-images/target.png new file mode 100644 index 0000000..5304f62 Binary files /dev/null and b/storefront/static/old-images/target.png differ diff --git a/storefront/static/old-images/tools.png b/storefront/static/old-images/tools.png new file mode 100644 index 0000000..51c1201 Binary files /dev/null and b/storefront/static/old-images/tools.png differ diff --git a/storefront/static/old-images/user_add.png b/storefront/static/old-images/user_add.png new file mode 100644 index 0000000..a14ab2f Binary files /dev/null and b/storefront/static/old-images/user_add.png differ diff --git a/storefront/static/old-images/user_add_2.png b/storefront/static/old-images/user_add_2.png new file mode 100644 index 0000000..98fec97 Binary files /dev/null and b/storefront/static/old-images/user_add_2.png differ diff --git a/storefront/static/old-images/users.png b/storefront/static/old-images/users.png new file mode 100644 index 0000000..cfc89c8 Binary files /dev/null and b/storefront/static/old-images/users.png differ diff --git a/storefront/static/old-images/users_2.png b/storefront/static/old-images/users_2.png new file mode 100644 index 0000000..e9fe1b0 Binary files /dev/null and b/storefront/static/old-images/users_2.png differ diff --git a/storefront/static/old-images/wordpress.png b/storefront/static/old-images/wordpress.png new file mode 100644 index 0000000..652f308 Binary files /dev/null and b/storefront/static/old-images/wordpress.png differ diff --git a/storefront/static/old-images/zip_file_download.png b/storefront/static/old-images/zip_file_download.png new file mode 100644 index 0000000..2821075 Binary files /dev/null and b/storefront/static/old-images/zip_file_download.png differ diff --git a/storefront/static/old-images/zip_file_download_32.png b/storefront/static/old-images/zip_file_download_32.png new file mode 100644 index 0000000..f2fa108 Binary files /dev/null and b/storefront/static/old-images/zip_file_download_32.png differ diff --git a/storefront/static/scripts.js b/storefront/static/scripts.js new file mode 100644 index 0000000..0e1b8e4 --- /dev/null +++ b/storefront/static/scripts.js @@ -0,0 +1,16 @@ +/*! + * jQuery JavaScript Library v1.4.1 + * http://jquery.com/ + * + * Copyright 2010, John Resig + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * Includes Sizzle.js + * http://sizzlejs.com/ + * Copyright 2010, The Dojo Foundation + * Released under the MIT, BSD, and GPL Licenses. + * + * Date: Mon Jan 25 19:43:33 2010 -0500 + */ +(function(z,v){function la(){if(!c.isReady){try{r.documentElement.doScroll("left")}catch(a){setTimeout(la,1);return}c.ready()}}function Ma(a,b){b.src?c.ajax({url:b.src,async:false,dataType:"script"}):c.globalEval(b.text||b.textContent||b.innerHTML||"");b.parentNode&&b.parentNode.removeChild(b)}function X(a,b,d,f,e,i){var j=a.length;if(typeof b==="object"){for(var n in b)X(a,n,b[n],f,e,d);return a}if(d!==v){f=!i&&f&&c.isFunction(d);for(n=0;n-1){i=j.data;i.beforeFilter&&i.beforeFilter[a.type]&&!i.beforeFilter[a.type](a)||f.push(j.selector)}else delete x[o]}i=c(a.target).closest(f,a.currentTarget);m=0;for(s=i.length;m)[^>]*$|^#([\w-]+)$/,Qa=/^.[^:#\[\.,]*$/,Ra=/\S/,Sa=/^(\s|\u00A0)+|(\s|\u00A0)+$/g,Ta=/^<(\w+)\s*\/?>(?:<\/\1>)?$/,O=navigator.userAgent,va=false,P=[],L,$=Object.prototype.toString,aa=Object.prototype.hasOwnProperty,ba=Array.prototype.push,Q=Array.prototype.slice,wa=Array.prototype.indexOf;c.fn=c.prototype={init:function(a,b){var d,f;if(!a)return this;if(a.nodeType){this.context=this[0]=a;this.length=1;return this}if(typeof a==="string")if((d=Pa.exec(a))&&(d[1]||!b))if(d[1]){f=b?b.ownerDocument||b:r;if(a=Ta.exec(a))if(c.isPlainObject(b)){a=[r.createElement(a[1])];c.fn.attr.call(a,b,true)}else a=[f.createElement(a[1])];else{a=ra([d[1]],[f]);a=(a.cacheable?a.fragment.cloneNode(true):a.fragment).childNodes}}else{if(b=r.getElementById(d[2])){if(b.id!==d[2])return S.find(a);this.length=1;this[0]=b}this.context=r;this.selector=a;return this}else if(!b&&/^\w+$/.test(a)){this.selector=a;this.context=r;a=r.getElementsByTagName(a)}else return!b||b.jquery?(b||S).find(a):c(b).find(a);else if(c.isFunction(a))return S.ready(a);if(a.selector!==v){this.selector=a.selector;this.context=a.context}return c.isArray(a)?this.setArray(a):c.makeArray(a,this)},selector:"",jquery:"1.4.1",length:0,size:function(){return this.length},toArray:function(){return Q.call(this,0)},get:function(a){return a==null?this.toArray():a<0?this.slice(a)[0]:this[a]},pushStack:function(a,b,d){a=c(a||null);a.prevObject=this;a.context=this.context;if(b==="find")a.selector=this.selector+(this.selector?" ":"")+d;else if(b)a.selector=this.selector+"."+b+"("+d+")";return a},setArray:function(a){this.length=0;ba.apply(this,a);return this},each:function(a,b){return c.each(this,a,b)},ready:function(a){c.bindReady();if(c.isReady)a.call(r,c);else P&&P.push(a);return this},eq:function(a){return a===-1?this.slice(a):this.slice(a,+a+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(Q.apply(this,arguments),"slice",Q.call(arguments).join(","))},map:function(a){return this.pushStack(c.map(this,function(b,d){return a.call(b,d,b)}))},end:function(){return this.prevObject||c(null)},push:ba,sort:[].sort,splice:[].splice};c.fn.init.prototype=c.fn;c.extend=c.fn.extend=function(){var a=arguments[0]||{},b=1,d=arguments.length,f=false,e,i,j,n;if(typeof a==="boolean"){f=a;a=arguments[1]||{};b=2}if(typeof a!=="object"&&!c.isFunction(a))a={};if(d===b){a=this;--b}for(;b
      a";var e=d.getElementsByTagName("*"),i=d.getElementsByTagName("a")[0];if(!(!e||!e.length||!i)){c.support={leadingWhitespace:d.firstChild.nodeType===3,tbody:!d.getElementsByTagName("tbody").length,htmlSerialize:!!d.getElementsByTagName("link").length,style:/red/.test(i.getAttribute("style")),hrefNormalized:i.getAttribute("href")==="/a",opacity:/^0.55$/.test(i.style.opacity),cssFloat:!!i.style.cssFloat,checkOn:d.getElementsByTagName("input")[0].value==="on",optSelected:r.createElement("select").appendChild(r.createElement("option")).selected,checkClone:false,scriptEval:false,noCloneEvent:true,boxModel:null};b.type="text/javascript";try{b.appendChild(r.createTextNode("window."+f+"=1;"))}catch(j){}a.insertBefore(b,a.firstChild);if(z[f]){c.support.scriptEval=true;delete z[f]}a.removeChild(b);if(d.attachEvent&&d.fireEvent){d.attachEvent("onclick",function n(){c.support.noCloneEvent=false;d.detachEvent("onclick",n)});d.cloneNode(true).fireEvent("onclick")}d=r.createElement("div");d.innerHTML="";a=r.createDocumentFragment();a.appendChild(d.firstChild);c.support.checkClone=a.cloneNode(true).cloneNode(true).lastChild.checked;c(function(){var n=r.createElement("div");n.style.width=n.style.paddingLeft="1px";r.body.appendChild(n);c.boxModel=c.support.boxModel=n.offsetWidth===2;r.body.removeChild(n).style.display="none"});a=function(n){var o=r.createElement("div");n="on"+n;var m=n in o;if(!m){o.setAttribute(n,"return;");m=typeof o[n]==="function"}return m};c.support.submitBubbles=a("submit");c.support.changeBubbles=a("change");a=b=d=e=i=null}})();c.props={"for":"htmlFor","class":"className",readonly:"readOnly",maxlength:"maxLength",cellspacing:"cellSpacing",rowspan:"rowSpan",colspan:"colSpan",tabindex:"tabIndex",usemap:"useMap",frameborder:"frameBorder"};var G="jQuery"+J(),Ua=0,xa={},Va={};c.extend({cache:{},expando:G,noData:{embed:true,object:true,applet:true},data:function(a,b,d){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==z?xa:a;var f=a[G],e=c.cache;if(!b&&!f)return null;f||(f=++Ua);if(typeof b==="object"){a[G]=f;e=e[f]=c.extend(true,{},b)}else e=e[f]?e[f]:typeof d==="undefined"?Va:(e[f]={});if(d!==v){a[G]=f;e[b]=d}return typeof b==="string"?e[b]:e}},removeData:function(a,b){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==z?xa:a;var d=a[G],f=c.cache,e=f[d];if(b){if(e){delete e[b];c.isEmptyObject(e)&&c.removeData(a)}}else{try{delete a[G]}catch(i){a.removeAttribute&&a.removeAttribute(G)}delete f[d]}}}});c.fn.extend({data:function(a,b){if(typeof a==="undefined"&&this.length)return c.data(this[0]);else if(typeof a==="object")return this.each(function(){c.data(this,a)});var d=a.split(".");d[1]=d[1]?"."+d[1]:"";if(b===v){var f=this.triggerHandler("getData"+d[1]+"!",[d[0]]);if(f===v&&this.length)f=c.data(this[0],a);return f===v&&d[1]?this.data(d[0]):f}else return this.trigger("setData"+d[1]+"!",[d[0],b]).each(function(){c.data(this,a,b)})},removeData:function(a){return this.each(function(){c.removeData(this,a)})}});c.extend({queue:function(a,b,d){if(a){b=(b||"fx")+"queue";var f=c.data(a,b);if(!d)return f||[];if(!f||c.isArray(d))f=c.data(a,b,c.makeArray(d));else f.push(d);return f}},dequeue:function(a,b){b=b||"fx";var d=c.queue(a,b),f=d.shift();if(f==="inprogress")f=d.shift();if(f){b==="fx"&&d.unshift("inprogress");f.call(a,function(){c.dequeue(a,b)})}}});c.fn.extend({queue:function(a,b){if(typeof a!=="string"){b=a;a="fx"}if(b===v)return c.queue(this[0],a);return this.each(function(){var d=c.queue(this,a,b);a==="fx"&&d[0]!=="inprogress"&&c.dequeue(this,a)})},dequeue:function(a){return this.each(function(){c.dequeue(this,a)})},delay:function(a,b){a=c.fx?c.fx.speeds[a]||a:a;b=b||"fx";return this.queue(b,function(){var d=this;setTimeout(function(){c.dequeue(d,b)},a)})},clearQueue:function(a){return this.queue(a||"fx",[])}});var ya=/[\n\t]/g,ca=/\s+/,Wa=/\r/g,Xa=/href|src|style/,Ya=/(button|input)/i,Za=/(button|input|object|select|textarea)/i,$a=/^(a|area)$/i,za=/radio|checkbox/;c.fn.extend({attr:function(a,b){return X(this,a,b,true,c.attr)},removeAttr:function(a){return this.each(function(){c.attr(this,a,"");this.nodeType===1&&this.removeAttribute(a)})},addClass:function(a){if(c.isFunction(a))return this.each(function(o){var m=c(this);m.addClass(a.call(this,o,m.attr("class")))});if(a&&typeof a==="string")for(var b=(a||"").split(ca),d=0,f=this.length;d-1)return true;return false},val:function(a){if(a===v){var b=this[0];if(b){if(c.nodeName(b,"option"))return(b.attributes.value||{}).specified?b.value:b.text;if(c.nodeName(b,"select")){var d=b.selectedIndex,f=[],e=b.options;b=b.type==="select-one";if(d<0)return null;var i=b?d:0;for(d=b?d+1:e.length;i=0;else if(c.nodeName(this,"select")){var x=c.makeArray(s);c("option",this).each(function(){this.selected=c.inArray(c(this).val(),x)>=0});if(!x.length)this.selectedIndex=-1}else this.value=s}})}});c.extend({attrFn:{val:true,css:true,html:true,text:true,data:true,width:true,height:true,offset:true},attr:function(a,b,d,f){if(!a||a.nodeType===3||a.nodeType===8)return v;if(f&&b in c.attrFn)return c(a)[b](d);f=a.nodeType!==1||!c.isXMLDoc(a);var e=d!==v;b=f&&c.props[b]||b;if(a.nodeType===1){var i=Xa.test(b);if(b in a&&f&&!i){if(e){b==="type"&&Ya.test(a.nodeName)&&a.parentNode&&c.error("type property can't be changed");a[b]=d}if(c.nodeName(a,"form")&&a.getAttributeNode(b))return a.getAttributeNode(b).nodeValue;if(b==="tabIndex")return(b=a.getAttributeNode("tabIndex"))&&b.specified?b.value:Za.test(a.nodeName)||$a.test(a.nodeName)&&a.href?0:v;return a[b]}if(!c.support.style&&f&&b==="style"){if(e)a.style.cssText=""+d;return a.style.cssText}e&&a.setAttribute(b,""+d);a=!c.support.hrefNormalized&&f&&i?a.getAttribute(b,2):a.getAttribute(b);return a===null?v:a}return c.style(a,b,d)}});var ab=function(a){return a.replace(/[^\w\s\.\|`]/g,function(b){return"\\"+b})};c.event={add:function(a,b,d,f){if(!(a.nodeType===3||a.nodeType===8)){if(a.setInterval&&a!==z&&!a.frameElement)a=z;if(!d.guid)d.guid=c.guid++;if(f!==v){d=c.proxy(d);d.data=f}var e=c.data(a,"events")||c.data(a,"events",{}),i=c.data(a,"handle"),j;if(!i){j=function(){return typeof c!=="undefined"&&!c.event.triggered?c.event.handle.apply(j.elem,arguments):v};i=c.data(a,"handle",j)}if(i){i.elem=a;b=b.split(/\s+/);for(var n,o=0;n=b[o++];){var m=n.split(".");n=m.shift();if(o>1){d=c.proxy(d);if(f!==v)d.data=f}d.type=m.slice(0).sort().join(".");var s=e[n],x=this.special[n]||{};if(!s){s=e[n]={};if(!x.setup||x.setup.call(a,f,m,d)===false)if(a.addEventListener)a.addEventListener(n,i,false);else a.attachEvent&&a.attachEvent("on"+n,i)}if(x.add)if((m=x.add.call(a,d,f,m,s))&&c.isFunction(m)){m.guid=m.guid||d.guid;m.data=m.data||d.data;m.type=m.type||d.type;d=m}s[d.guid]=d;this.global[n]=true}a=null}}},global:{},remove:function(a,b,d){if(!(a.nodeType===3||a.nodeType===8)){var f=c.data(a,"events"),e,i,j;if(f){if(b===v||typeof b==="string"&&b.charAt(0)===".")for(i in f)this.remove(a,i+(b||""));else{if(b.type){d=b.handler;b=b.type}b=b.split(/\s+/);for(var n=0;i=b[n++];){var o=i.split(".");i=o.shift();var m=!o.length,s=c.map(o.slice(0).sort(),ab);s=new RegExp("(^|\\.)"+s.join("\\.(?:.*\\.)?")+"(\\.|$)");var x=this.special[i]||{};if(f[i]){if(d){j=f[i][d.guid];delete f[i][d.guid]}else for(var A in f[i])if(m||s.test(f[i][A].type))delete f[i][A];x.remove&&x.remove.call(a,o,j);for(e in f[i])break;if(!e){if(!x.teardown||x.teardown.call(a,o)===false)if(a.removeEventListener)a.removeEventListener(i,c.data(a,"handle"),false);else a.detachEvent&&a.detachEvent("on"+i,c.data(a,"handle"));e=null;delete f[i]}}}}for(e in f)break;if(!e){if(A=c.data(a,"handle"))A.elem=null;c.removeData(a,"events");c.removeData(a,"handle")}}}},trigger:function(a,b,d,f){var e=a.type||a;if(!f){a=typeof a==="object"?a[G]?a:c.extend(c.Event(e),a):c.Event(e);if(e.indexOf("!")>=0){a.type=e=e.slice(0,-1);a.exclusive=true}if(!d){a.stopPropagation();this.global[e]&&c.each(c.cache,function(){this.events&&this.events[e]&&c.event.trigger(a,b,this.handle.elem)})}if(!d||d.nodeType===3||d.nodeType===8)return v;a.result=v;a.target=d;b=c.makeArray(b);b.unshift(a)}a.currentTarget=d;(f=c.data(d,"handle"))&&f.apply(d,b);f=d.parentNode||d.ownerDocument;try{if(!(d&&d.nodeName&&c.noData[d.nodeName.toLowerCase()]))if(d["on"+e]&&d["on"+e].apply(d,b)===false)a.result=false}catch(i){}if(!a.isPropagationStopped()&&f)c.event.trigger(a,b,f,true);else if(!a.isDefaultPrevented()){d=a.target;var j;if(!(c.nodeName(d,"a")&&e==="click")&&!(d&&d.nodeName&&c.noData[d.nodeName.toLowerCase()])){try{if(d[e]){if(j=d["on"+e])d["on"+e]=null;this.triggered=true;d[e]()}}catch(n){}if(j)d["on"+e]=j;this.triggered=false}}},handle:function(a){var b,d;a=arguments[0]=c.event.fix(a||z.event);a.currentTarget=this;d=a.type.split(".");a.type=d.shift();b=!d.length&&!a.exclusive;var f=new RegExp("(^|\\.)"+d.slice(0).sort().join("\\.(?:.*\\.)?")+"(\\.|$)");d=(c.data(this,"events")||{})[a.type];for(var e in d){var i=d[e];if(b||f.test(i.type)){a.handler=i;a.data=i.data;i=i.apply(this,arguments);if(i!==v){a.result=i;if(i===false){a.preventDefault();a.stopPropagation()}}if(a.isImmediatePropagationStopped())break}}return a.result},props:"altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode layerX layerY metaKey newValue offsetX offsetY originalTarget pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target toElement view wheelDelta which".split(" "),fix:function(a){if(a[G])return a;var b=a;a=c.Event(b);for(var d=this.props.length,f;d;){f=this.props[--d];a[f]=b[f]}if(!a.target)a.target=a.srcElement||r;if(a.target.nodeType===3)a.target=a.target.parentNode;if(!a.relatedTarget&&a.fromElement)a.relatedTarget=a.fromElement===a.target?a.toElement:a.fromElement;if(a.pageX==null&&a.clientX!=null){b=r.documentElement;d=r.body;a.pageX=a.clientX+(b&&b.scrollLeft||d&&d.scrollLeft||0)-(b&&b.clientLeft||d&&d.clientLeft||0);a.pageY=a.clientY+(b&&b.scrollTop||d&&d.scrollTop||0)-(b&&b.clientTop||d&&d.clientTop||0)}if(!a.which&&(a.charCode||a.charCode===0?a.charCode:a.keyCode))a.which=a.charCode||a.keyCode;if(!a.metaKey&&a.ctrlKey)a.metaKey=a.ctrlKey;if(!a.which&&a.button!==v)a.which=a.button&1?1:a.button&2?3:a.button&4?2:0;return a},guid:1E8,proxy:c.proxy,special:{ready:{setup:c.bindReady,teardown:c.noop},live:{add:function(a,b){c.extend(a,b||{});a.guid+=b.selector+b.live;b.liveProxy=a;c.event.add(this,b.live,na,b)},remove:function(a){if(a.length){var b=0,d=new RegExp("(^|\\.)"+a[0]+"(\\.|$)");c.each(c.data(this,"events").live||{},function(){d.test(this.type)&&b++});b<1&&c.event.remove(this,a[0],na)}},special:{}},beforeunload:{setup:function(a,b,d){if(this.setInterval)this.onbeforeunload=d;return false},teardown:function(a,b){if(this.onbeforeunload===b)this.onbeforeunload=null}}}};c.Event=function(a){if(!this.preventDefault)return new c.Event(a);if(a&&a.type){this.originalEvent=a;this.type=a.type}else this.type=a;this.timeStamp=J();this[G]=true};c.Event.prototype={preventDefault:function(){this.isDefaultPrevented=Z;var a=this.originalEvent;if(a){a.preventDefault&&a.preventDefault();a.returnValue=false}},stopPropagation:function(){this.isPropagationStopped=Z;var a=this.originalEvent;if(a){a.stopPropagation&&a.stopPropagation();a.cancelBubble=true}},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=Z;this.stopPropagation()},isDefaultPrevented:Y,isPropagationStopped:Y,isImmediatePropagationStopped:Y};var Aa=function(a){for(var b=a.relatedTarget;b&&b!==this;)try{b=b.parentNode}catch(d){break}if(b!==this){a.type=a.data;c.event.handle.apply(this,arguments)}},Ba=function(a){a.type=a.data;c.event.handle.apply(this,arguments)};c.each({mouseenter:"mouseover",mouseleave:"mouseout"},function(a,b){c.event.special[a]={setup:function(d){c.event.add(this,b,d&&d.selector?Ba:Aa,a)},teardown:function(d){c.event.remove(this,b,d&&d.selector?Ba:Aa)}}});if(!c.support.submitBubbles)c.event.special.submit={setup:function(a,b,d){if(this.nodeName.toLowerCase()!=="form"){c.event.add(this,"click.specialSubmit."+d.guid,function(f){var e=f.target,i=e.type;if((i==="submit"||i==="image")&&c(e).closest("form").length)return ma("submit",this,arguments)});c.event.add(this,"keypress.specialSubmit."+d.guid,function(f){var e=f.target,i=e.type;if((i==="text"||i==="password")&&c(e).closest("form").length&&f.keyCode===13)return ma("submit",this,arguments)})}else return false},remove:function(a,b){c.event.remove(this,"click.specialSubmit"+(b?"."+b.guid:""));c.event.remove(this,"keypress.specialSubmit"+(b?"."+b.guid:""))}};if(!c.support.changeBubbles){var da=/textarea|input|select/i;function Ca(a){var b=a.type,d=a.value;if(b==="radio"||b==="checkbox")d=a.checked;else if(b==="select-multiple")d=a.selectedIndex>-1?c.map(a.options,function(f){return f.selected}).join("-"):"";else if(a.nodeName.toLowerCase()==="select")d=a.selectedIndex;return d}function ea(a,b){var d=a.target,f,e;if(!(!da.test(d.nodeName)||d.readOnly)){f=c.data(d,"_change_data");e=Ca(d);if(a.type!=="focusout"||d.type!=="radio")c.data(d,"_change_data",e);if(!(f===v||e===f))if(f!=null||e){a.type="change";return c.event.trigger(a,b,d)}}}c.event.special.change={filters:{focusout:ea,click:function(a){var b=a.target,d=b.type;if(d==="radio"||d==="checkbox"||b.nodeName.toLowerCase()==="select")return ea.call(this,a)},keydown:function(a){var b=a.target,d=b.type;if(a.keyCode===13&&b.nodeName.toLowerCase()!=="textarea"||a.keyCode===32&&(d==="checkbox"||d==="radio")||d==="select-multiple")return ea.call(this,a)},beforeactivate:function(a){a=a.target;a.nodeName.toLowerCase()==="input"&&a.type==="radio"&&c.data(a,"_change_data",Ca(a))}},setup:function(a,b,d){for(var f in T)c.event.add(this,f+".specialChange."+d.guid,T[f]);return da.test(this.nodeName)},remove:function(a,b){for(var d in T)c.event.remove(this,d+".specialChange"+(b?"."+b.guid:""),T[d]);return da.test(this.nodeName)}};var T=c.event.special.change.filters}r.addEventListener&&c.each({focus:"focusin",blur:"focusout"},function(a,b){function d(f){f=c.event.fix(f);f.type=b;return c.event.handle.call(this,f)}c.event.special[b]={setup:function(){this.addEventListener(a,d,true)},teardown:function(){this.removeEventListener(a,d,true)}}});c.each(["bind","one"],function(a,b){c.fn[b]=function(d,f,e){if(typeof d==="object"){for(var i in d)this[b](i,f,d[i],e);return this}if(c.isFunction(f)){e=f;f=v}var j=b==="one"?c.proxy(e,function(n){c(this).unbind(n,j);return e.apply(this,arguments)}):e;return d==="unload"&&b!=="one"?this.one(d,f,e):this.each(function(){c.event.add(this,d,j,f)})}});c.fn.extend({unbind:function(a,b){if(typeof a==="object"&&!a.preventDefault){for(var d in a)this.unbind(d,a[d]);return this}return this.each(function(){c.event.remove(this,a,b)})},trigger:function(a,b){return this.each(function(){c.event.trigger(a,b,this)})},triggerHandler:function(a,b){if(this[0]){a=c.Event(a);a.preventDefault();a.stopPropagation();c.event.trigger(a,b,this[0]);return a.result}},toggle:function(a){for(var b=arguments,d=1;d0){y=t;break}}t=t[g]}l[q]=y}}}var f=/((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^[\]]*\]|['"][^'"]*['"]|[^[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g,e=0,i=Object.prototype.toString,j=false,n=true;[0,0].sort(function(){n=false;return 0});var o=function(g,h,k,l){k=k||[];var q=h=h||r;if(h.nodeType!==1&&h.nodeType!==9)return[];if(!g||typeof g!=="string")return k;for(var p=[],u,t,y,R,H=true,M=w(h),I=g;(f.exec(""),u=f.exec(I))!==null;){I=u[3];p.push(u[1]);if(u[2]){R=u[3];break}}if(p.length>1&&s.exec(g))if(p.length===2&&m.relative[p[0]])t=fa(p[0]+p[1],h);else for(t=m.relative[p[0]]?[h]:o(p.shift(),h);p.length;){g=p.shift();if(m.relative[g])g+=p.shift();t=fa(g,t)}else{if(!l&&p.length>1&&h.nodeType===9&&!M&&m.match.ID.test(p[0])&&!m.match.ID.test(p[p.length-1])){u=o.find(p.shift(),h,M);h=u.expr?o.filter(u.expr,u.set)[0]:u.set[0]}if(h){u=l?{expr:p.pop(),set:A(l)}:o.find(p.pop(),p.length===1&&(p[0]==="~"||p[0]==="+")&&h.parentNode?h.parentNode:h,M);t=u.expr?o.filter(u.expr,u.set):u.set;if(p.length>0)y=A(t);else H=false;for(;p.length;){var D=p.pop();u=D;if(m.relative[D])u=p.pop();else D="";if(u==null)u=h;m.relative[D](y,u,M)}}else y=[]}y||(y=t);y||o.error(D||g);if(i.call(y)==="[object Array]")if(H)if(h&&h.nodeType===1)for(g=0;y[g]!=null;g++){if(y[g]&&(y[g]===true||y[g].nodeType===1&&E(h,y[g])))k.push(t[g])}else for(g=0;y[g]!=null;g++)y[g]&&y[g].nodeType===1&&k.push(t[g]);else k.push.apply(k,y);else A(y,k);if(R){o(R,q,k,l);o.uniqueSort(k)}return k};o.uniqueSort=function(g){if(C){j=n;g.sort(C);if(j)for(var h=1;h":function(g,h){var k=typeof h==="string";if(k&&!/\W/.test(h)){h=h.toLowerCase();for(var l=0,q=g.length;l=0))k||l.push(u);else if(k)h[p]=false;return false},ID:function(g){return g[1].replace(/\\/g,"")},TAG:function(g){return g[1].toLowerCase()},CHILD:function(g){if(g[1]==="nth"){var h=/(-?)(\d*)n((?:\+|-)?\d*)/.exec(g[2]==="even"&&"2n"||g[2]==="odd"&&"2n+1"||!/\D/.test(g[2])&&"0n+"+g[2]||g[2]);g[2]=h[1]+(h[2]||1)-0;g[3]=h[3]-0}g[0]=e++;return g},ATTR:function(g,h,k,l,q,p){h=g[1].replace(/\\/g,"");if(!p&&m.attrMap[h])g[1]=m.attrMap[h];if(g[2]==="~=")g[4]=" "+g[4]+" ";return g},PSEUDO:function(g,h,k,l,q){if(g[1]==="not")if((f.exec(g[3])||"").length>1||/^\w/.test(g[3]))g[3]=o(g[3],null,null,h);else{g=o.filter(g[3],h,k,true^q);k||l.push.apply(l,g);return false}else if(m.match.POS.test(g[0])||m.match.CHILD.test(g[0]))return true;return g},POS:function(g){g.unshift(true);return g}},filters:{enabled:function(g){return g.disabled===false&&g.type!=="hidden"},disabled:function(g){return g.disabled===true},checked:function(g){return g.checked===true},selected:function(g){return g.selected===true},parent:function(g){return!!g.firstChild},empty:function(g){return!g.firstChild},has:function(g,h,k){return!!o(k[3],g).length},header:function(g){return/h\d/i.test(g.nodeName)},text:function(g){return"text"===g.type},radio:function(g){return"radio"===g.type},checkbox:function(g){return"checkbox"===g.type},file:function(g){return"file"===g.type},password:function(g){return"password"===g.type},submit:function(g){return"submit"===g.type},image:function(g){return"image"===g.type},reset:function(g){return"reset"===g.type},button:function(g){return"button"===g.type||g.nodeName.toLowerCase()==="button"},input:function(g){return/input|select|textarea|button/i.test(g.nodeName)}},setFilters:{first:function(g,h){return h===0},last:function(g,h,k,l){return h===l.length-1},even:function(g,h){return h%2===0},odd:function(g,h){return h%2===1},lt:function(g,h,k){return hk[3]-0},nth:function(g,h,k){return k[3]-0===h},eq:function(g,h,k){return k[3]-0===h}},filter:{PSEUDO:function(g,h,k,l){var q=h[1],p=m.filters[q];if(p)return p(g,k,h,l);else if(q==="contains")return(g.textContent||g.innerText||a([g])||"").indexOf(h[3])>=0;else if(q==="not"){h=h[3];k=0;for(l=h.length;k=0}},ID:function(g,h){return g.nodeType===1&&g.getAttribute("id")===h},TAG:function(g,h){return h==="*"&&g.nodeType===1||g.nodeName.toLowerCase()===h},CLASS:function(g,h){return(" "+(g.className||g.getAttribute("class"))+" ").indexOf(h)>-1},ATTR:function(g,h){var k=h[1];g=m.attrHandle[k]?m.attrHandle[k](g):g[k]!=null?g[k]:g.getAttribute(k);k=g+"";var l=h[2];h=h[4];return g==null?l==="!=":l==="="?k===h:l==="*="?k.indexOf(h)>=0:l==="~="?(" "+k+" ").indexOf(h)>=0:!h?k&&g!==false:l==="!="?k!==h:l==="^="?k.indexOf(h)===0:l==="$="?k.substr(k.length-h.length)===h:l==="|="?k===h||k.substr(0,h.length+1)===h+"-":false},POS:function(g,h,k,l){var q=m.setFilters[h[2]];if(q)return q(g,k,h,l)}}},s=m.match.POS;for(var x in m.match){m.match[x]=new RegExp(m.match[x].source+/(?![^\[]*\])(?![^\(]*\))/.source);m.leftMatch[x]=new RegExp(/(^(?:.|\r|\n)*?)/.source+m.match[x].source.replace(/\\(\d+)/g,function(g,h){return"\\"+(h-0+1)}))}var A=function(g,h){g=Array.prototype.slice.call(g,0);if(h){h.push.apply(h,g);return h}return g};try{Array.prototype.slice.call(r.documentElement.childNodes,0)}catch(B){A=function(g,h){h=h||[];if(i.call(g)==="[object Array]")Array.prototype.push.apply(h,g);else if(typeof g.length==="number")for(var k=0,l=g.length;k";var k=r.documentElement;k.insertBefore(g,k.firstChild);if(r.getElementById(h)){m.find.ID=function(l,q,p){if(typeof q.getElementById!=="undefined"&&!p)return(q=q.getElementById(l[1]))?q.id===l[1]||typeof q.getAttributeNode!=="undefined"&&q.getAttributeNode("id").nodeValue===l[1]?[q]:v:[]};m.filter.ID=function(l,q){var p=typeof l.getAttributeNode!=="undefined"&&l.getAttributeNode("id");return l.nodeType===1&&p&&p.nodeValue===q}}k.removeChild(g);k=g=null})();(function(){var g=r.createElement("div");g.appendChild(r.createComment(""));if(g.getElementsByTagName("*").length>0)m.find.TAG=function(h,k){k=k.getElementsByTagName(h[1]);if(h[1]==="*"){h=[];for(var l=0;k[l];l++)k[l].nodeType===1&&h.push(k[l]);k=h}return k};g.innerHTML="";if(g.firstChild&&typeof g.firstChild.getAttribute!=="undefined"&&g.firstChild.getAttribute("href")!=="#")m.attrHandle.href=function(h){return h.getAttribute("href",2)};g=null})();r.querySelectorAll&&function(){var g=o,h=r.createElement("div");h.innerHTML="

      ";if(!(h.querySelectorAll&&h.querySelectorAll(".TEST").length===0)){o=function(l,q,p,u){q=q||r;if(!u&&q.nodeType===9&&!w(q))try{return A(q.querySelectorAll(l),p)}catch(t){}return g(l,q,p,u)};for(var k in g)o[k]=g[k];h=null}}();(function(){var g=r.createElement("div");g.innerHTML="
      ";if(!(!g.getElementsByClassName||g.getElementsByClassName("e").length===0)){g.lastChild.className="e";if(g.getElementsByClassName("e").length!==1){m.order.splice(1,0,"CLASS");m.find.CLASS=function(h,k,l){if(typeof k.getElementsByClassName!=="undefined"&&!l)return k.getElementsByClassName(h[1])};g=null}}})();var E=r.compareDocumentPosition?function(g,h){return g.compareDocumentPosition(h)&16}:function(g,h){return g!==h&&(g.contains?g.contains(h):true)},w=function(g){return(g=(g?g.ownerDocument||g:0).documentElement)?g.nodeName!=="HTML":false},fa=function(g,h){var k=[],l="",q;for(h=h.nodeType?[h]:h;q=m.match.PSEUDO.exec(g);){l+=q[0];g=g.replace(m.match.PSEUDO,"")}g=m.relative[g]?g+"*":g;q=0;for(var p=h.length;q=0===d})};c.fn.extend({find:function(a){for(var b=this.pushStack("","find",a),d=0,f=0,e=this.length;f0)for(var i=d;i0},closest:function(a,b){if(c.isArray(a)){var d=[],f=this[0],e,i={},j;if(f&&a.length){e=0;for(var n=a.length;e-1:c(f).is(e)){d.push({selector:j,elem:f});delete i[j]}}f=f.parentNode}}return d}var o=c.expr.match.POS.test(a)?c(a,b||this.context):null;return this.map(function(m,s){for(;s&&s.ownerDocument&&s!==b;){if(o?o.index(s)>-1:c(s).is(a))return s;s=s.parentNode}return null})},index:function(a){if(!a||typeof a==="string")return c.inArray(this[0],a?c(a):this.parent().children());return c.inArray(a.jquery?a[0]:a,this)},add:function(a,b){a=typeof a==="string"?c(a,b||this.context):c.makeArray(a);b=c.merge(this.get(),a);return this.pushStack(pa(a[0])||pa(b[0])?b:c.unique(b))},andSelf:function(){return this.add(this.prevObject)}});c.each({parent:function(a){return(a=a.parentNode)&&a.nodeType!==11?a:null},parents:function(a){return c.dir(a,"parentNode")},parentsUntil:function(a,b,d){return c.dir(a,"parentNode",d)},next:function(a){return c.nth(a,2,"nextSibling")},prev:function(a){return c.nth(a,2,"previousSibling")},nextAll:function(a){return c.dir(a,"nextSibling")},prevAll:function(a){return c.dir(a,"previousSibling")},nextUntil:function(a,b,d){return c.dir(a,"nextSibling",d)},prevUntil:function(a,b,d){return c.dir(a,"previousSibling",d)},siblings:function(a){return c.sibling(a.parentNode.firstChild,a)},children:function(a){return c.sibling(a.firstChild)},contents:function(a){return c.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:c.makeArray(a.childNodes)}},function(a,b){c.fn[a]=function(d,f){var e=c.map(this,b,d);bb.test(a)||(f=d);if(f&&typeof f==="string")e=c.filter(f,e);e=this.length>1?c.unique(e):e;if((this.length>1||db.test(f))&&cb.test(a))e=e.reverse();return this.pushStack(e,a,Q.call(arguments).join(","))}});c.extend({filter:function(a,b,d){if(d)a=":not("+a+")";return c.find.matches(a,b)},dir:function(a,b,d){var f=[];for(a=a[b];a&&a.nodeType!==9&&(d===v||a.nodeType!==1||!c(a).is(d));){a.nodeType===1&&f.push(a);a=a[b]}return f},nth:function(a,b,d){b=b||1;for(var f=0;a;a=a[d])if(a.nodeType===1&&++f===b)break;return a},sibling:function(a,b){for(var d=[];a;a=a.nextSibling)a.nodeType===1&&a!==b&&d.push(a);return d}});var Fa=/ jQuery\d+="(?:\d+|null)"/g,V=/^\s+/,Ga=/(<([\w:]+)[^>]*?)\/>/g,eb=/^(?:area|br|col|embed|hr|img|input|link|meta|param)$/i,Ha=/<([\w:]+)/,fb=/"},F={option:[1,""],legend:[1,"
      ","
      "],thead:[1,"","
      "],tr:[2,"","
      "],td:[3,"","
      "],col:[2,"","
      "],area:[1,"",""],_default:[0,"",""]};F.optgroup=F.option;F.tbody=F.tfoot=F.colgroup=F.caption=F.thead;F.th=F.td;if(!c.support.htmlSerialize)F._default=[1,"div
      ","
      "];c.fn.extend({text:function(a){if(c.isFunction(a))return this.each(function(b){var d=c(this);d.text(a.call(this,b,d.text()))});if(typeof a!=="object"&&a!==v)return this.empty().append((this[0]&&this[0].ownerDocument||r).createTextNode(a));return c.getText(this)},wrapAll:function(a){if(c.isFunction(a))return this.each(function(d){c(this).wrapAll(a.call(this,d))});if(this[0]){var b=c(a,this[0].ownerDocument).eq(0).clone(true);this[0].parentNode&&b.insertBefore(this[0]);b.map(function(){for(var d=this;d.firstChild&&d.firstChild.nodeType===1;)d=d.firstChild;return d}).append(this)}return this},wrapInner:function(a){if(c.isFunction(a))return this.each(function(b){c(this).wrapInner(a.call(this,b))});return this.each(function(){var b=c(this),d=b.contents();d.length?d.wrapAll(a):b.append(a)})},wrap:function(a){return this.each(function(){c(this).wrapAll(a)})},unwrap:function(){return this.parent().each(function(){c.nodeName(this,"body")||c(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.appendChild(a)})},prepend:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.insertBefore(a,this.firstChild)})},before:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this)});else if(arguments.length){var a=c(arguments[0]);a.push.apply(a,this.toArray());return this.pushStack(a,"before",arguments)}},after:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this.nextSibling)});else if(arguments.length){var a=this.pushStack(this,"after",arguments);a.push.apply(a,c(arguments[0]).toArray());return a}},clone:function(a){var b=this.map(function(){if(!c.support.noCloneEvent&&!c.isXMLDoc(this)){var d=this.outerHTML,f=this.ownerDocument;if(!d){d=f.createElement("div");d.appendChild(this.cloneNode(true));d=d.innerHTML}return c.clean([d.replace(Fa,"").replace(V,"")],f)[0]}else return this.cloneNode(true)});if(a===true){qa(this,b);qa(this.find("*"),b.find("*"))}return b},html:function(a){if(a===v)return this[0]&&this[0].nodeType===1?this[0].innerHTML.replace(Fa,""):null;else if(typeof a==="string"&&!/ + + {% if request.user.is_authenticated %} + + {% endif %} +{% endblock %} + +{% block header %} + + {% include 'includes/header.html' %} + + + + + +
      + {% block search %} + {% include 'includes/search.html' %} + {% endblock %} +
      + +{% endblock %} + +{% block login_area %} + {% include 'includes/login_area.html' %} +{% endblock %} + +{% block content_head %} +

      {% block title %}??? content_head ???{% endblock %}

      {% block tryit %}{#
      TRY IT OUT
      #}{% endblock %} +{% endblock %} + +{% block content_body %}{% endblock %} + +{% block footer %} + {% include 'includes/footer.html' %} +{% endblock %} diff --git a/storefront/templates/change_password.html b/storefront/templates/change_password.html new file mode 100644 index 0000000..80f822c --- /dev/null +++ b/storefront/templates/change_password.html @@ -0,0 +1,33 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block title %}Password change{% endblock %}endblock %} + +{% block common_content %} + {% box 'form' %} +

      Change your password

      +

      You need to enter your current password again to confirm that you want to change it.

      +
      +
      + {% for f in form %} +
      + +
      +
      {% for e in f.errors %}{{e}} {% endfor %}
      +
      + {% endfor %} + {% if message %} +
      {{ message }}
      + {% else %} + {% endif %} +
      + +
      +
      + {% endbox %} +{% endblock %} + diff --git a/storefront/templates/close_account.html b/storefront/templates/close_account.html new file mode 100644 index 0000000..977e963 --- /dev/null +++ b/storefront/templates/close_account.html @@ -0,0 +1,48 @@ +{% extends "base.html" %} +{% load custom_tags %} + +{% block content %} +
      +
      +

      Closing account

      + +

      You are about to CLOSE your IndexTank account. Closing the account implies deleting all indexes linked to it and results + in losing all the information within them.

      +

      You need to enter your password in order to take this action.

      +
      + +
      + + + {% for f in form %} + + + + + {% endfor %} + {% if message %} + + + + + {% else %} + {% endif %} + + + + + +
      {{ f.label_tag }}{{ f }} +
      {% for e in f.errors %}{{e}} {% endfor %}
      +
      + {{ message }} +
      + CLOSE ACCOUNT +
      +
      +
      +
      + +
      +{% endblock %} + diff --git a/storefront/templates/coming-soon.html b/storefront/templates/coming-soon.html new file mode 100644 index 0000000..df6dafa --- /dev/null +++ b/storefront/templates/coming-soon.html @@ -0,0 +1,11 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block title %}Coming soon!{% endblock %}endblock %} + +{% block common_content %} + +

      This page is not ready yet. We are working on it and it should be available to you soon.

      +

      Go back to the homepage or read our documentation to continue.

      +       +{% endblock %} diff --git a/storefront/templates/common-base.html b/storefront/templates/common-base.html new file mode 100644 index 0000000..b39668c --- /dev/null +++ b/storefront/templates/common-base.html @@ -0,0 +1,23 @@ +{% extends "base.html" %} + +{% load messages %} + +{% block content_body %} +
      +
      +
      +
      +
      + {% render_messages messages %} + {% block common_content %} + {% endblock %} +
      +
      + {% block right_content %} + {% endblock %} +
      +
      +
      +
      +
      +{% endblock %} diff --git a/storefront/templates/dashboard.html b/storefront/templates/dashboard.html new file mode 100644 index 0000000..62a2a52 --- /dev/null +++ b/storefront/templates/dashboard.html @@ -0,0 +1,160 @@ +{% extends "common-base.html" %} +{% load custom_tags %} +{% load humanize %} + +{% block extrahead %} + {{ block.super }} + + +{% endblock %} + +{% block container_class %}{{ block.super }} dashboard wide{% endblock %} + +{% block title %}My Dashboard{% endblock %} + +{% block common_content %} + {% if account_status == 'NOINDEX' %} + + {% box 'largebox' %} +
      No indexes have been created yet
      +

      + Please download one of our client libraries to access the + IndexTank API. +

      +
      + + + +
      +
      If none of the libraries fit your needs, you can develop your own based on the API specification.
      +
      +

      + The next step is to create your first index. +

      +
      +

      + Note: you can also create indexes through our API, with a client or + with your own custom code. +

      +
      + {% endbox %} + {% else %} + + + + {% for index in indexes %} + {% box 'largebox' %} +
      +
      + {% if index.is_demo %} + Live Demo +  |  + {% endif%} + Search +  |  + Manage +  |  + {% if index.is_ready %} + {% if index.is_populating %} + Populating + {% else %} + Running + {% endif %} + {% else %} + {% if index.is_hibernated %} + Hibernated + {% else %} + Starting + {% endif %} + {% endif %} +
      +

      + {{ index.name }} (created at {{ index.creation_time|date:"g a, F j, Y" }}) +

      +
      +
      +

      Current Size: {{ index.current_docs_number|intcomma }} document{{ index.current_docs_number|pluralize }}

      +

      + + + + +
      + {% endbox %} + {% endfor %} + {% endif %} + +
      + + +
      +

      + Private URL +

      +

      + {% with account.get_private_apiurl as text %}{% with "#f5f5f5" as bgcolor %}{% with "1" as small %}{% include 'includes/clippy.html' %}{% endwith %}{% endwith %}{% endwith %} + {{ request.user.get_profile.account.get_private_apiurl }} +

      +

      + This is the base url for authenticated API calls. The easiest way to access our + API is instantiating one of our clients with this URL. +

      + + +

      + Public URL +

      +

      + {% with account.get_public_apiurl as text %}{% with "#f5f5f5" as bgcolor %}{% with "1" as small %}{% include 'includes/clippy.html' %}{% endwith %}{% endwith %}{% endwith %} + {{ request.user.get_profile.account.get_public_apiurl }} +

      +

      + This url gives you access to the public methods of the API, such as Autocomplete. +

      + + +

      Current plan: {{ account.package.name }}

      +
        +
      • Up to {{ account.package.max_indexes|apnumber }} index{{ account.package.max_indexes|pluralize:"es" }}.
        • +
      • Up to {{ account.package.index_max_size|intcomma }} documents.
        • +
      • Unlimited daily queries!
        • +
      • {% if account.package.code == 'FREE' %}"Powered by IndexTank" required in production{% else %}White label service{% endif %}
        • +
      +
      +

      To upgrade your account, please contact us.

      +
      + +
      + {% box 'chat' %} +

      HAVE A QUESTION?

      +
      LIVE CHAT NOW
      +

      Our team of experienced and friendly devs will respond almost as quickly as our search API.

      + {% endbox %} +
      + +{% endblock %} + diff --git a/storefront/templates/documentation/api.html b/storefront/templates/documentation/api.html new file mode 100644 index 0000000..91c6096 --- /dev/null +++ b/storefront/templates/documentation/api.html @@ -0,0 +1,1043 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block container_class %}{{ block.super }} documentation wide{% endblock %} + +{% block right_content %} +{% endblock %} + +{% block title %}IndexTank API{% endblock %} + +{% block common_content %} + + + +
      +

      The API defines methods to manage indexes (create and delete them), operate on them (add and delete documents, functions, etc), perform searches, etc.

      +

      All calls should be made to your specific private API url (which you'll find in your account's dashboard).

      + +

      Every call will report its success via the HTTP status code returned. If the status code is 4xx or 5xx then the response body may contain a plain error message providing further details. + If a 2xx code is returned the body will either be a JSON-serialized object or it'll be empty depending on the call. +

      +
      + +
      + +
      +

      Index Management

      +

      Before using an index you need to create an index and assign it a name. This name (NOT its code) has to be provided at creation time, and is used to reference that index in all other API calls.

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      HTTP Request / DescriptionHTTP Response codes / body
      +
      GET /v1/indexes
      +
      + Retrieves the metadata of every index in this account. +
      +       		+
      200 OK
      +
      + The response body will contain a JSON map.
      + Each key will be an index name, and its value, the index metadata. More details... +
      +
      +
      GET /v1/indexes/name
      +
      + Retrieves metadata for the index name.
      +
      +       		+
      200 OK
      +
      + The response body will contain a JSON map with metadata for the index. More details... +
      +
      +
      404 No index existed for the given name
      +
      + The response body will be empty. +
      +
      +
      PUT /v1/indexes/name
      +
      + Creates or updates an index with the given name.
      + It cannot contain forward slashes "/".

      + + The request body can contain a JSON object with:
      +
        +
      • "public_search": a boolean indicating
        + whether public API is enable
        • +
      +
      +       		+
      201 An index has been created
      +
      + The response body will contain a JSON map with metadata for the index. More details... +
      +
      +
      204 An index already existed for that name
      +
      + If body was sent, the index will be modified according to the options sent.
      + The response body will be empty. +
      +
      +
      409 Too many indexes for this account
      +
      + A descriptive error message will be found in the body. +
      +
      +
      DELETE /v1/indexes/name
      +
      + Removes the index name from the account. +
      +       		+
      200 OK
      +
      + The response body will be empty. +
      +
      +
      204 No index existed for that name
      +
      + The response body will be empty. +
      +
      + +
      + +

      Indexing

      + +

      Documents can be added, updated and deleted from the index. An identifier must be provided for each document. It can be your internal identifier for that document.

      +

      This identifier will be used in the search results to reference the matching documents. Every field defined in the indexed document will be searcheable. The indexed fields can be fetched or snippeted along with the search results.

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      HTTP Request / DescriptionHTTP Response codes / body
      +
      PUT /v1/indexes/name/docs
      +
      + Adds a document to the index name.
      +
      + The request body should contain a JSON object with:
      +
        +
      • "docid": your document identifier
        A non-empty String no longer than 1024 bytes
        • +
      • "fields": a map from field name to field value.
        The sum of the length of each field value MUST not be greater than 100kbytes
        • +
      + And optionally:
      +
        +
      • "variables": a map from the var number to float
        • +
      • "categories": a map from the category name to its value
        • +
      + +
      +       		+
      200 OK - Document indexed
      +
      + The response body will be empty. +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + Until the value for "started" in the metadata is true attempts to add a document will fail. +
      +
      +
      503 Service Unavailable
      +
      + A descriptive error message will be found in the body. +
      +
      +
      PUT /v1/indexes/name/docs
      +
      + Adds a batch of documents to the index name.
      +
      + The request body should contain a JSON list where each element should have the following attributes:
      +
        +
      • "docid": your document identifier
        A non-empty string no longer than 1024 bytes
        • +
      • "fields": a map from field name to field value.
        The sum of the length of each field value MUST not be greater than 100kbytes
        • +
      + And optionally:
      +
        +
      • "variables": a map from the var number to float
        • +
      • "categories": a map from the category name to its value
        • +
      + +
      +       		+
      200 OK - Batch processed
      +
      + The response body will be a JSON list where each element will have the following attributes: +
        +
      • "added": a boolean indicating whether the document in this position was successfully indexed
        • +
      • "error": a message detailing the reasons why a document was not successfully indexed
        • +
      +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body.
      + If any of the documents in the list was malformed, this response will be given and no document will be indexed. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + Until the value for "started" in the metadata is true attempts to add a document will fail. +
      +
      +
      503 Service Unavailable
      +
      + A descriptive error message will be found in the body. +
      +
      +
      DELETE /v1/indexes/name/docs
      +
      + Removes a document from the index name.
      +
      + The request body should contain a JSON object with:
      +
        +
      • "docid": the document identifier
        • +
      + For clients that do not support body in DELETE
      + requests the docid can be sent in the querystring + +
      +       		+
      200 OK - Document deleted
      +
      + The response body will be empty. +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + Until the value for "started" in the metadata is true attempts to delete a document will fail. +
      +
      +
      503 Service Unavailable
      +
      + A descriptive error message will be found in the body. +
      +
      +
      DELETE /v1/indexes/name/docs
      +
      + Removes a bulk of documents from the index name.
      +
      + The request body should contain a JSON List where each element should have the following attributes:
      +
        +
      • "docid": the document identifier
        • +
      + For clients that do not support body in DELETE
      + requests the docid can be sent in the querystring one time per document. + +
      +       		+
      200 OK - Bulk delete processed
      +
      + The response body will be a JSON list where each element will have the following attributes: +
        +
      • "deleted": a boolean indicating whether the document in this position was successfully deleted
        • +
      • "error": a message detailing the reasons why a document was not successfully deleted
        • +
      +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + Until the value for "started" in the metadata is true attempts to delete a document will fail. +
      +
      +
      503 Service Unavailable
      +
      + A descriptive error message will be found in the body. +
      +
      + +
      + +

      Scoring variables

      + +

      + Documents can have numeric variables attached to them. These variables can be used in scoring functions for sorting search results. + Variables can be updated rapidly; these updates don't count towards your indexing limits. +

      + + + + + + + + + + + + + + + + + + + + + +
      HTTP Request / DescriptionHTTP Response codes / body
      +
      PUT /v1/indexes/name/docs/variables
      +
      + Update the variables of a document in index name.
      +
      + The request body should contain a JSON object with:
      +
        +
      • "docid": your document identifier
        • +
      • "variables": a map from the var number to float
        • +
      + +
      +       		+
      200 OK - Variables indexed
      +
      + The response body will be empty. +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + Until the value for "started" in the metadata is true attempts to update variables will fail. +
      +
      +
      503 Service Unavailable
      +
      + A descriptive error message will be found in the body. +
      +
      + +
      + + +

      Categories

      + +

      Documents already added can be categorized. Categories are a way to partition your index for different dimensions. For instance, you may want to have your documents + grouped by type (cds, books), price range (0-99, 100-199), manufacturer, author, or any dimension in which your documents have meaning. For every category (the dimension) + every document can have at most one value. +

      +

      These categories can be used later to filter your query (e.g.: only return books that cost between 0 and 100 dollars) or to facet the results of a query (i.e. + showing the user how many documents, for each category and category value, match the query) +

      + + + + + + + + + + + + + + + + + + + + + + + +
      HTTP Request / DescriptionHTTP Response codes / body
      +
      PUT /v1/indexes/name/docs/categories
      +
      + Update the categories of a document in index name.
      +
      + The request body should contain a JSON object with:
      +
        +
      • "docid": your document identifier
        • +
      • "categories": a map from the categories' names
        to the values for this document
        • +
      + +
      +       		+
      200 OK - Categories indexed
      +
      + The response body will be empty. +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + Until the value for "started" in the metadata is true attempts to update categories will fail. +
      +
      +
      503 Service Unavailable
      +
      + A descriptive error message will be found in the body. +
      +
      + + +
      + +

      Scoring functions

      + +
      +

      Scoring functions can be defined through the API. These functions can later be used when searching the index + to provide specific orderings for the results. They can use the variables defined in the document as well + as some special variables such as the document's age and the textual relevance of the match. You can find more detail + in the function definition syntax page.

      +

      Geolocation can be used to prioritize documents closer to (or further from) the user's location. + Check the distance functions for this.

      +

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      HTTP Request / DescriptionHTTP Response codes / body
      +
      GET /v1/indexes/name/functions
      +
      + Retrieves all the functions defined for the index name.
      +
      +       		+
      200 OK
      +
      + The response body will contain a JSON map from the function number to its definition. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + This will happen until the value for "started" in the metadata is true. +
      +
      +
      PUT /v1/indexes/name/functions/num
      +
      + Defines the function number num for the index name.
      +
      + The request body should contain a JSON object with:
      +
        +
      • "definition": the formula that defines the function
        • +
      +
      +       		+
      200 OK - Function saved
      +
      + The response body will be empty. +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + This will happen until the value for "started" in the metadata is true. +
      +
      +
      DELETE /v1/indexes/name/functions/num
      +
      + Removes the function num from the index name.
      +
      +       		+
      200 OK - Function deleted
      +
      + The response body will be empty. +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + This will happen until the value for "started" in the metadata is true. +
      +
      +
      503 Service Unavailable
      +
      + A descriptive error message will be found in the body. +
      +
      + +
      + +

      Searching

      + +

      A query must be provided to search an index. Optional parameters include: start and length (for results paging), a scoring function (for results ordering), and a list of fetch and snippet fields (for getting stored data).

      +

      Search results will contain the document identifiers provided at indexing time. If fetch and snippet fields were specified, the field's content or a snippet of the content can be returned along with the identifiers.

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      HTTP Request / DescriptionHTTP Response codes / body
      +
      GET /v1/indexes/name/search
      +
      + Performs a search on the index name.
      +
      + The querystring should contain:
      +
        +
      • "q": the query to be performed
        • +
      + And optionally:
      +
        +
      • "start": for paging, the first position to return
        • +
      • "len": how many results to return (default: 10)
        • +
      • "function": the number of the scoring function to use (default: 0)
        • +
      • "fetch": comma-separated list of fields to fetch. '*' returns all present fields for each document1
        • +
      • "fetch_variables": 'true' returns all variables for each document as variable_<N> (unset vars return 0)
        • +
      • "fetch_categories": 'true' returns all categories for each document as category_<NAME>
        • +
      • "snippet": comma-separated list of fields to snippet 1
        • +
      • "var<N>": value of the query variable <N> 2
        • +
      • "category_filters": a json map from category name
        + to a list of the admitted values for those categories 3
        • +
      • "filter_docvar<N>": comma-separated list of
        + ranges to filter the values of variable <N>. Each range is
        + expressed as BOTTOM:TOP, where any of both limits can
        + be replaced by an * symbol to indicate it should be ignored
        • + +
      • "filter_function<N>": comma-separated list of
        + ranges to filter the values of function <N>. Each range is
        + expressed as BOTTOM:TOP, where any of both limits can
        + be replaced by an * symbol to indicate it should be ignored
        • + +
      +
      +       		+
      200 OK
      +
      + The response body will contain a JSON map with these fields: +
        +
      • "matches": the total number of matches for the query
        • +
      • "facets": a map from category name to a values count map
        • +
      • "results": a list of objects with the "docid" field
          +
        • query_relevance_score: query specific document relevance score
          • +
        • variable_<N>: variable value, from 0 to N
          • +
        • category_<NAME>: category value for the NAME document category / facet
          • +
        • +
      • "search_time": the time it took to search in seconds
        • +
      +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + This will happen until the value for "started" in the metadata is true. +
      +
      +
      503 Service Unavailable
      +
      + A descriptive error message will be found in the body. +
      +
      +
      DELETE /v1/indexes/name/search
      +
      + Performs a search on the index name.
      +
      + The querystring should contain:
      +
        +
      • "q": the query to be performed
        • +
      + And optionally:
      +
        +
      • "start": the first result to delete
        • +
      • "function": the number of the scoring function to use (default: 0) (only relevant if 'start' is being used)
        • +
      • "var<N>": value of the query variable <N> 2 (only relevant if 'start' is being used)
        • +
      • "category_filters": a json map from category name
        + to a list of the admitted values for those categories 3
        • +
      • "filter_docvar<N>": comma-separated list of
        + ranges to filter the values of variable <N>. Each range is
        + expressed as BOTTOM:TOP, where any of both limits can
        + be replaced by an * symbol to indicate it should be ignored
        • + +
      • "filter_function<N>": comma-separated list of
        + ranges to filter the values of function <N>. Each range is
        + expressed as BOTTOM:TOP, where any of both limits can
        + be replaced by an * symbol to indicate it should be ignored
        • + +
      +
      +       		+
      200 OK
      +
      + The response body will be empty. +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + This will happen until the value for "started" in the metadata is true. +
      +
      +
      503 Service Unavailable
      +
      + A descriptive error message will be found in the body. +
      +
      + + +

      Promoting results

      + +

      The API also allows you to promote a document to the top of a query's result page

      + + + + + + + + + + + + + + + + + + + + + +
      HTTP Request / DescriptionHTTP Response codes / body
      +
      PUT /v1/indexes/name/promote
      +
      + Promotes a document for a query on the index name.
      +
      + The request body should contain a JSON object with:
      +
        +
      • "docid": the id of the document to promote
        • +
      • "query": the query to in which to promote it
        • +
      +
      +       		+
      200 OK - The document has been promoted
      +
      + The response body will be empty. +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body. +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body. +
      +
      +
      409 The index was initializing
      +
      + This will happen until the value for "started" in the metadata is true. +
      +
      +
      503 Service Unavailable
      +
      + A descriptive error message will be found in the body. +
      +
      + +

      Index metadata

      + +

      Some methods will return metadata for an index. This metadata will be in the form of a JSON object and will include the following fields:

      + + + + + + + + + + + + + + + + + + + + + + + + + +
      Field nameContent
      +
      started
      +       		+
      boolean true / false
      +
      + A boolean value representing whether the given index has started.
      + False usually means that the index has been recently it created and isn't available yet. +
      +
      +
      code
      +       		+
      string alphanumeric
      +
      + A string with an alphanumeric code that uniquely identifies the index under the given name.
      + If an index is deleted and a new one is created with the same name, it will have a different code. +
      +
      +
      creation_time
      +       		+
      string ISO 8601 formatted date format
      +
      + A string with the time when this index was created formatted according to the ISO 8601 format.
      + For example: 2010-01-01T12:00:00 +
      +
      +
      size
      +       		+
      integer
      +
      + An integer with the size in documents of this index. The size is not updated in real-time so the value may be up to a minute old. +
      +
      +
      public_search
      +       		+
      boolean true / false
      +
      + A boolean value indicating whether public search API is enabled. +
      +
      + +

      Autocomplete

      + +

      JSON data source to work with Indextank-jQuery Autocomplete.

      +

      Additional support for JSONP is provided.

      + + + + + + + + + + + + + + + + + + + +
      HTTP Request / DescriptionHTTP Response codes / body
      +
      GET /v1/indexes/name/autocomplete
      +
      + Performs a search on the index name.
      +
      + The querystring parameters should contain:
      +
        +
      • "query": the query to be performed
        • +
      + And optionally:
      +
        +
      • "field": the field to take suggestions from. By default,
        + 'text' field will be chosen. +
        • +
      • "callback": the callback JS function on the client (enables JSONP),
        + if callback is present response will be JSONP,
        + if no callback is present, response will be a JSON map
        +
        • +
      +
      +       		+
      200 OK
      +
      + The response body will be JSONP or a JSON map with these fields: +
        +
      • "suggestions": JSON array with query matches
        • +
      • "query": requested query parameter (may be normalized)
        • +
      +
      +
      +
      409 The index was initializing
      +
      + This will happen until the value for "started" in the metadata is true. +
      +
      +
      400 Invalid or missing argument
      +
      + A descriptive error message will be found in the body +
      +
      +
      404 No index existed for the given name
      +
      + A descriptive error message will be found in the body +
      +
      + +{% endblock %} + diff --git a/storefront/templates/documentation/app-gallery.html b/storefront/templates/documentation/app-gallery.html new file mode 100644 index 0000000..77ec9d5 --- /dev/null +++ b/storefront/templates/documentation/app-gallery.html @@ -0,0 +1,62 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block title %}Gallery of Applications powered by IndexTank{% endblock %}endblock %} + +{% block common_content %} + +

      This is a showcase of some of the awesome apps that use IndexTank to power their search.

      + + + +{% endblock %} diff --git a/storefront/templates/documentation/app-gallery/app-base.html b/storefront/templates/documentation/app-gallery/app-base.html new file mode 100644 index 0000000..717ec40 --- /dev/null +++ b/storefront/templates/documentation/app-gallery/app-base.html @@ -0,0 +1,33 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block common_content %} + +{% endblock %} diff --git a/storefront/templates/documentation/app-gallery/crocker.html b/storefront/templates/documentation/app-gallery/crocker.html new file mode 100644 index 0000000..b0a547d --- /dev/null +++ b/storefront/templates/documentation/app-gallery/crocker.html @@ -0,0 +1,30 @@ +{% extends "documentation/app-gallery/app-base.html" %} +{% load custom_tags %} + +{% block picture_path %} + {% static 'images/developers/crocker.jpg' %} +{% endblock %} + +{% block author %} + by Jacques Crocker +{% endblock %} + +{% block app_name %} + HelpShelf +{% endblock %} + +{% block problem_solved %} + Allows you to search across your technical library of ebooks +{% endblock %} + +{% block instructions %} + Create an account, and start uploading some books. After it's done indexing your books you should be able to search across everything. +{% endblock %} + +{% block datasets %} + Uploaded PDFs +{% endblock %} + +{% block website %} + Go to http://helpshelf.com/ +{% endblock %} diff --git a/storefront/templates/documentation/app-gallery/culbreth.html b/storefront/templates/documentation/app-gallery/culbreth.html new file mode 100644 index 0000000..ae33393 --- /dev/null +++ b/storefront/templates/documentation/app-gallery/culbreth.html @@ -0,0 +1,30 @@ +{% extends "documentation/app-gallery/app-base.html" %} +{% load custom_tags %} + +{% block picture_path %} + {% static 'images/developers/culbreth.jpg' %} +{% endblock %} + +{% block author %} + by Matt Culbreth +{% endblock %} + +{% block app_name %} + Proggit FTW +{% endblock %} + +{% block problem_solved %} + Proggit FTW helps developers search within the comments of the programming subreddit on reddit.com +{% endblock %} + +{% block instructions %} + Just search proggit! Maybe search for "Ruby" or "list comprehensions" or "nosql", etc. +{% endblock %} + +{% block datasets %} + reddit.com/r/programming/comments +{% endblock %} + +{% block website %} + Go to http://proggitftw.com +{% endblock %} diff --git a/storefront/templates/documentation/app-gallery/dellinger.html b/storefront/templates/documentation/app-gallery/dellinger.html new file mode 100644 index 0000000..8bac8d2 --- /dev/null +++ b/storefront/templates/documentation/app-gallery/dellinger.html @@ -0,0 +1,30 @@ +{% extends "documentation/app-gallery/app-base.html" %} +{% load custom_tags %} + +{% block picture_path %} + {% static 'images/developers/dellinger.jpg' %} +{% endblock %} + +{% block author %} + by Chris Dellinger +{% endblock %} + +{% block app_name %} + GeekSeek +{% endblock %} + +{% block problem_solved %} + GeekSeek helps users find technical gurus on stackoverflow.com +{% endblock %} + +{% block instructions %} + Enter a query, and we will show you which registered stackoverflow users provided the most answers containing that query term. +{% endblock %} + +{% block datasets %} + Stack Overflow history +{% endblock %} + +{% block website %} + Go to http://geekseek.heroku.com/ +{% endblock %} diff --git a/storefront/templates/documentation/app-gallery/sharma.html b/storefront/templates/documentation/app-gallery/sharma.html new file mode 100644 index 0000000..9f84e93 --- /dev/null +++ b/storefront/templates/documentation/app-gallery/sharma.html @@ -0,0 +1,30 @@ +{% extends "documentation/app-gallery/app-base.html" %} +{% load custom_tags %} + +{% block picture_path %} + {% static 'images/developers/sharma.jpg' %} +{% endblock %} + +{% block author %} + by Bhaarat Sharma +{% endblock %} + +{% block app_name %} + Wordsearch +{% endblock %} + +{% block problem_solved %} + Wikidex helps you find words that have same definition or can be used in the same sentence. +{% endblock %} + +{% block instructions %} + Just type in a word! For example, enter ‘gracious’ and ‘amiable’ will appear in the search results. +{% endblock %} + +{% block datasets %} + Wiktionary. (It does not provide a simple api. I had to parse through raw xml dump and had to use tricky xpaths) +{% endblock %} + +{% block website %} + http://wikidex.heroku.com +{% endblock %} diff --git a/storefront/templates/documentation/app-gallery/spence.html b/storefront/templates/documentation/app-gallery/spence.html new file mode 100644 index 0000000..3c564b5 --- /dev/null +++ b/storefront/templates/documentation/app-gallery/spence.html @@ -0,0 +1,30 @@ +{% extends "documentation/app-gallery/app-base.html" %} +{% load custom_tags %} + +{% block picture_path %} + {% static 'images/developers/spence.jpg' %} +{% endblock %} + +{% block author %} + by Tim Spence +{% endblock %} + +{% block app_name %} + Toxosis +{% endblock %} + +{% block problem_solved %} + Responsive search of large volume of nationwide toxic spill data +{% endblock %} + +{% block instructions %} + Simple. Browse the set of spills or use the handy search box to query on city, state, zipcode, etc +{% endblock %} + +{% block datasets %} + E.P.A. Toxic Release Inventory National Data +{% endblock %} + +{% block website %} + Go to http://toxosis.heroku.com/ +{% endblock %} diff --git a/storefront/templates/documentation/app-gallery/waxman.html b/storefront/templates/documentation/app-gallery/waxman.html new file mode 100644 index 0000000..2e6b953 --- /dev/null +++ b/storefront/templates/documentation/app-gallery/waxman.html @@ -0,0 +1,30 @@ +{% extends "documentation/app-gallery/app-base.html" %} +{% load custom_tags %} + +{% block picture_path %} + {% static 'images/developers/waxman.jpg' %} +{% endblock %} + +{% block author %} + by Michael Waxman +{% endblock %} + +{% block app_name %} + QNA +{% endblock %} + +{% block problem_solved %} + QNA is an instant search of a website’s FAQs. Fast, easy way for a user to look up answers. +{% endblock %} + +{% block instructions %} + Press the Big Red Button on the landing page! IMPORTANT: you do NOT need to (and can't) sign up; you can experience the full functionality of the site through the 3 demos (useqna.com/, useqna.com/heroku, and useqna.com/indextank), which are all live and interactive (and powered by IndexTank, of course). +{% endblock %} + +{% block datasets %} + FAQ's and other similar info on Heroku.com +{% endblock %} + +{% block website %} + Go to http://useqna.com and http://useqna.com/heroku +{% endblock %} diff --git a/storefront/templates/documentation/badges.html b/storefront/templates/documentation/badges.html new file mode 100644 index 0000000..cc17e09 --- /dev/null +++ b/storefront/templates/documentation/badges.html @@ -0,0 +1,39 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block right_content %} +{% endblock %} + +{% block title %}"Powered-by IndexTank" badges.{% endblock %} + +{% block common_content %} +

      Big

      +
      +

      Sample

      +

      + IndexTank: hosted search you control +

      +

      Code: {% with "\"IndexTank:" as text %}{% with "#f5f5f5" as bgcolor %}{% include 'includes/clippy.html' %}{% endwith %}{% endwith %}

      +

      + {% box 'code' %} + <a href="http://indextank.com/?utm_campaign=poweredby" name="Powered-by Indextank" rel="friend"><img src="http://indextank.com/_static/images/poweredby/idt-badge-md.png" alt="IndexTank: hosted search you control"/></a> + {% endbox %} +

      +
      +

      Small

      +
      +

      Sample

      +

      + IndexTank: hosted search you control +

      +

      Code: {% with "\"IndexTank:" as text %}{% with "#f5f5f5" as bgcolor %}{% include 'includes/clippy.html' %}{% endwith %}{% endwith %}

      +

      + {% box 'code' %} + <a href="http://indextank.com/?utm_campaign=poweredby" name="Powered-by Indextank" rel="friend"><img src="http://indextank.com/_static/images/poweredby/idt-badge-sm.png" alt="IndexTank: hosted search you control"/></a> + {% endbox %} +

      +
      +{% endblock %} + diff --git a/storefront/templates/documentation/client-base.html b/storefront/templates/documentation/client-base.html new file mode 100644 index 0000000..c7a8be4 --- /dev/null +++ b/storefront/templates/documentation/client-base.html @@ -0,0 +1,390 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% load custom_tags %} +{% load macros %} + +{% enablemacros %} + +{% macro language-name %} +{% endblock %} + +{% macro gist %} +{% endblock %} + + +{% block extrahead %} + +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block right_content %} + {% box 'chatsimple' %} +

      DOWNLOAD

      + {% block download-box %}{% endblock %} +
      LIVE CHAT NOW
      + {% endbox %} + {% box 'table' %} + + {% endbox %} +{% endblock %} + +{% block title %}{% block summary-title %}{% endblock %}{% endblock %} + +{% block common_content %} +

      Summary

      + +

      +

      {% block summary-content %}The following examples assume you have downloaded the API client into your working directory.{% endblock %}

      + {% block extra-summary %}{% endblock %} +

      + + +
      +

      Basic usage

      + +

      If you already have created an index you'll need to use your index name to instantiate the client:

      +

      + +{% box 'code' %} +
      {% block code-instatiate %}{% endblock %}
      +{% endbox %} + +

      +

      Once you have an instance of the client all you need is the content you want to index.
      + The simplest way to add a document is sending that content in a single field called "text":

      +

      + +{% box 'code' %} +
      {% block code-simple-indexing %}{% endblock %}
      +{% endbox %} + +

      +

      That's it, you have indexed a document.

      +

      +

      You can now search the index for any indexed document by simply providing the search query:

      +

      + +{% box 'code' %} +
      {% block code-simple-searching %}{% endblock %}
      +{% endbox %} + +

      +

      As you can see, the results are the document ids you provided when indexing the documents. You use them to retrieve the actual documents from your DB to render the result page.

      +

      +

      You can get richer results using the fetch fields and snippet fields options:

      + +{% box 'code' %} +
      {% block code-snippet-searching %}{% endblock %}
      +{% endbox %} + +

      +

      Deleting an indexed document is also very easy:

      +{% box 'code' %} +
      {% block code-simple-deleting %}{% endblock %}
      +{% endbox %} + + +
      +

      Additional fields [back to top]

      + +

      +

      When you index a document you can define different fields by adding more elements to the document object:

      +

      + +{% box 'code' %} +
      {% block code-multiple-field-indexing %}{% endblock %}
      +{% endbox %} + +

      +

      By default, searches will only look at the "text" field, + but you can define special searches that look at other fields by prefixing a search term with the field name. + The following example filters results including only the user's content.

      +

      + +{% box 'code' %} +
      {% block code-filter-by-author %}{% endblock %}
      +{% endbox %} + +

      +

      There's also a special field named "timestamp" that is expected to contain the publication date of the content in + seconds since unix epoch (1/1/1970 00:00 UTC). If none is provided, the time of indexation will be used by default.

      +

      + +{% box 'code' %} +
      {% block code-index-with-timestamp %}{% endblock %}
      +{% endbox %} + +

      +

      + + +
      +

      Document variables [back to top]

      +

      +

      When you index a document you can define special floating point fields that can be used in the results scoring. + The following example can be used in an index that allows 3 or more variables:

      +

      + +{% box 'code' %} +
      {% block code-index-with-boosts %}{% endblock %}
      +{% endbox %} + +

      +

      You can also update a document variables without having to re-send the entire document. + This is much faster and should always be used if no changes were made to the document itself.

      +

      +{% box 'code' %} +
      {% block code-update-boosts %}{% endblock %}
      +{% endbox %} + +

      +

      + + +
      +

      Scoring functions [back to top]

      +

      +

      To use the variables in your searches you'll need to define scoring functions. These functions can be defined + in your dashboard by clicking "Manage" in your index details or directly through the API client. +

      +

      + +{% box 'code' %} +
      {% block code-redefine-functions %}{% endblock %}
      +{% endbox %} + +

      +

      Read the function definition syntax for help on how to write functions.

      +

      +

      If you don't define any functions, you will get the default function 0 which sorts by timestamp (most recent first). + By default, searches will use the function 0. You can specify a different function when searching by using the option scoring_function

      +

      + +{% box 'code' %} +
      {% block code-search-with-relevance-function %}{% endblock %}
      +{% endbox %} + +

      +

      + + +
      +

      Query variables and Geolocation[back to top]

      +

      +

      Besides the document variables, in the scoring functions you can refer to query variables. These variables + are defined at query time and can be different for each query.

      +

      A common use-case for query variables is geolocation, where you use two variables for latitude and longitude + both in the documents and in the query, and use a distance function to sort by proximity to the user. For this + example will assume that every document stores its position in variables 0 and 1 representing latitude and + longitude respectively.

      +

      Defining a proximity scoring function:

      + +{% box 'code' %} +
      {% block code-proximity-scoring-function %}{% endblock %}
      +{% endbox %} + +

      +

      Searching from a user's position:

      + +{% box 'code' %} +
      {% block code-search-with-geo %}{% endblock %}
      +{% endbox %} + +

      +

      + + +
      + +

      Faceting[back to top]

      +

      +

      Besides being able to define numeric variables on a document you can tag documents with category values. + Each category is defined by string, and its values are alse defined by strings. So for instance, you can define + a category named "articleType" and its values can be "camera", "laptop", etc... You can have another category + called "priceRange" and its values can be "$0 to $49", "$50 to $100", etc...

      +

      Documents can be tagged with a single value for each category, so if a document is in the "$0 to $49" priceRange + it can't be in any other, and retagging over the same category results in overwriting the value.

      +

      You can tag several categories at once like this:

      + +{% box 'code' %} +
      {% block code-update-categories %}{% endblock %}
      +{% endbox %} + +

      +

      When searching, you will get an attribute in the results called "facets", and it will contain a dictionary with + categories for keys. For each category the value will be another map, with category value as key and occurences as + value. So for instance:

      + +{% box 'code' %} +
      {% block code-sample-facets-result %}{% endblock %}
      +{% endbox %} + +

      Means that from the matches, 5 are of the "camera" articleType and 3 are "laptop". Also, 4 of them all are in the "$0 to $299" priceRange, + and 4 on the "$300 to $599".

      +

      Then, you can also filter a query by restricting it to a particular set of category/values. For instance the following will only return + results that are of the "camera" articleType and also are either in th "$0 to $299" or "$300 to $599" price range.

      + +{% box 'code' %} +
      {% block code-faceting-filtering %}{% endblock %}
      +{% endbox %} + + +
      +

      Range queries [back to top]

      +

      +

      + Document variables and scoring functions can also be used to filter your query results. When performing a search + it is possible to add variable and function filters. This will allow you to only retrieve, in the search results, + documents whose variable values are within a specific range (e.g.: posts that have more than 10 votes but less than a 100). + Or only return documents for the which a certain scoring function returns values within a specific range. +

      +

      +

      + You can specify more than one range for each variable or function (the value must be within at least ONE range) filter, + and you can use as many filters as you want in every search (all filters must be met): +

      + +{% box 'code' %} +
      {% block code-range-queries %}{% endblock %}
      +{% endbox %} + + +
      +

      Batch indexing [back to top]

      +

      + + {% block batch-indexing %} + +

      + When populating an index for the first time or when a batch task for adding documents makes sense, + you can use the batch indexing call. +

      +

      + When using batch indexing, you can add a large batch of documents to the Index with just one call. + There is a limit to how many documents you can add in a single call, though. This limit is not + related to the number of documents, but to the total size of the resulting HTTP request, which + should be less than 1MB. +

      +

      + Making a batch indexing call reduces the number of request needed (reducing the latency introduced by round-trips) + and increases the maximum throughput which can be very useful when initially loading a large index. +

      +

      + The indexing of individual documents may fail and your code should handle that and retry indexing them. If there + are formal errors in the request, the entire batch will be rejected with an exception. +

      +

      + +{% box 'code' %} +
      {% block code-batch-indexing %}{% endblock %}
      +{% endbox %} + +

      + The response will be an array with the same length as the sent batch. + Each element will be a map with the key "added" denoting whether the document in this position of + the batch was successfully added to the index. If it's false, an error message will also be in the + map with the key "error". +

      +

      + +{% box 'code' %} +
      {% block code-batch-indexing-failed %}{% endblock %}
      +{% endbox %} + + {% endblock %} + + +
      +

      Bulk Delete [back to top]

      +

      + + {% block bulk-delete %} +

      + With this method, you can delete a batch of documents (reducing the latency introduced by round-trips). + The total size of the resulting HTTP request should be less than 1MB. +

      +

      + The deletion of individual documents may fail and your code should handle that and retry deleting them. If there + are formal errors in the request, the entire batch will be rejected with an exception. +

      +

      + + {% box 'code' %} + 
      {% block code-bulk-delete %}{% endblock %}
      + {% endbox %} + +

      + The response will be an array with the same length as the sent batch. + Each element will be a map with the key "deleted" denoting whether the document with the id in this position of + the batch was successfully deleted from the index. If it's false, an error message will also be in the + map with the key "error". +

      +

      + + {% box 'code' %} + 
      {% block code-bulk-delete-failed %}{% endblock %}
      + {% endbox %} + + {% endblock %} + + +
      +

      Delete by Search [back to top]

      +

      + + {% block query-delete %} +

      + With this method, you can delete a batch of documents that match a particular search query. You can use many of the same arguments applied to a normal search - start (which will preserve the results found before the value of start), scoring function, category filters, variables, and docvar filters. + + {% box 'code' %} + 

      {% block code-query-delete %}{% endblock %}
      + {% endbox %} + + {% endblock %} + +
      +

      Index management [back to top]

      +

      +

      You can create and delete indexes directly with the API client. These methods are equivalent to + their corresponding actions in the dashboard. Keep in mind that index creation may take a few seconds. +

      +

      +

      The create_index methods will return the new index's client: +

      + +{% box 'code' %} +
      {% block code-index-creation %}{% endblock %}
      +{% endbox %} + +

      +

      The delete_index method completely removes the index represented by the object. +

      + +{% box 'code' %} +
      {% block code-index-deletion %}{% endblock %}
      +{% endbox %} + + +

      + +{% endblock %} + diff --git a/storefront/templates/documentation/clients.html b/storefront/templates/documentation/clients.html new file mode 100644 index 0000000..e4cce05 --- /dev/null +++ b/storefront/templates/documentation/clients.html @@ -0,0 +1,59 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block title %}Client libraries and Plug-ins{% endblock %}endblock %} + +{% block common_content %} + +

      We provide client libraries in several popular languages and a few plug-ins for various frameworks.

      + +

      All libraries are published in a public repository, and we encourage you to share your improvements. If you have comments, suggestions or questions please let us know.

      + +

      Remember that if none of the libraries fit your needs, you can develop your own based on the API specification.

      + + + + + + + + + + + + + + + + + + + + + +
      + CLIENTS + + PLUG-INS +
      + Python client library
      + Python implementation of the API calls +       		+ Wordpress Plug-in
      + Insert IndexTank into your Wordpress blog +
      + Ruby client library
      + Ruby implementation of the API calls +
      + PHP client library
      + PHP implementation of the API calls +       		+ Heroku Add-on
      + Use IndexTank directly in your Heroku account +
      + Java client library
      + Java implementation of the API calls +       		+
      + {% endblock %} + diff --git a/storefront/templates/documentation/documentation.html b/storefront/templates/documentation/documentation.html new file mode 100644 index 0000000..61ba0e4 --- /dev/null +++ b/storefront/templates/documentation/documentation.html @@ -0,0 +1,281 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block container_left_class %}document_left{% endblock %} + +{% block title %}Docs & Support{% endblock %} + +{% block right_content %} +
      +
      +
      +
      +
      +

      TEAM & SUPPORT

      +
      +
        +
      • Diego
        • +
      • Santi
        • +
      • Adrian
        • +
      +
        +
      • Spike
        • +
      • Leandro
        • +
      • Nacho
        • +
      +
        +
      • Mono
        • +
      • Jorge
        • +
      • Sean
        • +
      +

      Have a question? We're here to help.

      +

      Our team of experienced and friendly devs will respond almost as quickly as our search API.

      +
      +
      LIVE CHAT NOW
      + +
      +
      +
      +
      +
      +{% endblock %} + +{% block common_content %} + +

      DOCUMENTATION

      +
      +
      +

      Getting Started

      + + +

      Programmer’s Guide

      + + + + + +

      Reference

      + + +

      Client Libraries

      + + + + +
      +
      +

      Plug-ins

      + + + +

      Demo Apps Writeups

      + + + +

      White papers

      + +

      Case studies

      + +
      +
      +
      +

      LATEST DEV NEWS

      +
        +
      • Delete by Search by Sean, July 11 +

        + You can now delete all the documents that match a particular query. You can use the same filters as a normal search, and you can even use the 'start' parameter to preserve high scoring documents. You can find the appropriate method in our Ruby, PHP, Python, and Java clients.
        Let us know if there's anything else you'd like to see! +

        +
        • +
      • Drupal plugin by Leandro, June 26 +

        + We've been working on this one for a few weeks now, and supporting both drupal 7 and 6 was not easy. But thanks to the invaluable help of Xavier, here it is! + Give it a try, and let us know. There're still a few small issues with css, but it is completely functional now. +

        +
        • +
      • Search Scores and Variables Returned by Adrian, June 7 +

        + Until today, document variables could be used in your relevance formulas but you could not fetch them when querying the index. You had to create separate text fields with the same information, which was redundant. Also the document scores that were used to sort weren't returned, which made debugging relevance issues complicated.
        + That is no longer the case, now you can fetch variables and scores along with your search results. Many of our users had requested this feature so here you go! +

        +
        • +
      • Javascript "Static" Search by Mono, May 20 +

        + Now it's possible to create a search page from a completely static html page in (literally) a few minutes! + Indextank-jquery provide a series of building blocks that provide basic search, instant search, autocomplete, + and faceting. +

        +
        • +
      • Reddit Case Study by Manolo, February 15 +

        + Learn how after a bout with infamously disappointing search, social news site Reddit found IndexTank, + which easily handles the company’s huge index, delivers real-time results, and has helped re-polish their reputation. +

        +
        • +
      • Wordpress Plugin by Nacho, January 27 +

        + You now can now add our search, including auto-complete, to your WordPress blog by following + some quick steps. Some themes may require you to make additional changes to support snippets. +

        +
        • +
      • Autocomplete (updated) by Adrian, January 18 +

        + Thanks to everyone who asked for a more complete write-up of one of our more popular + features. You’ll find a quick-start guide, as well as help customizing with partial highlighting. +

        +
        • +
      +
      + + + + + + + + +{% comment %} +

      Documentation

      +

      What's here?

      +

      + You can get IndexTank working in minutes. That’s half because it’s so easy to setup, + and half because we give you a lot of great documentation and support to guide your + implementation path, including: +

      +
        +
      • Tutorials to get you started
        • +
      • Client plug-in writeups
        • +
      • Detailed API reference
        • +
      • White papers
        • +
      • ...and more
        • +

        What's new?

        +

        + Here’s a list of the most recent additions we’ve made: +

        +
        +

        Reddit Case Study (new!) by Manolo, February 15

        +

        + Learn how after a bout with infamously disappointing search, social news site Reddit found IndexTank, which easily handles the company’s huge index, delivers real-time results, and has helped re-polish their reputation. +

        +

        Wordpress Plugin by Nacho, January 27

        +

        + You now can now add our search, including auto-complete, to your WordPress blog by following + some quick steps. Some themes may require you to make additional changes to support snippets. +

        +

        Autocomplete (updated) by Adrian, January 18

        +

        + Thanks to everyone who asked for a more complete write-up of one of our more popular + features. You’ll find a quick-start guide, as well as help customizing with partial highlighting. +

        +
        +

        Support

        +

        + Have a question? Our team of experienced and friendly developers are quick to respond, and + never more than a few clicks away, any time. Sometimes we’re burning the midnight oil too. +

        +
          +
        • Give us a shout on our live chat support (click the link at the bottom of any indextank.com page).
          • +
        • Email us to explain your question.
          • +
        • Follow our blog and twitter to read the latest news.
          • +
        + + {% include 'includes/staff-banner.html' %} + +{% endblock %} + +{% block sidebar_content %} +

        Contents

        + +{% endcomment %} +{% endblock %} + diff --git a/storefront/templates/documentation/faq.html b/storefront/templates/documentation/faq.html new file mode 100644 index 0000000..db784e0 --- /dev/null +++ b/storefront/templates/documentation/faq.html @@ -0,0 +1,183 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block right_content %} + {% box 'table' %} + + {% endbox %} +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} +{% block title %}Frequently Asked Questions (FAQ){% endblock %} + +{% block common_content %} +

        General / Overview

        +
        +

        What can I do with IndexTank?

        +

        + You can use IndexTank to create a finely-tuned search function for your web site or application. + If you have content you want your users to find, but you don’t want to deal with complicated coding + and system administration tasks, IndexTank is for you. Check out our plug-in page. +

        + +

        I am not a developer. What can I do with IndexTank?

        +

        + IndexTank is a service intended to be used mainly by developers or system administrators. It's not a standalone web search engine, and we don't currently have a way for you to set it up directly through the Web. It requires downloading software such as a Wordpress plug-in (if you wanted to add better search to your blog, for example) or writing a program to interact with our servers. +

        +

        + Please let us know what you would like to do with IndexTank, and we'll do our best to help you. Thanks for your interest! +

        + +

        Do I download IndexTank and run it on my computer?

        +

        IndexTank is a hosted service, which means that it runs on our servers over your Internet connection, what we like to call “in the cloud.†Using IndexTank saves you from having to do any system administrations tasks or pay for hardware.

        + +

        How much set up and management does IndexTank require?

        +

        Very little. You can be up and running in minutes, and we take care of all the maintenance.

        + +

        What languages do you support?

        +

        + We provide clients for Python, Ruby, PHP and Java. Our API spec is public and consists of REST calls that can be made through HTTP, + so if you don’t use one of our client libraries, it’s easy to implement your own client or even use the API from the command line + with tools like Curl. You can find open source clients in Clojure, Perl, C#, and more here. +

        + +

        What level of support do you provide?

        +

        + We provide support via email at support@indextank.com and via chat through the pop-up window in the lower left corner of our site. We’re also in the process of setting up a discussion group. +

        + +

        How do I get my data to you?

        +

        + To index your existing data for the first time, you need to iterate through your document database and send each document to the index via the “add_document†function. +

        +

        + To keep your index up-to-date with the latest additions and changes on your site, you need to call the “add_document†function for new or changed documents, and the “delete_document†function for deleted documents. +

        + +

        How fast does IndexTank deliver search results?

        +

        IndexTank resolves most queries in well under the 50ms mark. For very large data sets and complex queries, times can go up to 300ms.

        + +

        Can IndexTank handle a large web site with a lot of users and a lot of queries?

        +

        Absolutely. Basic accounts support 20-30 queries per second out of the box. If that’s not sufficient for your needs, we can set up custom accounts that can easily handle several hundred queries per second.

        + +

        I have tens of millions of documents to index. Will IndexTank work for me?

        +

        Absolutely! Check out reddit’s search to see IndexTank in action over a huge data set. For more information about a custom plan tailored to your needs, please contact us.

        +
        + +

        Sign-up, Pricing and Billing

        +
        +

        How do I sign up?

        +

        + Go to our sign-up page and select a plan. For free plans, we only need your email address and a password. For plans that require payment, we also need your billing information. +

        + +

        Can I cancel any time I want?

        +

        Yes. You can cancel your account at any time without any prior notification.

        + +

        How do I cancel my service?

        +

        Send us an email telling us you want to cancel, and we’ll take care of it.

        + +

        When will you charge me?

        +

        We charge for the paid plans every month in advance, and we provide the first month free of charge. This means that if you sign up, for example, on January 15th, you will not be charged until February 15th, and that will pay for the period from February 15th until March 14th.

        + +

        Can I switch plans?

        +

        Sure. We’re working on automating this process, but in the meantime, contact us and we’ll do it for you.

        + +

        How do I upgrade from the Free package to the Starter package?

        +

        Contact us via email

        +
        + + +
        +

        What is an index?

        +

        + An index is a structure designed for quick and efficient keyword searches. Think of the index at the back of a book, and how you can find a page containing a particular word without having to scan every single page. When you create a new index in IndexTank and start adding entries to it, it is as if you are starting with an empty book and updating the index every time you add a section. +

        +

        + For example, let’s say you have a blog and you want to index it so your users can search through it. You index the first post (post #1) which contains the word “appleâ€. The index will contain an entry for “apple†pointing to the post #1. When a user searches for “apple†the index shows that “apple†is mentioned in post #1. +

        +

        + You can also split your document into several fields and index them as separate entities. For example, your blog post might be composed of a “title†field, an “author†field and a “content†field, etc. When you index the document, you provide its unique identifier (the document ID, or docid for short) and the contents of each field. You can later restrict your search to a specific field by using the field name in the query, for example “title:appleâ€. +

        + +

        Can I organize the results in different ways?

        +

        Yes. IndexTank gives you the ability to fine-tune your search results based on your specific needs. You can define your own scoring functions and document variables to influence the search results.

        + +

        What are scoring functions and document variables?

        +

        + IndexTank uses scoring functions to customize the order of search results. Scoring functions are mathematical formulas that we apply to every document on search time to calculate a value (score) for it. We then use this value to sort the results. +

        +

        + The scoring functions can refer to the document’s age, its relevance to the query and special variables that you can supply for a document, such as number of votes, or any numeric value that you can associate with your document and that somehow describes the importance of that document. +

        +

        + IndexTank is optimized to allow very frequent updates to document variables without slowing down the index. +

        +

        + You can define several scoring functions and choose which one to use for each query. When you change a scoring function, the new version immediately takes effect. +

        +

        + Scoring functions are different from typical database sort clauses in that they allow much greater flexibility. +

        +

        + See our Scoring Functions documentation page for more information. +

        + +

        How often will my data be indexed?

        +

        As often as you wish. IndexTank doesn’t actively fetch data from you as a web crawler would do. Instead, your application sends IndexTank the data as soon as it is created or updated.

        + +

        How is IndexTank different than Lucene and Solr?

        +

        + There are many functional differences. For example, documents added to an IndexTank index are immediately searchable; document variables can be updated without having to re-index the whole document and they can be updated very quickly and at a very rapid pace without affecting the index performance; results can be sorted by arbitrary functions that include geolocation support. +

        +

        + But more importantly, IndexTank is a cloud-based, robust and scalable service, so you don’t have to worry about managing servers, configuring the software and scalabilty. +

        +

        + For a more detailed discussion of how IndexTank compares to Lucene and Solr, download our white paper, A Technical Discussion about Search Solutions. +

        + +

        Is IndexTank better than Google Custom Search?

        +

        + Yes. IndexTank provides more complete, relevant, and timely search results, because we give you the ability + to control what gets indexed, when it gets indexed, and how the results are sorted. Google Custom Search results + are often stale or incomplete in comparison. Further, Google Custom Search requires you to show ads, which Google + chooses in order to increase its own advertising revenue, so you may end up serving sponsored links for your competitors. + IndexTank does not require ads. +

        + +

        Is my data secure in the cloud?

        +

        + Yes. From a security standpoint, we use Amazon AWS security best practices, and our servers are not accessible in any way except to provide the search + service. Since each customer has a unique key to access their own index, no unauthorized access is possible. Your data is as private as you choose to make it. +

        +

        + In terms of integrity, storing your data in the cloud can be more secure than storing it on-site, because cloud data is typically replicated in several + locations and supported by a lot more hardware than most companies want to own and maintain. +

        + +

        Can IndexTank search though PDF or Microsoft Word documents?

        +

        + IndexTank, like other full-text search alternatives, indexes only text. However, for common formats like PDF or Word, it is very easy to parse them to obtain the readable text by using open source tools. +

        + +

        What types of queries work with IndexTank?

        +

        IndexTank implements Boolean operators (and, or, and not) and phrase queries.

        + +

        Can I control what appears at the top of my search results?

        +

        + Yes. IndexTank’s scoring functions have enormous flexibility and will help you build a scoring system that perfectly matches your data’s semantics. + In some situations, however, you may prefer to have manual control of the top result. For this, IndexTank implements a “promote†API call that enables you + to pin down a particular document as the top result for a given query. Take a look at the “Promoting results†section of our API spec for more information. +

        +
        +
        +{% endblock %} + diff --git a/storefront/templates/documentation/function-definition.html b/storefront/templates/documentation/function-definition.html new file mode 100644 index 0000000..7b23609 --- /dev/null +++ b/storefront/templates/documentation/function-definition.html @@ -0,0 +1,563 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + + +{% block right_content %} + {% box 'table' %} + + {% endbox %} +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block title %}Scoring Function Formulas{% endblock %} + +{% block common_content %} +

        Introduction

        + +

        + Scoring functions are defined by mathematic formulas that take data from the document, the query and the textual relevance in order to + assign a score to each matching document for a query. The resulting scores are used when searching the index to provide specific orderings for the results. +

        +

        + You can modify these formulas in real-time and they can be as complex as you need them to be. +

        + +

        When writing the formula, have in mind that: +

          +
        • + Variable and function names are case sensitive. +
          • +
        • + Formulas must be well formed. A missing parenthesis, an unknown function or variable name or an undefined + operator will result in a syntax error when editing the function. +
          • +
        • + Variable values and the resulting score are float numbers. +
          • +
        • + All expressions except conditions are float expressions. In any context where you can use a variable + you can also use a function or a literal scalar value ("1", "0.5", "-5"). +
          • +
        +

        + +

        Operators

        + +

        Formulas allow the following operators to work with expressions: + +, -, *, / +

        +

        + These are all binary operators except for "-" which can also be used to negate (being a unary operator). +

        + + + +
        + +

        Variables

        + +
        +

        Scoring function formulas are computed for each document matching a given query. Hence there is a list of variables related to + the document or the query that can be used in the formula:

        +

        +

        +

        +
        + + + + + + + + + + + + + + + + + +
        Textual Relevance
        + Description: For each document matching a query a textual relevance (how relevant is the documents text for the query) is calculated. You may or may not consider this + value in your formula, or decide how important it is in the final calculation (e.g.: if you want to sort your results just by creation time, you may + discard this variable in your scoring function). + + Syntax: relevance +
        + Short syntax: rel or r or R +
        + Values: Relevance is always a positive float number. Because of precision issues, relevance CAN be zero. +
        + Sample:
        +{% box 'code' %} +
          
+    -age * relevance (sorts documents considering how new and how relevant to the query the document is equally)
+
        +{% endbox %} + +
        + + + + + + + + + + + + + + + + + +
        Document's Age
        + Description: When indexed, every document is assigned a timestamp, an integer value which usually describes its creation time. The larger the value, the newer the document. + The timestamp field can be provided when adding the document. Otherwise, IndexTank automatically assigns a value representing the number of seconds since + Unix Epoch (00:00:00 UTC on 1 January 1970) until the moment the document was indexed. +

        + When writing formulas you can use the document's age, which is the result of subtracting the documents timestamp to the number of seconds since Unix Epoch until + the moment the query was executed. When using UNIX time for the documents' timestamps, this variable represents the age of the document in seconds. +         		+ Syntax: doc.age +
        + Short syntax: age or a or A +
        + Values: Since there are no restrictions to the documents' timestamps provided, age can contain negative values. +
        + Sample:
        +{% box 'code' %} +
          
+    -age * relevance (sorts documents considering how new and how relevant to the query the document is equally)
+
        +{% endbox %} + +
        + + + + + + + + + + + + + + + + + + +
        Document's Variables
        + Description: When a document is indexed, it is possible to assign numeric (float) variables to it. These + variables may represent rapidly changing numeric values that have some implication on the document's possible valuation in a sorting function + (number of positive and negative votes, number of comments, user generated score, review score, number of visits, etc.). Once a document is + indexed, its variables can be changed any time it is necessary with no cost other than the one related to the HTTP communication. +

        + The variables are identified by an integer number from zero to the the variables' limit minus one. + The maximum number of variables available for each document will depend on the package of the account. +         		+ Syntax: doc.var[n] (where n is the variable's integer identifier) +
        + Short syntax: d[n] or D[n] +
        + Values: Any float entered when the document was indexed or afterwards. + For negative values, NaN (not a number) will be returned. A zero value will return negative infinity. +
        + Sample:
        +{% box 'code' %} +
          
+    log(doc.var[0]) - age/86400 (sorts documents considering natural logarithm of the variable #0 of the document minus its age in days)
+
        +{% endbox %} + +
        + + + + + + + + + + + + + + + +
        Query Variables
        + Description: When performing a search in the index, it is possible to pass float variables along with + the query (check the searching documentation). + These variables can be later used in the scoring function's formula. +

        +         		+ Syntax: query.var[n] (where n is the variable's integer identifier) +
        + Short syntax: q[n] or Q[n] +
        + Values: Any float passed as a query variable. +
        +
        + + +
        + +

        Available functions

        + +
        +

        There is a set of mathematical functions available for writing formulas:

        +

        +

        +

        + +
        + + + + + + + + + + + + + + + + +
        Natural logarithm
        + Description: Calculates the natural logarithm of an expression.
        + The logarithm function is useful when there's a need to consider the order of magnitude of an expression instead of its actual value + (for example, it is comparable to considering the number of digits of the value). Mathematically speaking, it is the inverse to + an exponential function. +         		+ Syntax: log(val) +
        + Arguments: +
        val: a float expression to the which apply the logarithm. For negative values, NaN +
        + Sample:
        +{% box 'code' %} +
          
+    log(doc.var[0]) - age/86400 (sorts documents considering natural logarithm of the variable #0 of the document minus its age in days)
+
        +{% endbox %} + +
        + + + + + + + + + + + + + + + + + +
        Power
        + Description: Raises a given float expression to a given power (integer).
        + The power function can be used to create exponential functions or to weight different factors (make one more important than another + in a product). +         		+ Syntax: pow(base, exponent) +
        + Arguments: +
        base: a float expression, the base. +
        exponent: a integer expression, the exponent. Zero, and negative values + can be used. Float expressions (variables, function results) can be used, but will be truncated (the integer value closest + to zero will be considered). +
        + Sample:
        +{% box 'code' %} +
          
+   pow(doc.var[0], 3) * doc.var[1] (sorts documents considering variable #0 three times as important as variable #1)
+
        +{% endbox %} + +
        + + + + + + + + + + + + + + + + +
        Max
        + Description: Returns the greater of two values. + + Syntax: max(a, b) +
        + Arguments: +
        a: a float expression. +
        b: a float expression. +
        + Sample:
        +{% box 'code' %} +
          
+   max(doc.var[0], doc.var[1]) (sorts documents considering variable #0 or variable #1 wichever is greater)
+
        +{% endbox %} + +
        + + + + + + + + + + + + + + + + +
        Min
        + Description: Returns the smaller of two values. + + Syntax: min(a, b) +
        + Arguments: +
        a: a float expression. +
        b: a float expression. +
        + Sample:
        +{% box 'code' %} +
          
+   min(doc.var[0], doc.var[1]) (sorts documents considering variable #0 or variable #1 wichever is smaller)
+
        +{% endbox %} + +
        + + + + + + + + + + + + + + + + +
        Absolute
        + Description: Returns the absolute value of a double value. For positive values, the argument is + returned. For negative values, the negation of the value is returned. + + Syntax: abs(value) +
        + Arguments: +
        value: a float expression, zero, positive, or negative. +
        + Sample:
        +{% box 'code' %} +
          
+   abs(doc.var[0]) (sorts documents considering variable #0 equally when its value is 1 or -1)
+
        +{% endbox %} + +
        + + + + + + + + + + + + + + +
        Square root
        + Description: Calculates the square root of a double value. This function is a variant of + the power function that considers one case of non integer exponent (1/2). + + Syntax: sqrt(value) +
        + Arguments: +
        value: a float expression. For negative values, NaN (not a number) will be returned. +
        + + + + + + + + + + + + + + + + +
        If clause
        + Description: Evaluates a condition and returns the corresponding expression. This function takes + three arguments: a boolean condition, the expression to evaluate when the condition is met and the expression to consider when it is not.
        + The expressions are regular float expressions (a variable, the result of a function, the result of an operation, a literal).
        + The boolean condition is expressed by comparing two float expressions (no boolean operations allowed) with one of this comparators:
        +
          +
        • + a == b: true when both expressions are equal. +
          • +
        • + a <= b: true when expression a is smaller than or equal to expression b. +
          • +
        • + a >= b: true when expression a is greater than or equal to expression b. +
          • +
        • + a < b: true when expression a is smaller than expression b. +
          • +
        • + a > b: true when expression a is greater than expression b. +
          • +
        • + a != b: true when expression a and b are not equal. +
          • +
        +         		+ Syntax: if(cond, true, false) +
        + Arguments: +
        cond: the boolean condition comparing two expressions. +
        true: the expression to evaluate when cond: is met. +
        false: the expression to evaluate when cond: is not met. +
        + Sample:
        +{% box 'code' %} +
          
+   if(doc.var[0] < 1, doc.var[0], rel) (sorts documents considering variable #0 while its value is less than 1, otherwise considering textual relevance)
+
        +{% endbox %} +
        + + + + + + + + + + + + + + + + +
        Kms/miles calculator
        + Description: Calculates the distance between two geographical points expressed as longitude/latitude coordinates. + The distance can be expressed in kilometers or in miles. + + Syntax: km(lat1, long1, lat2, long2) or miles(lat1, long1, lat2, long2) +
        + Arguments: +
        lat1: latitude of point 1. +
        long1: longitude of point 1. +
        lat2: latitude of point 2. +
        long2: longitude of point 2. +

        All coordinates are float values and they are expressed in degrees (non integer values ARE considered). +
        + Sample:
        +{% box 'code' %} +
          
+   miles(query.var[0], query.var[1], doc.var[0], doc.var[1]) 
+   (sorts documents considering the distance between doc and a point passed in the query)
+
        +{% endbox %} +
        + +
        +
        +
        +{% endblock %} + diff --git a/storefront/templates/documentation/how-it-works.html b/storefront/templates/documentation/how-it-works.html new file mode 100644 index 0000000..42d5ac4 --- /dev/null +++ b/storefront/templates/documentation/how-it-works.html @@ -0,0 +1,29 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block title %}How it works{% endblock %}endblock %} + +{% block common_content %} +

        IndexTank provides online search to your site or product taking care of all the heavy lifting for you.

        + +

        It is very simple to use.

        + +
        +

        1. Sign up to get an API key.

        +

        2. Create an index (or more than one) using the dashboard or through the API. +

        3. Then, add a few lines of code and a search box to your site + (access the API directly + or through the client library) + +

        That's it.

        +
        + +

        You can start submitting your content to the index and it will become searcheable in real-time.

        +
        + +
        +
        +
        + +{% endblock %} + diff --git a/storefront/templates/documentation/java-client.html b/storefront/templates/documentation/java-client.html new file mode 100644 index 0000000..3da19a7 --- /dev/null +++ b/storefront/templates/documentation/java-client.html @@ -0,0 +1,356 @@ +{% extends "documentation/client-base.html" %} +{% load custom_tags %} + +{% block language-name %}java{% endblock %} +{% block gist %}910790{% endblock %} + + +{% block download-box %} +
          +
        • + Stable Version - + zip | + tgz +
          • +
        • + All versions +
          • +
        • + Source code (github) +
          • +
        • + Maven artifact: + 
          +<groupId>com.indextank</groupId>
+<artifactId>indextank-java</artifactId>
+<version>1.0.9</version>
          +
          • +
        +{% endblock %} + + +{% block summary-title %}Java client{% endblock %} + + +{# #} +{% block code-instatiate %} +import com.flaptor.indextank.apiclient.IndexTankClient; +import com.flaptor.indextank.apiclient.IndexTankClient.Index; +import com.flaptor.indextank.apiclient.IndexTankClient.Query; + +... + +IndexTankClient client = new IndexTankClient("<YOUR API URL HERE>"); +Index index = client.getIndex("<YOUR INDEX NAME HERE>"); +{% endblock %} + + +{# #} +{% block code-simple-indexing %} +String documentId = "<YOUR DOCUMENT ID>"; +String documentText = "<THE TEXTUAL CONTENT>"; + +Map<String, String> fields = new HashMap<String, String>(); +fields.put("text", documentText); + +index.addDocument(documentId, fields); +{% endblock %} + + +{# #} +{% block code-simple-searching %} +import com.flaptor.indextank.apiclient.IndexTankClient.SearchResults; + +... + +String query = "<YOUR QUERY STRING>"; + +SearchResults results = index.search(Query.forString(query)); + +System.out.println("Matches: " + results.matches); + +for (Map<String, Object> document : results.results) { + System.out.println("doc id: " + document.get("docid")); +} +{% endblock %} + +{# #} +{% block code-snippet-searching %} +results = index.search(Query.forString(query) + .withSnippetFields("text") + .withFetchFields("title", "timestamp")); + + +System.out.println("Matches: " + results.matches); +for (Map<String, Object> document : results.results) { + System.out.println("id: " + document.get("docid") + + ";title: " + document.get("title") + + ";timestamp: " + document.get("timestamp") + + ";snippet: " + document.get("snippet_text")); +} +{% endblock %} + +{% block code-simple-deleting %} +String documentId = "<YOUR DOCUMENT ID>"; + +index.deleteDocument(documentId); +{% endblock %} + +{# #} +{% block code-multiple-field-indexing %} +// INDEX MULTIPLE FIELDS +String title = "<DOCUMENT TITLE>"; +String author = "<DOCUMENT AUTHOR>"; + +fields = new java.util.HashMap<String, String>(); +fields.put("text", documentText); +fields.put("title", title); +fields.put("author", author); + +index.addDocument(documentId, fields); +{% endblock %} + + +{# #} +{% block code-filter-by-author %} +// FILTER TO USER'S CONTENT +results = index.search(Query.forString(query + " author:" + author)); +{% endblock %} + + +{# #} +{% block code-index-with-timestamp %} +fields.put("timestamp", + Long.toString(System.currentTimeMillis() / 1000L)); +index.addDocument(documentId, fields); +{% endblock %} + + +{# #} +{% block code-index-with-boosts %} +// INDEX DOCUMENT WITH VARIABLES +Float rating = 0.5f; +Float reputation = 1.5f; +Float visits = 10.0f; + +Map<Integer, Float> variables = new HashMap<Integer, Float>(); +variables.put(0, rating); +variables.put(1, reputation); +variables.put(2, visits); + +index.addDocument(documentId, fields, variables); +{% endblock %} + +{# #} +{% block code-update-boosts %} +// UPDATE DOCUMENT VARIABLES ONLY +variables.put(0, newRating); +variables.put(2, newVisits); + +index.updateVariables(documentId, variables); +{% endblock %} + +{# #} +{% block code-redefine-functions %} +// FUNCTION 0 : sorts by most recent +index.addFunction(0, "-age"); + +// FUNCTION 1 : standard textual relevance +index.addFunction(1, "relevance"); + +// FUNCTION 2 : sorts by rating +index.addFunction(2, "doc.var[0]"); + +// FUNCTION 3 : sorts by reputation +index.addFunction(3, "d[1]"); + +// FUNCTION 4 : advanced function +index.addFunction(4, "log(d[0]) - age/50000"); +{% endblock %} + + +{# #} +{% block code-search-with-relevance-function %} +index.search(Query.forString(query).withScoringFunction(2)); +{% endblock %} + +{# #} +{% block code-query-delete %} +index.deleteBySearch(Query.forString(query)) +{% endblock %} + +{# #} +{% block code-proximity-scoring-function %} +// FUNCTION 5 : inverse distance calculated in miles +index.addFunction(5, "-mi(d[0], d[1], q[0], q[1])"); +{% endblock %} + + +{# #} +{% block code-search-with-geo %} +results = index.search(Query.forString(query) + .withScoringFunction(5) + .withQueryVariable(0, latitude) + .withQueryVariable(1, longitude)); +{% endblock %} + +{# #} +{% block code-range-queries %} +/* + In this sample, the results will only include documents + whose variable #0 value is between 5 and 10 or between 15 + and 20, and variable #1 value is less than or equal to 3 +*/ +results = index.search(Query.forString(query) + .withDocumentVariableFilter(0, 5d, 10d) + .withDocumentVariableFilter(0, 15d, 25d) + .withDocumentVariableFilter(1, Double.NEGATIVE_INFINITY, 3)); + +// The same applies to functions +results = index.search(Query.forString(query) + .withFunctionFilter(0, 0.5d, Double.POSITIVE_INFINITY) +{% endblock %} + +{% block batch-indexing %} + +

        + When populating an index for the first time or when a batch task for adding documents makes sense, + you can use the batch indexing call. +

        +

        + When using batch indexing, you can add a large batch of documents to the Index with just one call. + There is a limit to how many documents you can add in a single call, though. This limit is not + related to the number of documents, but to the total size of the resulting HTTP request, which + should be less than 1MB. +

        +

        + Making a batch indexing call reduces the number of request needed (reducing the latency introduced by round-trips) + and increases the maximum throughput which can be very useful when initially loading a large index. +

        +

        + The indexing of individual documents may fail and your code should handle that and retry indexing them. If there + are formal errors in the request, the entire batch will be rejected with an exception. +

        +

        + In the JAVA client, you can use a convenient Document object for the documents and you can pass a simple Iterable to the addDocuments() method. +

        + +

        + +{% box 'code' %} +
        +List<Document> documents = new ArrayList<Document>();
+
+Map<String, String> fields1 = new HashMap<String, String>();
+fields1.put("text", "document 1 text");
+
+// Document is built with:
+// - String docId
+// - Map<String, String> fields
+// - Map<Integer, Float> variables 
+// - Map<String, String> facetingCategories
+Document document1 = new Document("1", fields1, null, null);
+documents.add(document1); 
+
+Map<String, String> fields2 = new HashMap<String, String>();
+fields1.put("text", "document 2 text");
+Map<Integer, Float> variables2 = new HashMap<Integer, Float>();
+variables2.put(1, 0.4f);
+Document document2 = new Document("2", fields2, variables2, null);
+documents.add(document2); 
+
+BatchResults results = index.addDocuments(documents);
+
        +{% endbox %} + +

        + The results object provides you with a way to retry when some of the documents failed to be indexed: +

        + +{% box 'code' %} +
        +Iterable<Document> failedDocuments = results.getFailedDocuments();
+index.addDocuments(failedDocuments);
+
        +{% endbox %} +{% endblock %} + + +{# #} +{% block code-bulk-delete %} +List<String> docids = new ArrayList<String>(); +docids.add("doc1"); +docids.add("doc2"); +docids.add("doc3"); +docids.add("doc4"); + +BulkDeleteResults results = index.deleteDocuments(docids); +{% endblock %} + +{% block code-bulk-delete-failed %} +Iterable<String> failedDocids = results.getFailedDocids(); +index.deleteDocuments(failedDocuments); +{% endblock %} + + +{# #} +{% block code-update-categories %} +Map<String, String> categories = new HashMap<String, String>(); +categories.put("priceRange", "$0 to $299"); +categories.put("articleType", "camera"); + +index.updateCategories(docId, categories); +{% endblock %} + +{# #} +{% block code-sample-facets-result %} +Map<String, Map<String, Integer>> facets = results.facets; + +... + +{ + 'articleType': { + 'camera': 5, + 'laptop': 3 + }, + 'priceRange': { + '$0 to $299': 4, + '$300 to $599': 4 + } +} +{% endblock %} + +{# #} +{% block code-faceting-filtering %} +Map<String, List<String>> filters = + new HashMap<String, List<String>>(); +filters.put("priceRange", + Arrays.asList("$0 to $299", "$300 to $599")); +filters.put("articleType", + Arrays.asList("camera")); + +results = index.search(Query.forString(query) + .withCategoryFilters(filters)); +{% endblock %} + + + + +{# #} +{% block code-index-creation %} + +// this parameter allows you to create indexes with public search enabled. +// default is false. + +boolean publicSearchEnabled = false; +Index index = client.createIndex("test_name", publicSearchEnabled); + +while (!index.hasStarted()) { + Thread.sleep(300); +} +{% endblock %} +{# #} +{% block code-index-deletion %} +client.deleteIndex("test_name"); +{% endblock %} + diff --git a/storefront/templates/documentation/java-client.html.old b/storefront/templates/documentation/java-client.html.old new file mode 100644 index 0000000..4550932 --- /dev/null +++ b/storefront/templates/documentation/java-client.html.old @@ -0,0 +1,156 @@ +{% extends "base.html" %} +{% load custom_tags %} + + +{% block extrahead %} + +{% endblock %} + +{% block wrapper_class %}nosidebar{% endblock %} + +{% block content %} + +
        +
        +

        Java Client API and Tools

        +
        + +
        +
        Download client
        + +
        +
        +
        Sample code: Indexing a document
        + 
        +  
+  import com.flaptor.indextank.apiclient.IndexTank;
+  import java.util.Map;
+  import java.util.HashMap;
+  
+  ...
+
+  IndexTank it = new IndexTank("{% if request.user.is_authenticated %}{{ request.user.get_profile.account.apikey }}{% else %}YOUR API KEY HERE{% endif %}");
+  try {
+      // Add a document
+      String indexCode = "YOUR INDEX CODE";
+      String documentId = "1";
+      Map<String, String> document = new HashMap<String, String>();
+      document.put("title", "This is the title");
+      document.put("text", "This is the text");
+      it.addDocument(indexCode, documentId, document);
+  } catch (Exception e) {
+      System.out.println(e.getMessage());
+  }
+  
+
        +
        + +
        +
        Sample code: Searching
        + 
        +  
+  import com.flaptor.indextank.apiclient.IndexTank;
+  import java.util.List;
+  
+  ...
+
+  IndexTank it = new IndexTank("{% if request.user.is_authenticated %}{{ request.user.get_profile.account.apikey }}{% else %}YOUR API KEY HERE{% endif %}");
+  
+  try {
+    // Search documents with the string 'title'
+    String idxCode = "YOUR INDEX CODE";
+    List<String> docIds = it.search(idxCode, "title");
+    
+    for (String id : listIndexes) {
+      System.out.println("Doc Id: " + id);
+    }
+    
+  } catch (Exception e) {
+    System.out.println(e.getMessage());
+  }
+
        +
        + + +
        +
        Sample code: Modifying dynamic boosts
        + 
        +  
+  import com.flaptor.indextank.apiclient.IndexTank;
+  import java.util.Map;
+  
+  ...
+
+  IndexTank it = new IndexTank("{% if request.user.is_authenticated %}{{ request.user.get_profile.account.apikey }}{% else %}YOUR API KEY HERE{% endif %}");
+  
+  try {
+    
+    String idxCode = "YOUR INDEX CODE";
+    String documentId = "1"; // The id of the existing document
+    
+    /*
+     * A map with the boosts' values to be modified.
+     * The map's keys are the boosts' indexes (zero based)
+     * and the values are doubles. 
+     * Depending on the IndexTank plan, the number of available
+     * boosts varies.  
+     */
+    Map<Integer, Double> boosts = new HashMap<Integer, Double>();
+    
+    /*
+     * The boost index is the way to reference
+     * it in the scoring functions. 
+     */  
+    boosts.put(0, 1.5d);
+
+    /*
+     * Calling updateBoosts doesn't imply reindexing
+     * the document and, therefore, is much less expensive
+     */
+    it.updateBoosts(idxCode, documentId, boosts);
+    
+  } catch (Exception e) {
+    System.out.println(e.getMessage());
+  }
+
        +
        + +
        +
        Sample code: Creating (or updating) scoring functions
        + 
        +  
+  import com.flaptor.indextank.apiclient.IndexTank;
+  
+  ...
+
+  IndexTank it = new IndexTank("{% if request.user.is_authenticated %}{{ request.user.get_profile.account.apikey }}{% else %}YOUR API KEY HERE{% endif %}");
+  
+  try {
+    String idxCode = "YOUR INDEX CODE";
+    
+    /*
+     * The scoring function formula. 
+     * Check the functions' syntax documentation.
+     */
+    String formula = "pow(b(0),2) * age"
+     
+    // The function numeric identifier (zero based)
+    Integer functionId = 0;
+
+    it.addFunction(idxCode, functionId, definition);
+    
+  } catch (Exception e) {
+    System.out.println(e.getMessage());
+  }
+
        +
        + +
        + +
        + +{% endblock %} + diff --git a/storefront/templates/documentation/mysql-import.html b/storefront/templates/documentation/mysql-import.html new file mode 100644 index 0000000..13afb25 --- /dev/null +++ b/storefront/templates/documentation/mysql-import.html @@ -0,0 +1,57 @@ +{% extends "base.html" %} +{% load custom_tags %} + +{% block wrapper_class %}nosidebar{% endblock %} + +{% block content %} +
        + +
        + +

        MySQL import

        + +
        +

        + The MySql import tool allows you to feed your MySql database's contents to an IndexTank index without much effort. + Download the importer and configure the following settings section: + +

        +
        Download client
        + +
        + +
        +
        Importer settings
        +{% box 'code' %} +
        +   # The results for this query will be the indexed documents
+   # considering the field names in the query as the field names
+   # for the index.
+   # One of the fields must have the "document_id" alias to 
+   # indicate that it is the DOCUMENT IDENTIFIER field.
+
+   SELECT_QUERY = '<QUERY>'
+
+   # Database config  
+   DATABASE_HOST = '<HOST>'
+   DATABASE_USER = '<USER>'
+   DATABASE_PASSWORD = '<PASS>'
+   DATABASE_SCHEMA = '<DATABASE>'
+
+   # IndexTank config
+   API_KEY = '<YOUR API KEY>'
+   INDEX_NAME = '<YOUR INDEX CODE>'
+
        +{% endbox %} +
        + +

        +
        + +
        + +{% endblock %} + diff --git a/storefront/templates/documentation/php-client.html b/storefront/templates/documentation/php-client.html new file mode 100644 index 0000000..c0d7e22 --- /dev/null +++ b/storefront/templates/documentation/php-client.html @@ -0,0 +1,279 @@ +{% extends "documentation/client-base.html" %} +{% load custom_tags %} + + +{% block language-name %}php{% endblock %} + +{% block download-box %} + +
        +PHP client + +
        + +{% endblock %} + + +{% block summary-title %}PHP client{% endblock %} + + +{# #} +{% block code-instatiate %} + include_once "indextank.php"; + + $API_URL = '<YOUR API URL>'; + + $client = new Indextank_Api($API_URL); + $index = $client->get_index("<YOUR INDEX NAME>"); +{% endblock %} + + +{# #} +{% block code-simple-indexing %} + $text = 'this is a text'; + $doc_id = 1234; + $index->add_document($doc_id, array('text'=>$text)); +{% endblock %} + + +{# #} +{% block code-simple-searching %} + $query = "<YOUR QUERY>"; + $res = $index->search($query); + + echo $res->matches . " results\n"; + foreach($res->results as $doc) { + print_r($doc); + echo "\n"; + } +{% endblock %} + +{# #} +{% block code-query-delete %} + $query = "<YOUR QUERY>"; + $index->delete_by_search($query); + {% endblock %} + +{# #} +{% block code-snippet-searching %} + $query = "<YOUR QUERY>"; + $fetch_fields = "title,timestamp"; + $snippet_fields = "text"; + + $res = $index->search($query); + + $res = $index->search($query, NULL, NULL, NULL, + $snippet_fields, $fetch_fields); + echo $res->matches." results\n"; + foreach($res->results as $doc) { + $docid = $doc->doc_id; + $title = $doc->title; + $text = $doc->snippet_text; + echo "{% filter force_escape %}$title

        $text

        {% endfilter %}\n"; + } +{% endblock %} + +{% block code-simple-deleting %} + $doc_id = "<YOUR DOCUMENT ID>"; + + $index->delete_document($doc_id); +{% endblock %} + + +{# #} +{% block code-multiple-field-indexing %} + $index->add_document($doc_id, array('text'=>$text, + 'title'=>$title, + 'author'=>$author)); +{% endblock %} + + +{# #} +{% block code-filter-by-author %} + $res = $index->search($query." author:$author"); +{% endblock %} + + +{# #} +{% block code-index-with-timestamp %} + $index->add_document($doc_id, array('text'=>$text, 'timestamp'=>time())); +{% endblock %} + + +{# #} +{% block code-index-with-boosts %} + $variables = array('0'=>$rating, '1'=>$reputation, '2'=>$visits); + $index->add_document($doc_id, array('text'=>$text), $variables); +{% endblock %} + + +{# #} +{% block code-update-boosts %} + $index->update_variables($doc_id, $variables); +{% endblock %} + + +{# #} +{% block code-redefine-functions %} +# FUNCTION 0 : sorts by most recent + $index->add_function(0, "-age"); + +# FUNCTION 1 : standard textual relevance + $index->add_function(1, "relevance"); + +# FUNCTION 2 : sorts by rating + $index->add_function(2, "doc.var[0]"); + +# FUNCTION 3 : sorts by reputation + $index->add_function(3, "d[1]"); + +# FUNCTION 4 : advanced function + $index->add_function(4, "log(d[0]) - age/50000"); +{% endblock %} + +{# #} +{% block code-batch-indexing %} +$documents = array(); +$documents[]= array( "docid" => "doc1", "fields" => array( "text" => "text1 is short" ) ); +$documents[]= array( "docid" => "doc2", "fields" => array( "text" => "text2 is a bit longer" ) ); +$documents[]= array( "docid" => "doc3", "fields" => array( "text" => "text3 is longer than text2" ), "variables" => array( 0 => 1.5 ) ); +$response = $index->add_documents($documents); +{% endblock %} + +{% block code-batch-indexing-failed %} +$failed_documents = array(); +foreach ($response as $r) { + if (!$r->added) + $failed_documents[]= $r; +} +{% endblock %} + +{# #} +{% block code-bulk-delete %} +$docids = array("doc1", "doc2", "doc3", "doc4"); +$response = $index->delete_documents($docids); +{% endblock %} + +{% block code-bulk-delete-failed %} +$failed_documents = array(); +foreach ($response as $r) { + if (!$r->deleted) + $failed_documents[]= $r; +} +{% endblock %} + + +{# #} +{% block code-search-with-relevance-function %} +$relevance_function = 2; +$index->search($query, NULL, NULL, $relevance_function); +{% endblock %} + +{# #} +{% block code-proximity-scoring-function %} +# FUNCTION 5 : inverse distance calculated in miles +$index->add_function(5, "-mi(d[0], d[1], q[0], q[1])"); +{% endblock %} + + +{# #} +{% block code-search-with-geo %} +$variables = array( 0 => $latitude, 1 => $longitude ); + +$index->search($query, NULL, NULL, 5, NULL, NULL, NULL, $variables); + +{% endblock %} + +{# #} +{% block code-update-categories %} +categories = array('priceRange' => '$0 to $299', + 'articleType' => 'camera'); +$index->update_categories($docid, $categories); +{% endblock %} + +{# #} +{% block code-sample-facets-result %} +{ + 'matches': 8, + 'results': [ {'docid': 'doc1'}, ... ], + 'facets': { + 'articleType': { + 'camera': 5, + 'laptop': 3 + }, + 'priceRange': { + '$0 to $299': 4, + '$300 to $599': 4 + } + } +} +{% endblock %} + +{# #} +{% block code-faceting-filtering %} +$index.search($query, NULL, NULL, NULL, NULL, NULL, + array('priceRange' => array('$0 to $299', '$300 to $599'), + 'articleType' => array('camera')) + ); +{% endblock %} + +{% block code-range-queries %} +# Ranges are specified by a two elements list: +# bottom and top bounds. +# Both top and bottom can be NULL indicating they should be ignored. +# +# In this sample, the results will only include documents +# whose variable #0 value is between 5 and 10 or between 15 +# and 20, and variable #1 value is less than or equal to 3 + +$index.search($query, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + array( 0 => array(array(5,10), array(15, 20)), + 1 => array(array(NULL, 3)) + )); + +# This also applies to functions + +$index.search($query, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL + array( 0 => array(array(0.5,NULL)) ) + ); + +{% endblock %} + + +{# #} +{% block code-index-creation %} + include_once "indextank_client.php"; + + $API_URL = '<YOUR API URL>' + + $client = new ApiClient($API_URL); + + // this parameter allows you to create indexes with public search enabled. + // default is false. + $public_search_enabled = false; + + $index = $client->create_index("<YOUR INDEX NAME>", $public_search_enabled); + + while (!$index->has_started()) { + sleep(1); + } +{% endblock %} + + +{# #} +{% block code-index-deletion %} + $index->delete_index(); +{% endblock %} + diff --git a/storefront/templates/documentation/privacy_policy.html b/storefront/templates/documentation/privacy_policy.html new file mode 100644 index 0000000..d8b4d06 --- /dev/null +++ b/storefront/templates/documentation/privacy_policy.html @@ -0,0 +1,63 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block title %}Privacy Policy{% endblock %}endblock %} + +{% block common_content %} + +

        Flaptor, Inc. (“Flaptor,†“we,†or “usâ€) knows that you care how information about you is used and shared. This Privacy Policy explains what information of yours will be collected by Flaptor when you access the Flaptor Service (including through the websites of our partners), how the information will be used, and how you can control the collection, correction and/or deletion of information. We will not use or share your information with anyone except as described in this Privacy Policy. This Privacy Policy does not apply to information we collect by other means (including offline) or from other sources. Capitalized terms that are not defined in this Privacy Policy have the meaning given them in our Terms of Use [link].

        +

        Information We Collect

        + +

        Our Service: We may receive personally identifiable information about you through our website or through our partners who use our Service (including by embedding our code on their websites or through our APIs) to improve the offerings of their websites. When you visit such websites, your browser may send us certain information about you as described below. If you correspond with us by email, we may retain the content of your email messages, your email address and our responses. We may also retain any search query strings you send through the Service, including on a partner website.

        + +

        Cookies Information: We may send one or more cookies - a small text file containing a string of alphanumeric characters - to your computer that uniquely identifies your browser and lets learn about your behavior and usage patterns. A cookie may also convey anonymous information about how you browse the a partner website to us. A cookie does not collect personal information about you. A persistent cookie remains on your hard drive after you close your browser. Persistent cookies may be used by your browser on subsequent visits to the site. Persistent cookies can be removed by following your web browser’s directions. A session cookie is temporary and disappears after you close your browser. You can reset your web browser to refuse all cookies or to indicate when a cookie is being sent.

        + +

        Log File Information: Log file information is automatically reported by your browser each time you access a web page. When you access the Service, our servers automatically record certain information that your web browser sends whenever you visit any website. These server logs may include information such as your web request, Internet Protocol (“IPâ€) address, browser type, referring / exit pages and URLs, number of clicks, domain names, landing pages, pages viewed, and other such information.

        + +

        Clear Gifs Information: When you access the Service, we may employ clear gifs (also known as web beacons) which are used to track the online usage patterns of our users anonymously. No personally identifiable information from your Flaptor account is collected using these clear gifs. The information is used to enable more accurate reporting, improve the effectiveness of our Service, and make Flaptor better for our users and partners.

        + +

        How We Use Your Information

        + +

        We use the information that we collect to operate and maintain our Service, and to help our partners improve their online offerings.

        + +

        We use cookies, clear gifs, and log file information to: (a) monitor the effectiveness of our Service and the offerings of our partners; (b) monitor aggregate metrics such as total number of visitors, traffic, and demographic patterns; (c) diagnose or fix technology problems;.

        + +

        How We Share Your Information

        + +

        Personally Identifiable Information: Flaptor will not rent or sell your personally identifiable information to others. We may store personal information in locations outside the direct control of Flaptor (for instance, on servers or databases co-located with hosting providers).

        + +

        As we develop our business, we may buy or sell assets or portions of our business, Customer, Email and visitor information is generally one of the transferred business assets in these types of transactions. We may also transfer or assign such information in the course of corporate divestitures, mergers, or dissolution.

        + +

        Non-Personally Identifiable Information: We may share non-personally identifiable information (such as anonymous usage data, anonymous and/or aggregated search queries, referring/exit pages and URLs, platform types, number of clicks, etc.) with interested third parties to help them understand the usage patterns for certain Flaptor services and those of our partners. Non-personally identifiable information may be stored indefinitely.

        + +

        How We Protect Your Information

        + +

        Flaptor is concerned with protecting your privacy and data, but we cannot ensure or warrant the security of any information you transmit to Flaptor or guarantee that your information on the Service may not be accessed, disclosed, altered, or destroyed by breach of any of our physical, technical, or managerial safeguards. Credit card information that you disclose to us is stored by secure payment providers, and Flaptor complies with the PCI DSS security standard.

        + +

        Compromise of Personal Information

        + +

        In the event that personal information is compromised as a result of a breach of security, Flaptor will promptly notify those persons whose personal information has been compromised, in accordance with the notification procedures set forth in this Privacy Policy, or as otherwise required by applicable law.

        + +

        Your Choices About Your Information

        + + +

        You may update preferences at any time by [describe how to change, including any opt outs]. You can review and correct the information about you that Flaptor keeps on file by contacting us directly at support@indextank.com.

        + +

        Children's Privacy

        + +

        Protecting the privacy of young children is especially important. For that reason, Flaptor does not knowingly collect or solicit personal information from anyone under the age of 13. If you are under 13, please do not send any information about yourself to us, including your name, address, telephone number, or email address. No one under age 13 is allowed to provide any personal information to Flaptor. In the event that we learn that we have collected personal information from a child under age 13 without verification of parental consent, we will delete that information as quickly as possible. If you believe that we might have any information from or about a child under 13, please contact us at privacy@flaptor.com.

        + +

        Notification Procedures

        + +

        It is our policy to provide notifications, whether such notifications are required by law or are for marketing or other business related purposes, to you via email notice, written or hard copy notice, or through conspicuous posting of such notice on the Service, as determined by Flaptor in its sole discretion. We reserve the right to determine the form and means of providing notifications to you, provided that you may opt out of certain means of notification as described in this Privacy Policy.

        + +

        Changes to Our Privacy Policy

        +

        If we change our privacy policies and procedures, we will post those changes on the Service to keep you aware of what information we collect, how we use it and under what circumstances we may disclose it. Changes to this Privacy Policy are effective when they are posted on this page.

        + +

        If you have any questions about this Privacy Policy, the practices of this site, or your dealings with this website, please contact us at privacy@flaptor.com, or send mail to:

        + +

        Flaptor, Inc.

        +{% endblock %} + + + diff --git a/storefront/templates/documentation/python-client.html b/storefront/templates/documentation/python-client.html new file mode 100644 index 0000000..63a015b --- /dev/null +++ b/storefront/templates/documentation/python-client.html @@ -0,0 +1,284 @@ +{% extends "documentation/client-base.html" %} +{% load custom_tags %} + +{% block language-name %}python{% endblock %} + + +{% block download-box %} +
          +
        • + Package: pip install indextank {% with "pip install indextank" as text %}{% with "1" as small %}{% with "#999999" as bgcolor %}{% include 'includes/clippy.html' %}{% endwith %}{% endwith %}{% endwith %} +
          • +
        • + Stable Version - + zip | + tgz +
          • +
        • + All versions +
          • +
        • + Source code (github) +
          • +
        +{% endblock %} + + +{% block summary-title %}Python client{% endblock %} + + +{# #} +{% block code-instatiate %} +from indextank.client import ApiClient + +api = ApiClient({% if request.user.is_authenticated %}'{{ request.user.get_profile.account.get_private_apiurl }}'{% else %}'<YOUR API URL HERE>'{% endif %}) + +index = api.get_index('<YOUR INDEX NAME HERE>') +{% endblock %} + + +{# #} +{% block code-simple-indexing %} +id = '<YOUR DOCUMENT ID>' +text = '<THE TEXTUAL CONTENT>' + +index.add_document(id, { 'text': text }) +{% endblock %} + + +{# #} +{% block code-simple-searching %} +query = '<YOUR QUERY STRING>' +results = index.search(query) + +print '%d results' % results['matches'] +for doc in results['results']: + print 'docid: %s' % doc['docid'] +{% endblock %} + +{# #} +{% block code-query-delete %} +query = '<YOUR QUERY STRING>' +index.delete_by_search(query) +{% endblock %} + +{# #} +{% block code-snippet-searching %} +query = '<YOUR QUERY STRING>' +results = index.search(query, + fetch_fields=['title','timestamp'], + snippet_fields=['text']) + +print '%d results' % results['matches'] +for doc in results['results']: + docid = doc['docid'] + title = doc['title'] + timestamp = doc['timestamp'] + text = doc['snippet_text'] + print "{% filter force_escape %}%s

        %s

        {% endfilter %}" % (docid,title,text) +{% endblock %} + +{% block code-simple-deleting %} +id = "<YOUR DOCUMENT ID>" + +index.delete_document(id) +{% endblock %} + + +{# #} +{% block code-multiple-field-indexing %} +#### INDEX MULTIPLE FIELDS +index.add_document(id, { 'text': text, + 'title': title, + 'author': author }) +{% endblock %} + + +{# #} +{% block code-filter-by-author %} +#### FILTER TO USER'S CONTENT +index.search('%s author:%s' % (query,user)) +{% endblock %} + + +{# #} +{% block code-index-with-timestamp %} +index.add(id, { 'text': text, + 'timestamp': time.mktime(time.gmtime()) + }) +{% endblock %} + + +{# #} +{% block code-index-with-boosts %} +#### INDEX DOCUMENT WITH VARIABLES +doc = { 'text': text } +variables = { 0: rating, 1: reputation, 2: visits } + +index.add_document(id, doc, variables=variables) +{% endblock %} + + +{# #} +{% block code-update-boosts %} +#### UPDATE DOCUMENT VARIABLES +variables = { 0: new_rating, 1: new_reputation, 2: new_visits } +index.update_variables(id, variables=variables) +{% endblock %} + + +{# #} +{% block code-redefine-functions %} +# FUNCTION 0 : sorts by most recent +index.add_function(0, "-age") + +# FUNCTION 1 : standard textual relevance +index.add_function(1, "relevance") + +# FUNCTION 2 : sorts by rating +index.add_function(2, "doc.var[0]") + +# FUNCTION 3 : sorts by reputation +index.add_function(3, "d[1]") + +# FUNCTION 4 : advanced function +index.add_function(4, "log(d[0]) - age/50000") +{% endblock %} + + +{# #} +{% block code-search-with-relevance-function %} +index.search(query, scoring_function=2) +{% endblock %} + +{# #} +{% block code-proximity-scoring-function %} +# FUNCTION 5 : inverse distance calculated in miles +index.add_function(5, "-mi(d[0], d[1], q[0], q[1])") +{% endblock %} + + +{# #} +{% block code-search-with-geo %} +variables = { 0: latitude, 1: longitude } + +index.search(query, + scoring_function=5, + variables=variables) +{% endblock %} + +{# #} +{% block code-update-categories %} +categories = { + 'priceRange': '$0 to $299', + 'articleType': 'camera' + } +index.update_categories(docid, categories) +{% endblock %} + +{# #} +{% block code-sample-facets-result %} +{ + 'matches': 8, + 'results': [ {'docid': 'doc1'}, ... ], + 'facets': { + 'articleType': { + 'camera': 5, + 'laptop': 3 + }, + 'priceRange': { + '$0 to $299': 4, + '$300 to $599': 4 + } + } +} +{% endblock %} + +{# #} +{% block code-faceting-filtering %} +index.search(query, + category_filters={ + 'priceRange': ['$0 to $299', '$300 to $599'], + 'articleType': ['camera'] + }) +{% endblock %} + + +{# #} +{% block code-range-queries %} +""" + Ranges are specified by string with the format: + [BOTTOM_LIMIT,TOP_LIMIT] + Both top and bottom can be replaced by 'None' + to indicate they should be ignored. + + In this sample, the results will only include documents + whose variable #0 value is between 5 and 10 or between 15 + and 20, and variable #1 value is less than or equal to 3 + +""" +index.search(query, + docvar_filters={ + 0: [[5,10], [15,20]], + 1: [[None,3]] + }) + +""" + The same applies to functions +""" +index.search(query, + function_filters={ + 1: [[0.5,None]] + }) +{% endblock %} + + +{# #} +{% block code-batch-indexing %} +documents = [] +documents.append({ 'docid': 'doc1', 'fields': { 'text': text1 } }) +documents.append({ 'docid': 'doc2', 'fields': { 'text': text2 } }) +documents.append({ 'docid': 'doc3', 'fields': { 'text': text2 }, 'variables': { 0 : 1.5 } }) +documents.append({ 'docid': 'doc4', 'fields': { 'text': text2 }, 'variables': { 0 : 1.5 }, 'categories': { 'Price': '0 to 100' } }) + +response = index.add_documents(documents) +{% endblock %} + +{% block code-batch-indexing-failed %} +failed_documents = [documents[i] for i in xrange(len(response)) if not response[i]['added']] +{% endblock %} + +{# #} +{% block code-bulk-delete %} +docids = ['doc1', 'doc2', 'doc3', 'doc4'] +response = index.delete_documents(docids) +{% endblock %} + +{% block code-bulk-delete-failed %} +failed_documents = [docids[i] for i in xrange(len(response)) if not response[i]['deleted']] +{% endblock %} + +{# #} +{% block code-index-creation %} +from indextank.client import ApiClient +import time + +api = ApiClient({% if request.user.is_authenticated %}'{{ request.user.get_profile.account.get_private_apiurl }}'{% else %}'<YOUR API URL HERE>'{% endif %}) + +# this parameter allows you to create indexes with public search enabled. +# default is False. +public_search_enabled = False + +index = api.create_index('test_name', public_search_enabled) + +while not index.has_started(): + time.sleep(0.5) + +{% endblock %} + + +{# #} +{% block code-index-deletion %} +api.delete_index('test_name') +{% endblock %} + diff --git a/storefront/templates/documentation/query-syntax.html b/storefront/templates/documentation/query-syntax.html new file mode 100644 index 0000000..ca0829e --- /dev/null +++ b/storefront/templates/documentation/query-syntax.html @@ -0,0 +1,198 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block extrahead %} + +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block right_content %} + {% box 'table' %} + + {% endbox %} +{% endblock %} + +{% block title %}Query Syntax{% endblock %} +{% block common_content %} +

        A guide to understand IndexTank supported Query Format

        + +
        +

        Example dataset

        +

        All the examples in this page use an index composed of the following documents:

        + +
        +    documentA = { 
+        "text" : "Hello world. I'm the content of the 'text' field. It's the default field for queries.",
+        "city" : "San Francisco, CA",
+        "author" : "Java Power Coder"
+    } 
+
+    documentB = {
+        "text" : "Good morning! The default field of this document contains 'tea'.",
+        "city" : "London, UK",
+        "author" : "World Class Developer"
+    }
+    
+    documentC = {
+        "text" : "Goodbye world. This is the default field of the last document.",
+        "city" : "San Diego, CA",
+        "author" : "PHP Power Coder"
+    }
+
        + +
        +

        Basic Queries

        +

        There are two types of basic queries: Terms and Phrases. A Term is a single word such as "hello" or "world". + For example: + 

        hello
        + will match: + 
        documentA
        + + and + 
        world
        + will match: + 
        documentA, documentC
        + +
        Notice that documentB does not match, even when the field "author" contains world. See Specifying fields for queries for an explanation.
        + + + +

        A Phrase is a group of words by double quotes such as "hello world". Documents matching phrase queries will contain the exact phrase.

        + For example: + 
        "hello wold"
        + will match: + 
        documentA
        + + and + 
        "default field"
        + will match: + 
        documentA, documentB, documentC
        + + + + +
        +

        Specifying fields for queries

        +

        By default, all basic queries are executed against the Text field of the indexed documents. It's possible to select a different field, by prefixing a basic query with a field name.

        + For example: + 
        author:world
        + will match: + 
        documentB
        + + and + 
        author:"power coder"
        + will match: + 
        documentA, documentC
        + + + +
        +

        Boolean Operators

        +

        You can combine basic queries with Boolean operators to form a more complex query. + + Boolean operators define the relationships between Terms or Phrases. IndexTank supports the following Boolean operators: AND, "+", OR, NOT and "-". Please note that Boolean operators must be all uppercase.

        + +
        • AND
        +

        + + This is the default operator. It will be used if there is no Boolean operator between two terms. + For example: + 

        default document
        + is the same as + 
        default AND document
        + and will match: + 
        documentB, documentC
        +

        + +
        • OR
        +

        + This operator makes its surrounding terms optional, but at least one must match the document. + For example: + 

        hello OR last
        + will match: + 
        documentA, documentC
        +

        + +
        • NOT
        +

        + The NOT operator excludes documents that contain the term (or phrase) after NOT. + For example: + 

        world NOT content
        + will match: + 
        documentC
        + + and + 
        world NOT "default field"
        + will match: + 
        documentA
        + + + You can use the NOT operator several times in the same query. + For example: + 
        world NOT content NOT author:coder
        + will not match any document. + 
         # no documents matching
        +
        IndexTank DOES NOT support queries that ONLY have NOT terms.
        +

        + + + + +
        +

        Precedence

        + The order in which you enter search terms does not effect your results. AND and OR operators have the same precedence and group from left to right. + For example: + 
        hello OR last
        + is the same as + 
        last OR hello
        + and will match the same documents, + 
        documentA, documentC
        + + If you need to modify precedence for complex queries, you can use parentheses. + For example: + 
        (good AND world) OR author:power
        + will match: + 
        documentA, documentC
        + + and + 
        good AND (world OR author:power)
        + will match: + 
        documentC
        + + + +
        +

        Field Grouping

        +

        + You can also use parentheses to specify the field only once. +

        + For example: + 
        author:(world class)
        + is the same as: + 
        author:world author:class
        + and will match: + 
        documentB
        + +
        +

        Term Weights (caret operator)

        +

        + You can boost terms in a query by appending the caret operator at the end of a term. Example: +

        + 
        author:world^2 OR city:san^10
        + This will match all documents, and documentC will be the last result. +

        + + + +{% endblock %} + diff --git a/storefront/templates/documentation/ruby-client.html b/storefront/templates/documentation/ruby-client.html new file mode 100644 index 0000000..ad34213 --- /dev/null +++ b/storefront/templates/documentation/ruby-client.html @@ -0,0 +1,298 @@ +{% extends "documentation/client-base.html" %} +{% load custom_tags %} + +{% block language-name %}ruby{% endblock %} +{% block download-box %} +

          +
        • + Package: gem install indextank {% with "gem install indextank" as text %}{% with "1" as small %}{% with "#999999" as bgcolor %}{% include 'includes/clippy.html' %}{% endwith %}{% endwith %}{% endwith %} +
          • +
        • + Source code (github) +
          • +
        + +{% endblock %} + + +{% block summary-title %}Ruby client{% endblock %} +{% block summary-content %}Interface details at indextank yard documentation (from rubydoc.info).{% endblock %} +{% block extra-summary %} +

        If you're using IndexTank in Heroku through the Add-on program, usage of this client will be slightly different. You can see those differences in the Heroku Add-on documentation.

        + +
        +

        Installation

        +

        (Please make sure you have gem installed)

        +

        Get the indextank gem (might need administrative privileges):

        +

        + +{% box 'code' %} +
        +$ gem install indextank
+
        +{% endbox %} + +{% endblock %} + +{# #} +{% block code-instatiate %} +require 'rubygems' +require 'indextank' + +api = IndexTank::Client.new {% if request.user.is_authenticated %}"{{ request.user.get_profile.account.get_private_apiurl }}"{% else %}"<YOUR API URL HERE>"{% endif %} + +index = api.indexes "<YOUR INDEX NAME>" +{% endblock %} + + +{# #} +{% block code-simple-indexing %} +docid = "<YOUR DOCUMENT ID>" +text = "<THE TEXTUAL CONTENT>" + +index.document(docid).add({ :text => text }) +{% endblock %} + + +{# #} +{% block code-simple-searching %} +query = "<YOUR QUERY STRING>" + +results = index.search query + +print results['matches'], " results\n" +results['results'].each {|doc| + docid = doc['docid'] + print "docid: #{docid}" +} +{% endblock %} + +{# #} +{% block code-query-delete %} +query = "<YOUR QUERY STRING>" + +index.delete_by_search query +{% endblock %} + +{# #} +{% block code-snippet-searching %} +query = "<YOUR QUERY STRING>" +results = index.search(query, + :fetch => 'title,timestamp', + :snippet => 'text') + +print results['matches'], " results\n" +results['results'].each {|doc| + docid = doc['docid'] + title = doc['title'] + timestamp = doc['timestamp'] + text = doc['snippet_text'] + print "{% filter force_escape %}#{title}

        #{text}

        {% endfilter %}" +} +{% endblock %} + +{% block code-simple-deleting %} +docid = "<YOUR DOCUMENT ID>" + +index.document(docid).delete() +{% endblock %} + + +{# #} +{% block code-multiple-field-indexing %} +#### INDEX MULTIPLE FIELDS +index.document(docid).add({ :text => text, + :title => title, + :author => author }) +{% endblock %} + + +{# #} +{% block code-filter-by-author %} +#### FILTER TO USER'S CONTENT +index.search "#{query} author:#{user}" +{% endblock %} + + +{# #} +{% block code-index-with-timestamp %} +index.document(docid).add({ :text => text, + :timestamp => Time.now.to_i }) +{% endblock %} + + +{# #} +{% block code-index-with-boosts %} +#### INDEX DOCUMENT WITH VARIABLES +fields = { :text => text } +variables = { + 0 => rating, + 1 => reputation, + 2 => visits + } + +index.document(docid).add(fields, :variables => variables) +{% endblock %} + + +{# #} +{% block code-update-boosts %} +#### UPDATE DOCUMENT VARIABLES ONLY +new_variables = { + 0 => new_rating, + 1 => new_reputation, + 2 => new_visits + } + +index.document(docid).update_variables(new_variables) +{% endblock %} + + +{# #} +{% block code-redefine-functions %} +# FUNCTION 0 : sorts by most recent +index.functions(0, "-age").add + +# FUNCTION 1 : standard textual relevance +index.functions(1, "relevance").add + +# FUNCTION 2 : sorts by rating +index.functions(2, "doc.var[0]").add + +# FUNCTION 3 : sorts by reputation +index.functions(3, "d[1]").add + +# FUNCTION 4 : advanced function +index.functions(4, "log(d[0]) - age/50000").add +{% endblock %} + + +{# #} +{% block code-search-with-relevance-function %} +index.search(query, :function => 2) +{% endblock %} + +{# #} +{% block code-proximity-scoring-function %} +# FUNCTION 5 : inverse distance calculated in miles +index.functions(5, "-miles(d[0], d[1], q[0], q[1])").add +{% endblock %} + + +{# #} +{% block code-search-with-geo %} +index.search(query, + :function => 5, + :var0 => latitude, + :var1 => longitud) +{% endblock %} + +{# #} +{% block code-range-queries %} +# Ranges are specified by a two elements list: +# bottom and top bounds. +# Both top and bottom can be nil indicating they should be ignored. +# +# In this sample, the results will only include documents +# whose variable #0 value is between 5 and 10 or between 15 +# and 20, and variable #1 value is less than or equal to 3 +index.search(query, + :docvar_filters => { 0 => [ [5, 10], [15, 20] ], + 1 => [ [nil, 3] ]}) +# This also applies to functions +index.search(query, + :function_filters => { 0 => [ [0.5, nil]) +{% endblock %} + +{# #} +{% block code-update-categories %} +categories = { + 'priceRange' => '$0 to $299', + 'articleType' => 'camera' + } +index.document(docid).update_categories(categories) +{% endblock %} + +{# #} +{% block code-sample-facets-result %} +{ + 'matches' => 8, + 'results' => [ {'docid' => 'doc1'}, ... ], + 'facets' => { + 'articleType' => { + 'camera' => 5, + 'laptop' => 3 + }, + 'priceRange' => { + '$0 to $299' => 4, + '$300 to $599' => 4 + } + } +} +{% endblock %} + +{# #} +{% block code-faceting-filtering %} +index.search(query, + :category_filters => { + 'priceRange' => ['$0 to $299', '$300 to $599'], + 'articleType' => ['camera'] + }) +{% endblock %} + + +{# #} +{% block code-batch-indexing %} +documents = [] +documents << { :docid => 'doc1', :fields => { :text => text1 } } +documents << { :docid => 'doc2', :fields => { :text => text2 } } +documents << { :docid => 'doc3', :fields => { :text => text3 }, :variables => { 0 => 1.5 } } +documents << { :docid => 'doc4', :fields => { :text => text4 }, :variables => { 0 => 2.1 }, :categories => { 'Price' => '0 to 100' } } +response = index.batch_insert(documents) +{% endblock %} + +{% block code-batch-indexing-failed %} +failed_documents = [] +response.each_with_index do |r, i| + failed_documents << documents[i] unless r['added'] +end +{% endblock %} + +{# #} +{% block code-bulk-delete %} +docids = ["doc1", "doc2", "doc3", "doc4"] +response = index.bulk_delete(docids) +{% endblock %} + +{% block code-bulk-delete-failed %} +failed_documents = [] +response.each_with_index do |r, i| + failed_documents << docids[i] unless r['deleted'] +end +{% endblock %} + +{# #} +{% block code-index-creation %} +require 'indextank' + +api = IndexTank::Client.new {% if request.user.is_authenticated %}"{{ request.user.get_profile.account.get_private_apiurl }}"{% else %}"<YOUR API URL HERE>"{% endif %} +index = api.indexes "<YOUR INDEX NAME>" + +# this parameter allows you to create indexes with public search enabled. +# default is false. +index.add :public_search => false + +while not index.running? + sleep 0.5 +end + +# use the index +{% endblock %} + + +{# #} +{% block code-index-deletion %} +index = api.indexes "<YOUR INDEX NAME>" +index.delete +{% endblock %} + diff --git a/storefront/templates/documentation/staff.html b/storefront/templates/documentation/staff.html new file mode 100644 index 0000000..d6fc0cf --- /dev/null +++ b/storefront/templates/documentation/staff.html @@ -0,0 +1,150 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block title %} Our Team{% endblock %}endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block right_content %} +
        +
        +
        +
        +
        +

        Contact Us

        + +
        +
        +
        +
        +
        +{% endblock %} + + +{% block common_content %} +
        +

        About us

        +

        We've been working together as a team on search solutions since 2005. We've implemented search solutions at Reddit, Automattic (WordPress), BitTorrent, MyLife.com, Kayak, and dozens more startups and industry-leaders.

        +

        You can read more about IndexTank on our blog and twitter.

        +

        Have a question? Our team of experienced and friendly developers are quick to respond, and never more than a few clicks away, any time. Sometimes we’re burning the midnight oil too.

        + +
        + +
        +

        Diego Founder / CEO

        +

        Leading the team since 2005, Diego gained his search experience working with Inktomi. He is a + rock climber, piano/guitar/bass player, and, as confirmed by many, an armchair mad-scientist.

        +
        I wrote some of the world's first web-scale link analysis algorithms, and now am on a mission to make every search box blazing fast and useful.
        + +
        + +
        +
        +
        + +
        +

        Spike Sofware engineer / architect

        +

        Spike has been part of the team since 2006 working on search with clients such as WordPress, + SideStep/Kayak, BitTorrent, Hounder.org and Wink/MyLife.com. He’s also led full-text search + workshops here in San Francisco. Spike is an electronic hobbyist, gadget builder, and amateur baritone.

        +
        I like to work on fault-tolerance and distributed
        decision-making algorithms.
        + +
        + +
        +
        +
        + +
        +

        Santi Software engineer and math enthusiast

        +

        Santi joined back in 2007. Before that he had worked on core java libraries at Google. He worked + on the open source search project Hounder.org and more recently, + he has developed the Trendistic search engine. Now he's working on IndexTank's searching infrastracture. On the weekends, when he is not challenging himself with math problems (ACM ICPC Latin American champion 2008, Google Code Jam Finalist, 2005), he plays soccer, enjoys traveling and going out to enjoy Buenos Aires.

        +
        I like solving complex algorithmic challenges and designing good, efficient and maintainable code.
        + +
        + +
        +
        +
        + +
        +

        Nacho Software engineer, linguistics specialist

        +

        Ignacio — call him Nacho — joined the team back in 2007, a week after his younger brother, + Santi. Nacho has worked on search for Hounder.org, WordPress, Trendistic and many others. Most recently, + he’s been focused on developing Instant Insights. Nacho is a chess player, marathon runner and a tenor + in-training. He also teaches Semiotics at the University of Buenos Aires.

        +
        I like problems dealing with semantic text processing, statistical semantic models, NLP in general.
        + +
        + +
        +
        +
        + +
        +

        Jorge Software engineer / architect

        +

        Jorge came to work with us back in 2005, and has primarily focused on back-end systems, infrastructure + and architecture design. He led the development of search for BitTorrent. He also authored one of our + white papers, and leads our work with Amazon Web Services. Jorge is a pilot and likes to spoil his kids.

        +
        I like challenges, problems that re­quire crea­ti­vi­ty, keeping things clean and simple.
        + +
        + +
        +
        +
        + +
        +

        Mono Software engineer

        +

        Diego Buthay, nicknamed Mono, joined back in 2006, and has worked on search projects with Healthcare.com, Clarin (largest newspaper in Argentina), WordPress and BitTorrent. Most recently he has worked on our WordPress plug-in. Mono is a Rubik cube solver, console freak and Counter-Strike sniper. He can put together any demo in two days, but just needs to make sure the coffee machine is plugged in.

        +
        I like working with backend stuff, everything from web servers and load balancer to our full-text-search engine.
        + +
        + +
        +
        +
        + +
        +

        Adrian Software engineer

        +

        Adrian Fernandez recently joined our team, coming from Technisys, where he designed banking systems for + a variety of clients. He is a self-proclaimed statistics junkie, amateur illustrator and fan of French + electronic music.

        +
        I like designing software using modularity, with elegant solutions and thinking with simplicity. Most of the time I'm focused on distributed systems.
        + +
        + +
        +
        +
        + +
        +

        Leandro Software engineer

        +

        Leandro recently joined the team from MercadoLibre.com, Latin America’s number one e-commerce site with 42 million registered users. He is an avid trekker, climber, mountain enthusiast and soccer player.

        +
        I like solving problems in which creativity is a must have. When all standard solutions have failed, here I come with my crazy ideas.
        + +
        + +
        +
        +
        + +
        +

        Sean Software engineer

        +

        Sean recently joined the team from the University of Chicago. Sean is a soccer player, gamer, cheese fan, and amateur polymath.

        +
        I like working on algorithms and improving the user experience- sometimes this means brevity.
        + +
        + +
        +
        +
        + +{% endblock %} + diff --git a/storefront/templates/documentation/terms_of_service.html b/storefront/templates/documentation/terms_of_service.html new file mode 100644 index 0000000..4693b6b --- /dev/null +++ b/storefront/templates/documentation/terms_of_service.html @@ -0,0 +1,99 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block right_content %} +

        Table of contents

        + +{% endblock %} + +{% block title %}Terms of Service (ToS){% endblock %} + +{% block common_content %} + +

        About

        +
        +
          +
        • + When you sign up to use IndexTank, a service of Flaptor, Inc. ("IndexTank"), you automatically confirm that you accept all the terms in this agreement. +
          • + +
        • + IndexTank may change the terms of this agreement from time to time without prior notice. You can review the most recent version of the Terms of Service at http://indextank.com/documentation/terms_of_service. +
          • +
        +
        + +

        Ownership

        +
        +
          +
        • + We don't claim ownership or copyright of your data. +
          • + +
        • + You agree that you are responsible for, and that you own the rights to the data you send to IndexTank. +
          • + +
        • + You are responsible for keeping your private data private (including your account password and your private API url), and for protecting any sensitive content you send to IndexTank by using encrypted transmission and showing your data (as presented by search, autocomplete, instant-search, instant-links, did-you-mean results or any other way in which IndexTank provides your content in full or partial form through its API or website) only to authorized persons. +
          • +
        +
        + +

        Payment

        +
        +
          +
        • + IndexTank uses third party Payment Processing providers to process your payment information. We do not store your credit card number on our servers. +
          • + +
        • + If you fail to pay for a paid account and don't respond to reasonable attempts to contact you on the email address you provided, we may cancel your account or revert it to a free account, with the possible loss of your content. +
          • + +
        • + IndexTank doesn't refund partially used or unused months (for example if you cancel your account mid-way through a month, already paid for). +
          • + +
        • + We reserve the right to terminate your account at any time for any reason. If you abuse the service in any way (hacking attempts, denial of service attacks, illegal content, etc) we will terminate your account. +
          • + +
        • + You understand and agree that the service is provided "as is" and "as available", and that access may involve third party fees (such as bandwidth charges from your internet provider). +
          • +
        +
        + +

        Quality of Service

        +
        +
          +
        • + We commit to providing responses within 250 ms for 95% of the search requests, as measured from the time a request arrives to our servers to the time the response leaves our servers. +
          • + +
        • + We commit to a monthly uptime of 99.3%, provided you keep your indexes within the limits defined for your account. +
          • +
        +
        + +

        Liability

        +
        +
          +
        • + You expressly understand and agree that Flaptor, Inc., its affiliates, officers, agents, partners, or employees are not liable for any direct, indirect, incidental, special, consequential or exemplary damages, including but not limited to damages for loss of profits, goodwill, use, data or other intangible losses, resulting from the use or inability to use the service, the unauthorized access to or alteration of the transmission of your data, or any other matter relating to the service. +
          • +
        +
        + +{% endblock %} + diff --git a/storefront/templates/documentation/thinkingtank.html b/storefront/templates/documentation/thinkingtank.html new file mode 100644 index 0000000..29c70fb --- /dev/null +++ b/storefront/templates/documentation/thinkingtank.html @@ -0,0 +1,118 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load macros %} + +{% enablemacros %} + +{% macro language-name %} +{% endblock %} + +{% block extrahead %} + +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block right_content %} + {% box 'table' %} + + {% endbox %} +{% endblock %} + +{% block title %}ThinkingTank, a Ruby Gem{% endblock %}endblock %} + +{% block common_content %} +

        +

        ThinkingTank is a simple ActiveRecord extension that allows you to define how to index your models and gives you easy tools to + search them. +

        +

        + + +
        +

        Setting up ThinkingTank

        + +

        First you'll need to install the ThinkingTank ruby gem

        +

        +{% box 'code' %} +
        +gem install thinkingtank
+
        +{% endbox %} + +

        +

        Then in your rails app you'll have to require this gem by editing your Gemfile:

        +

        +{% box 'code' %} +
        +gem "thinkingtank"
+
        +{% endbox %} + +

        +

        You'll also need to add a config/indextank.yml file to configure your API url, for instance:

        +

        +{% box 'code' %} +
        +development:
+    api_url: '<YOUR API URL HERE>'
+    index_name: '<YOUR DEVELOPMENT INDEX NAME HERE>'
+
        +{% endbox %} + +

        + + +
        +

        Indexing your models

        + +

        To make your models indexable, you'll need to add a define_index block to your model class. For instance, if you have + a Post model that defines a title and a body and you want both fields to be indexed you could do it like this:

        +

        +{% box 'code' %} +
        +class Post < ActiveRecord::Base
+  define_index do
+    indexes title
+    indexes body
+  end
+end
+
        +{% endbox %} + +

        +

        This will allow you to use the search method in Post, like this:

        +

        +{% box 'code' %} +
        +Post.search(query)
+
        +{% endbox %} + +

        +

        Behind the scenes, ThinkingTank will submit a post for indexing after you've saved it. + It will index each indexable field respecting the field names and it will create two special fields. + One called "__any" with the union of every indexable field (this will be used by default when searching) + and another one called "__type" which will have information to discriminate different models inside an + index. You'll be able to search across all fields by using the default field but you can also ask for terms + in specific fields by using the field:term syntax.

        +

        Once you have configured all the model you wanted you can use the rake task reindex to index all the + existing that. This task will re-create the index (if it had never been created, this creates it for you), + losing any data indexed in that index that is not in your DB and index every instance of the indexable models.

        +

        +{% box 'code' %} +
        +rake indextank:reindex
+
        +{% endbox %} + +

        +

        If you change the way your models are indexed you can also use that task to apply your changes retroactively

        +

        +{% endblock %} + diff --git a/storefront/templates/documentation/tutorial-autocomplete.html b/storefront/templates/documentation/tutorial-autocomplete.html new file mode 100644 index 0000000..e324dca --- /dev/null +++ b/storefront/templates/documentation/tutorial-autocomplete.html @@ -0,0 +1,183 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block extrahead %} + +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block right_content %} + {% box 'table' %} + + {% endbox %} + +{% endblock %} + +{% block title %}AutoComplete Tutorial{% endblock %}endblock %} + +{% block common_content %} +
        + +

        How to implement autocomplete, a user convenience feature

        +

        About Autocomplete

        +

        To make life easier for users of your IndexTank-powered search application, +add autocomplete functionality. With autocomplete, your +app predicts what search term the user is trying to enter while the user is +still typing. A list of suggestions is displayed and constantly updated as the +user types more characters, until the user is able to pick the correct term +from the list.

        +

        Why Use Autocomplete?

        +

        Selecting a term rather than typing every character +saves users unnecessary keystrokes, increases perceived +ease-of-use, and makes users happy. +It also reduces spelling errors that can adversely affect search result quality. +Finally, autocomplete is now so common in search applications, its absence +might be surprising to users. Luckily, it's not too difficult to set up.

        +

        How It Works

        +

        Autocomplete is performed by two components: +

          +
        • Client, which sends requests with a partial query term. + This is the part you need to code, and this page shows you how.
          • +
        • Server, which responds with a list of matches + for the partial query string. + IndexTank provides this part. If you're curious about how the server handles data, see + API Specification.
          • +
        +

        The client and server exchange data in JSON/JSONP format. +This is handled through jQuery's AJAX support.

        +

        Setup

        +

        Before you start, you'll need to get a few things ready

        +

        Tools for This How-To

        +

        In this page, we'll show how to implement autocomplete by using:

        +
          +
        • JavaScript +

          You don't have to do anything to set up JavaScript, but + users will need to have JavaScript enabled in their browsers + for the autocomplete scripts to work.

          +
          • +
        • jQuery, jQuery UI and Indextank-jQuery +

          In Quick Start, we show how to get + jQuery, jQuery UI and indextank-jQuery set up.

          +
          • +
        • Ruby on Rails + ERB + templating (optional) +

          Rails ERB is used to construct the form containing the search box + where autocomplete will appear to the user, but you can build the form + using your own favorite technique.

          • +
        +

        Know Your Public URL

        +

        Every index on IndexTank has a unique public URL, +provided for the purpose of interacting with clients. +You'll need to use this URL in requests from your autocomplete client. +Find the public URL on the Dashboard. +

        In general, a public URL has the form:

        + +{% box 'code' %} +
        +http://<public part>.api.indextank.com
+
        +{% endbox %} + +

        Example:

        + +{% box 'code' %} +
        +http://123abc.api.indextank.com
+
        +{% endbox %} + + +

        Quick Start: Copy & Paste

        + +

        Tweak the following code snippets for your +needs, and you're ready to go

        + +

        For the client side implementation it's recommended (but not required) that you use +Indextank-jQuery + +

        (Optional) Replace the value flick +with another jQuery UI theme name to change the appearance of the widget. +See Theming jQuery UI.

        + +{% box 'code' %} +
        +<script src='https://ajax.googleapis.com/ajax/libs/jquery/1.5.2/jquery.min.js' type='text/javascript'></script>
+<script src='https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.11/jquery-ui.min.js' type='text/javascript'></script>
+<script src='https://raw.github.com/flaptor/indextank-jquery/master/jquery.indextank.ize.js' type='text/javascript'></script>
+<script src='https://raw.github.com/flaptor/indextank-jquery/master/jquery.indextank.autocomplete.js' type='text/javascript'></script> 
+<link href='http://ajax.googleapis.com/ajax/libs/jqueryui/1.8.7/themes/flick/jquery-ui.css' rel='stylesheet'> 
+
        +{% endbox %} + +

        The following code sets up some values for later use. +

        Replace the values yourPublicURL +and yourIndexName with your own values +from your Dashboard. + +{% box 'code' %} +

        +<script type='text/javascript'> 
+var publicApiUrl = "yourPublicURL";
+var indexName = "yourIndexName";
+</script> 
+
        +{% endbox %} + +

        The following code shows the "document ready" callback function, +which enables autocomplete suggestions on the input box. You can change #myform and #query to match +your templates structure.

        + +{% box 'code' %} +
        +<script type='text/javascript'>
+
+$(document).ready(function(){
+    // let the form be 'indextank-aware'
+    $("#myform").indextank_Ize(publicApiUrl, indexName);
+    // let the query box have autocomplete
+    $("#query").indextank_Autocomplete();
+});
+
+
+</script> 
+
        +{% endbox %} + +

        Finally, the following code shows the form with the search box, with id query +This form is written using Ruby on Rails +ERB. +If you already have a form set up, you can use it +instead of this example, +as long as the search box name in your form matches up +with the rest of our sample code +(change your search box's id to query, and your form's id to myform). +

        + +{% box 'code' %} +
        +<%= form_tag search_path, :method => :get do %>
+  <%= text_field_tag :query %>
+  <button type="submit">Search</button>
+<% end %>
+
        +{% endbox %} + + +

        More Information

        + + +
        +{% endblock %} diff --git a/storefront/templates/documentation/tutorial-base.html b/storefront/templates/documentation/tutorial-base.html new file mode 100644 index 0000000..1a93f39 --- /dev/null +++ b/storefront/templates/documentation/tutorial-base.html @@ -0,0 +1,740 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block extrahead %} +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block right_content %} + +{% box 'table' %} + +{% endbox %} +{% endblock %} + +{% block title %}“Hello, World” Tutorial ({% block language %}{% endblock %}){% endblock %}endblock %} + +{% block common_content %} +
        +

        This Tutorial is for webmaster/programmers. By practicing simple tasks at + the command line, you will learn the basics of how to: +

        +
          +
        • + Sign up for IndexTank +
          • +
        • + Create an index and populate it with searchable text +
          • +
        • + Send search queries to the index and understand the results +
          • +
        • Tweak the search results and influence the order in which they appear (with scoring functions and variables) +
          • +
        • + Delete documents +
          • +
        +

        Before You Start

        +

        + To run the tutorial, you will need: +

        +
          +
        • + {% block language_prerequisite %} + {% endblock %} +
          • +
        • + Internet access + + (You must be able to view indextank.com + in a Web browser) + . +
          • +
        • + Knowledge of your computer's command prompt + + (You will need to open a console window and issue commands at the prompt in order to perform the steps in the tutorial) + . +
          • +
        +

        + System requirements: +

        +
          + {% block system_requirements %} + {% endblock %} +
        +

        + The following would be helpful before you start, but are not required: +

        +
          +
        • + Working knowledge of a programming language + + (You will be able to complete the Tutorial just by following along with our example code, but in order to build a real application using what you've learned, you will have to do some real programming through our HTTP API or, more likely, one of our + client libraries) + . +
          • +
        • + Read the FAQ to familiarize yourself with the vocabulary and general architecture of IndexTank + + (The Tutorial will go more smoothly if you already know what kinds of things you can do with IndexTank, what an index is, and so on. Each step in the tutorial will provide links to any relevant FAQ sections, so, if you prefer, you can learn each concept at the moment you need to know it rather than reading all the concepts ahead of time) + . +
          • +
        +

        About The Example App

        +

        + The Tutorial shows the process for setting up an index and performing some example queries on a fictitious web site that includes a forum where members discuss video games. +

        +

        Step 1: Sign Up for IndexTank's Free Plan

        +
        + Background Concepts: + See Sign-Up, Pricing, and Billing + in the FAQ. +
        +
          +
        1. +

          Open your browser and go to http://indextank.com/get-started.

          +
          2. +
        2. +

          Type your email address.

          +
          3. +
        3. +

          Go to http://indextank.com/dashboard.

          +
          4. +
        +

        + Result: +
        + The IndexTank dashboard appears, showing your new account: +

        +

        + +

        +

        Step 2: Create an Empty Index

        +
        + Background Concepts: + See What is an index? + in the FAQ +
        +
          +
        1. +

          Click new index.

          +

          + {% box 'mediumbox' %} + + {% endbox %} +

          +
          2. +
        2. +

          In Index Name, type test_index.

          +
          3. +
        3. +

          Click Create Index.

          +

          + {% box 'mediumbox' %} + + {% endbox %} +

          +

          + Result: +
          + The dashboard is displayed. In INDEX NAME, test_index + appears. In STATUS, you can see whether the index is ready to use: +

          +

          + {% box 'mediumbox' %} + + {% endbox %} +

          +
          4. +
        4. +

          Wait for a short time (typically less than one minute) to give IndexTank time to set up the cloud resources for your index.

          +
          5. +
        5. +

          Click your browser's Refresh button.

          +

          + If STATUS has changed to Running, you can proceed to the next part of the tutorial. If STATUS is still Initializing, wait a bit, then hit Refresh again. +

          +

          + {% box 'mediumbox' %} +

          + {% endbox %} + +
          6. +
        +

        + Result: +
        + You now have an empty index that is ready to be populated with content. +

        +

        Step 3: Download the Client Library

        +
        + Background Concepts: + See What languages do you support? + in the FAQ +
        +
          +
        1. +

          Click client documentation or go to http://indextank.com/documentation.

          +

          + {% box 'mediumbox' %} + + {% endbox %} +

          +
          2. + {% block download_client %} + {% endblock %} +

          Step 4: Instantiate the Client

          + {% block instantiate_client %} + {% endblock %} +

          Step 5: Set Up the Index

          +
          + Background Concepts: + See How do I get my data to you? + in the FAQ +
          +
            +
          1. +

            Get a handle to your test index. + {% block get_handle %} + {% endblock %} +

            +
            2. +
          2. +

            Add some documents to the index.

            + {% block add_documents1 %} + {% endblock %} +
            + NOTE: + In a real application, the doc ID would most likely be a URL. +
            +

            + + post1, + + post2, and + + post3 + + are unique alphanumeric IDs you give to the documents. If the doc ID is not unique, you will overwrite the existing document with the same ID, so watch out for typos during this Tutorial! +

            +

            + The method parameters are name:value pairs that build up the index for one document. In this example, there is a single name:value pair for each forum post. +

            +

            + + text + + is a field name, and it has a special meaning to IndexTank: in search queries, IndexTank defaults to searching + + text + + if the query does not specify a different field. You'll learn more about this later, in Use Fields to Divide Document Text. +

            +
          +

          + Result: +
          + test_index now contains: +

          +

          + + + + + + + + + + + + + + + + + + + + + +
          + Doc ID + + Field + + Value +
          + post1 + + text + + I love Bioshock +
          + post2 + + text + + Need cheats for Bioshock +
          + post3 + + text + + I love Tetris +
          +

          + +
        + +
        + Background Concepts: + See What types of queries work with IndexTank? + in the FAQ +
        +
          +
          + NOTE: + The documents so far have only one field, + + text, + so the search query doesn't have to specify a field. +
          +
        1. +

          Suppose you're interested in a particular game, and you want to find all the posts that contain Bioshock:

          + {% block search1 %} + {% endblock %} +

          + The output should look like this (you can ignore + + facets + + for now). The search term was found in post1 and post2: +

          + {% block search1_output %} + {% endblock %} +
          + NOTE: + AND is the default search operator, so you can just list all the search terms. +
          +
          2. +
        2. +

          Suppose you want to find only the true enthusiasts on the forum. You can search for posts that contain Bioshock and love.

          + {% block search2 %} + {% endblock %} +

          + The output should look like this. The two search terms were found together only in post1: +

          + {% block search2_output %} + {% endblock %} +
          3. +
        3. +

          You can also use the query operators OR and NOT. Let's try OR, which would come in handy if you play more than one game and you want to find posts that mention any of your favorites.

          + {% block search3 %} + {% endblock %} +

          + The output should look like this: +

          + {% block search3_output %} + {% endblock %} +
          4. + {% block fetch_fields %} + {% endblock %} +

          + Here, + + text + + means we would like the full text of the document where the search term was found. This would be useful, for example, to construct an output page that provides complete result text for the reader to look at. The output should look like this: +

          + {% block fetch_fields_output %} + {% endblock %} + + {% block snippet_fields %} + {% endblock %} +

          + The output should look like this: +

          + {% block snippet_fields_output %} + {% endblock %} + +
        +

        Step 7: Use Fields to Divide Document Text

        +
        + Background Concepts: + See the discussion of field names in What is an index? + in the FAQ +
        +

        + So far, we have worked with simple document index entries that contain only a single field, + + text, + containing the complete text of the document. Let's redefine the documents now and add some more fields to enable more targeted searching. +

        +
          +
        1. +

          Set up two fields for each document: the original + + text + + field plus a new field, + + game, + that contains the name of the video game that is the subject of the forum post.

          + {% block add_fields %} + {% endblock %} +

          + Result: +
          + test_index now contains: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + Doc ID + + Field + + Value +
          + post1 + + text + + I love Bioshock +
          + game + + Bioshock +
          + post2 + + text + + Need cheats for Bioshock +
          + game + + Bioshock +
          + post3 + + text + + I love Tetris +
          + game + + Tetris +
          + +

          2. +
        2. + {% block search_field %} + {% endblock %} +

          + Note that we can return the contents of a field even if it is not being searched. The output should look like this: +

          + {% block search_field_output %} + {% endblock %} +
          3. +
        +

        Step 8: Customize Result Ranking with Scoring Functions

        +
        + Background Concepts: + See What is a scoring function? + in the FAQ +
        +

        + A scoring function + is a mathematical formula that you can reference in a query to influence the ranking of search results. Scoring functions are named with integers starting at 0 and going up to 5. Function 0 is the default and will be applied if no other is specified; it starts out with an initial definition of + + -age, + which sorts query results from most recently indexed to least recently indexed (newest to oldest). +

        +

        + Function 0 uses the + + timestamp + + field which IndexTank provides for each document. The time is recorded as the number of seconds since epoch. IndexTank automatically sets each document's timestamp to the current time when the document is indexed, but you can override this timestamp. To make this scoring function tutorial easier to follow, that's what we are going to do. +

        +
          +
        1. +

          Assign timestamps to some new posts in the index.

          + {% block timestamps %} + {% endblock %} +
          2. +
        2. +

          Search using the default scoring function.

          + {% block scoring1 %} + {% endblock %} +

          + The output should look like this. The default scoring function has sorted the documents from newest to oldest: +

          + {% block scoring1_output %} + {% endblock %} +
          3. +
        3. +

          Redefine function 0 to sort in the opposite order, by removing the negative sign from the calculation.

          + {% block scoring2_redefine %} + {% endblock %} +
          4. +
        4. +

          Search again.

          + {% block scoring2_search %} + {% endblock %} +

          + The output should look like this. The oldest document is now first: +

          + {% block scoring2_output %} + {% endblock %} +
          5. +
        5. +

          Let's try creating another scoring function, function 1, using a different IndexTank built-in score called + + relevance. + Relevance is calculated using a proprietary algorithm, and indicates which documents best match a query. First, add some test documents that will more clearly illustrate the effect of the relevance score.

          + {% block add_documents2 %} + {% endblock %} +
          6. +
        6. +

          Now define function 1.

          + {% block define_function %} + {% endblock %} +
          7. +
        7. +

          Search using the new scoring function.

          + {% block search4 %} + {% endblock %} +

          + The output should look like this. The most relevant document is now first: +

          + {% block search4_output %} + {% endblock %} +
          8. +
        +

        Step 9: Add Document Variables To Your Scoring Functions

        +
        + Background Concepts: + See What is a scoring function? + in the FAQ +
        +

        + In addition to textual information, each document can have up to three (3) document variables + to store any numeric data you would like. Each variable is referred to by number, starting with variable 0. Document variables provide additional useful information to create more subtle and effective scoring functions. +

        +

        + For example, assume that in the video game forum, members can vote for posts that they like. The forum application keeps track of the number of votes. These vote totals can be used to push the more popular posts up higher in search results. +

        +

        + Let's also assume that the forum software assigns a spam score + by examining each new post for evidence that it is from a legitimate forum member and contains relevant content, and then assigning a confidence value from 0 (almost certainly spam) to 1 (high confidence that the post is legitimate). +

        +
          +
        1. +

          Assign the total votes to document variable 0 and the spam score to document variable 1.

          + {% block assign_vars %} + {% endblock %} +
          2. +
        2. +

          Use the document variables in a scoring function.

          + {% block function_vars %} + {% endblock %} +
          3. +
        3. +

          Run a query using the scoring function.

          + {% block scoring3 %} + {% endblock %} +

          + The output should look like this: +

          + {% block scoring3_output %} + {% endblock %} +
          4. +
        4. +

          When more readers vote for a post, update the vote total in variable 0.

          + {% block update_var %} + {% endblock %} +
          5. +
        5. +

          Now run the query again with the same scoring function.

          + {% block scoring4 %} + {% endblock %} +

          + The output should show the new most-popular post first: +

          + {% block scoring4_output %} + {% endblock %} +
          6. +
        +
        + Learn More: + Scoring Functions +
        +

        Step 10: Delete a Document

        +

        + If you're 100% confident something should not be in the index, it makes sense to remove it. +

        +
          +
        1. +

          Take out that spam document.

          + {% block delete_document %} + {% endblock %} +
          2. +
        2. +

          Search again to confirm the deletion.

          + {% block search5 %} + {% endblock %} +

          + The output should show only two results: +

          + {% block search5_output %} + {% endblock %} +
          3. +
        +

        Step 11: Use Variables to Refine Queries

        +

        + You can pass variables with a query and use them as input to a scoring function. This is useful, for example, to customize results for a particular user. Suppose we're dealing with the search on the forum site. It makes sense to index the poster's gamerscore + to use it as part of the matching process. +

        +
          +
        1. +

          Add a document variable. This will be compared to the query variable later. Here variable 0 holds the gamerscore of the person who made the post:

          + {% block add_doc_var %} + {% endblock %} +
          2. +
        2. +

          Suppose we want to boost posts from forum members with a gamerscore closer to that of the searcher. Let's define a scoring function to do that.

          + {% block scoring5 %} + {% endblock %} +

          + This scoring function prioritizes gamerscores close to the searcher's own (query.var[0]). + The absolute (abs) + function is there to provide symmetry for gamerscores above and below the searcher's. The + + max + + function ensures that we never divide by 0, and evens out all gamerscore differences less than 1 (in case we use floats for the gamerscores, instead of integers). +

          +
          3. +
        3. +

          Suppose John is the searcher, with a gamerscore of 25. In the query, set variable 0 to the gamerscore.

          + {% block search6 %} + {% endblock %} +

          + The output should look like this: +

          + {% block search6_output %} + {% endblock %} +
          4. +
        4. +

          For Isabelle, gamerscore 15,000:

          + {% block search7 %} + {% endblock %} +

          + The output should look like this: +

          + {% block search7_output %} + {% endblock %} +
          5. +
        +

        Next Steps

        +

        + Now that you have learned some of the basic functionality of IndexTank, you are ready to go more in-depth: +

        +
          +
        • + Client library documentation + will tell you more about the specific capabilities and syntax of the client in your programming language. +
          • +
        • + Scoring functions + goes into much more detail about formulas, operators, variables, and functions. +
          • +
        +

        + Enjoy using IndexTank to improve the quality of search on your website. +

        +
        +{% endblock %} + diff --git a/storefront/templates/documentation/tutorial-faceting.html b/storefront/templates/documentation/tutorial-faceting.html new file mode 100644 index 0000000..74d7085 --- /dev/null +++ b/storefront/templates/documentation/tutorial-faceting.html @@ -0,0 +1,480 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block extrahead %} + + + +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block right_content %} + {% box 'table' %} + + {% endbox %} + +{% endblock %} + +{% block content_head %}

        Faceting Tutorial

        {% endblock %} + +{% block common_content %} +
        + +

        How to implement faceting, an enhancement to the display of search results

        +

        About Faceting

        +

        +To make it easier for users to quickly spot the most relevant search results, add faceting functionality. +With faceting, search results are grouped under useful headings, using tags you apply ahead of time to the documents in +your index. For example, the results of a shopping query for books might be grouped according to the type of book and the price: +

        +
        + + + + +
        NARROW YOUR RESULTS BY:
        + + + + + + + + + + + + + + + + + + + + +
        Most popular
        +
          +
        • + Top rated (5) +
          • +
        • + "Very Good" (25) +
          • +
        +
        Content
        +
          +
        • + Cookbooks (10) +
          • +
        • + Romance (15) +
          • +
        • + Science Fiction (13) +
          • +
        • + Travel (30) +
          • +
        +
        Price
        +
          +
        • + Less than $10 (5) +
          • +
        • + $10-$19.99 (200) +
          • +
        • + $20-$29.99 (140) +
          • +
        • + $30 and up (46) +
          • +
        +
        +
        + +

        +Each time the user clicks a facet value, the set of results is reduced to only the items that have that value. +Additional clicks continue to narrow down the search—the previous facet values are remembered and applied again. +For example, the user might click "Cookbooks," and see a list of all 10 cookbook titles next to an updated navigation +list like this: +

        + +
        + + + + +
        NARROW YOUR RESULTS BY:
        + + + + + + + + + + + + + + +
        Most popular
        +
          +
        • + Top rated (1) +
          • +
        • + "Very Good" (3) +
          • +
        +
        Price
        +
          +
        • + Less than $10 (1) +
          • +
        • + $10-$19.99 (4) +
          • +
        • + $20-$29.99 (4) +
          • +
        • + $30 and up (1) +
          • +
        +
        +
        + +

        +If price is more important to the user than the reviews of fellow shoppers, the next +click would be on a price range, say "$10-19.99." Now the list might look like this: +

        + +
        + + + + +
        NARROW YOUR RESULTS BY:
        + + + + + + + + +
        Most popular
        +
          +
        • + "Very Good" (1) +
          • +
        +
        +
        +

        + At this point, it is very easy for the user to choose the right cookbook. +

        +

        Why Use Faceting?

        +

        Faceted search results provide an easy-to-scan, browsable display that helps users quickly narrow down each search. +The faceting tags that you store with your IndexTank documents provide a way to add your own taxonomy to directly control +the presentation of search results. In the end, it's about helping the user find the right information. +Faceted search gives a user the power to create an individualized navigation path, drilling down through successive +refinements to reach the right document. This more effectively mirrors the intuitive thought patterns of most users. +Faceted search has become an expected feature, particularly for commerce sites.

        +

        How It Works

        +

        Faceted search is performed in several parts: +

          +
        • + Index: to each document in the index, add tags to specify a value for each facet. For example, + for each book in the index, tag it with the type of material and the price range. +
          • +
        • + Search results: for every search, the IndexTank server returns a count of how many matching documents were + tagged with each value within each facet. For example, if the query was for "books," you might find out that in the + facet "type of material," your index contains 13 science fiction books, 15 romance novels, and 10 cookbooks; and in + the price facet, there are 5 books under $10, 200 books from $10-19.99, and so on. +
          • +
        • + Query: you can include facet values as query criteria. For example, you can write a query that returns only + the romance novels under $10. +
          • +
        • + Web page: use the facets and document counts returned by the server to create a set of facet links on your web page, + like the example shown above. Then construct queries to be activated by each facet link, passing in the appropriate values—say, + querying only for romance novels when the user clicks the "Romance (15)" link. +
          • + +
        + +

        Quick Start: Copy & Paste

        +

        Tweak the following code snippets for your +needs, and you're ready to go

        + +

        Before you start

        + +
          +
        • +
          +
            +
          • ruby
            • +
          • python
            • +
          • php
            • +
          • java
            • +
          + +
          + {% bg 'work' %} +
          +

          For Ruby environments we provide a gem that handles all the REST calls for you in a very Ruby-fashioned way.

          +
            +
          • + Download the Ruby client + if you have not already done so. +
            • +
          • + Know your index's public URL. You'll need + it to instantiate the client. Find the public + URL on the Dashboard. +
            • +
          +
          + + + + {% endbg %} + +
          • +
        + +

        Tag the documents

        +
        +
        +{% box 'table' %} +
          +

        • Syntax rules

          • +
        • Each facet name is defined as a string, and all values for the facet are also defined as strings.
          • +
        • A given document can be tagged with at most one value for each facet. That is, a book can not be of two types.
          • +

        • Tips

          • +
        • + If you like planning ahead, take the time to consider your taxonomy design. + You can also simply add facets and values on the fly. +
          • +
        • + The tags applied to documents need not be the same strings that you will display to users. + Using internal codes instead of display strings will make it easier to modify your UI later if desired. + In our example, we have used display strings to make it easier to follow. +
          • +
        +{% endbox %} +
        +
        +

        + Store the facets and facet values as metadata by adding tags to documents in your index. +

        +

        + The following code shows how to tag a document with several facets at the same time, using + the updateCategories() method from the Java client library: +

        +{% box 'code' %} +
        +categories = { 
+                  'priceRange' => '$10-$19.99',
+                  'bookType' => 'cookbook'
+             }
+
+index.document(docid).update_categories(categories) 
+
        + + + +{% endbox %} + + +

        See Facets in Query Results

        +

        + After you have tagged documents, the IndexTank server will start to show + faceting data in the results it returns for search requests. For example, + if you search for books about France, you might get results like the following: +

        + +{% box 'code' %} +
        +{
+   'bookType': {
+       'cookbook': 2,
+       'travel': 4
+   },
+   'priceRange': {
+       'Less than $10': 1,
+       '$10-$19.99': 5
+   }
+}
+
        +{% endbox %} +
        +
        + +

        + This shows that there are both cookbooks and travel books related to France, and most of them are between $10 and $20. +

        + +

        Use Facets as Query Criteria

        + +

        + You can filter a search by using facet values. This is similar to using document variables, + but uses different syntax. For example, suppose you want to find cookbooks that cost less + than $20.00. The following code shows how to do that search by using .withCategoryFilters + to include facet values in a query: +

        + +{% box 'code' %} +
        +index.search(query,
+             :category_filters => {
+                'priceRange' => ['Less than $10', '$10-$19.99'],
+                'bookType' => ['cookBook']
+             })
+
        + + + +{% endbox %} + +

        Designing a Faceting Taxonomy

        + +

        + Before you start implementing faceting, take some time to decide on which facets and values make sense for your index. + When you consider how to categorize information, all sorts of interesting questions can arise. Depending on the size of + your index and whether you are working in a large enterprise, you might need to hold a few meetings involving key people + such as website designers, product managers, information architects, and others. +

        + +

        + The goal of your design phase is to arrive at a scheme, probably a written list, that defines the facets and their values. + For example, suppose you are dealing with the books database from our earlier example. If you decide that one facet is + "type of book," what are the book types you want to feature? You'll need to answer questions like these: +

          +
        • + Will you base the list on which types are most numerous in your index, on which types people most often purchase, + or on some other criteria? +
          • +
        • + How many book types do you want to tag before grouping the remaining books into an "Other" category? +
          • +
        • + At what cutoff point does a type of book fall into "Other": if there are fewer than 100 books of that type in the + index ... fewer than 10 ... or fewer than 10 purchased in the past quarter? +
          • +
        • + Do you want to keep the book type groups fairly even in size? For example, if you have 100 mystery novels and only + a few books of every other type, do you want to break down the "mystery" group into smaller sub-groups, such as "Noir," "Thriller," and "Sherlock Holmes"? +
          • +
        +

        +

        + From this example, you can see that a taxonomy can be both complex and dynamic—the design might need to change over time to + reflect changes in index contents, business goals, and sales fluctuations. +

        + +

        More Information

        + + +
        +{% endblock %} diff --git a/storefront/templates/documentation/tutorial-geolocation.html b/storefront/templates/documentation/tutorial-geolocation.html new file mode 100644 index 0000000..826078d --- /dev/null +++ b/storefront/templates/documentation/tutorial-geolocation.html @@ -0,0 +1,263 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block extrahead %} + + + +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block right_content %} + {% box 'table' %} + + {% endbox %} +{% endblock %} + +{% block content_head %}

        Geolocation Tutorial

        {% endblock %} + +{% block common_content %} +
        + +

        How to implement geolocation, a way to increase the relevance of search results

        +

        About Geolocation

        +

        +To provide search results customized to the user's location, add geolocation functionality. With geolocation, +a query can include not only the target search terms but also latitude and longitude data pinpointing the +location of the person submitting the search. IndexTank can use this information to provide search results +related to that location.

        +

        Why Use Geolocation?

        +

        +For some types of search results, the location of the person doing the search matters. A search for "coffee shop" +or "plumber" is most likely not an abstract or academic one, but a practical search for an immediate need. If you +know the location from which a query is submitted, you can potentially use that information to provide more useful +search results. Finding a nearby place to get coffee, or a local plumber to fix a broken water pipe, is the result +most likely to make users happy. +

        +

        +You might want to use location data as a ranking signal for every query. Alternatively, you can let the user decide +whether location is important by providing a special UI control, such as a "Find Near Me" button or a Settings checkbox, +and run the location-based query only when the user selects to do so. +

        +

        How It Works

        +

        Geolocation is performed in several parts: +

          +
        • + Index: a location (latitude and longitude) can be associated with each document in the index. The latitude and + longitude are stored in the index as document variables. Location data can be associated with all or only some of the + documents in the index. +
          • +
        • + Scoring function: a built-in distance scoring function + can be applied to any query to calculate the distance between the user's location and the lat/long point stored for a document. +
          • +
        • + Query: in addition to the scoring function, pass the user's location in the query by including lat/long values as query variables. + With this data, the user's location can be compared to the document location. +
          • +
        • + User interface: You might already know the user's location: perhaps your users are authenticated and you have stored profiles for + them, you are using IP location software, or your users are accessing your application through GPS-enabled devices. If not, you need to + provide a way for the user to specify a location (typically using a postal code or street address). By adding a little code to convert + locations into lat/long coordinates, your application is ready to put location-based result ranking into effect. +
          • +
        +

        +

        Quick Start: Copy & Paste

        +

        Tweak the following code snippets for your +needs, and you're ready to go

        + +

        Before you start

        + +
          +
        • +
          +
            +
          • ruby
            • +
          • python
            • +
          • php
            • +
          • java
            • +
          + +
          + {% bg 'work' %} +
          +

          For Ruby environments we provide a gem that handles all the REST calls for you in a very Ruby-fashioned way.

          +
            +
          • + Download the Ruby client + if you have not already done so. +
            • +
          • + Know your index's public URL. You'll need + it to instantiate the client. Find the public + URL on the Dashboard. +
            • +
          +
          + + + + {% endbg %} + +
          • +
        + +

        Add Lat/Long Data to the Documents

        +
        +

        + For those documents in your index that can be associated with a particular geographic location, store that location as a pair of document + variables: one to store the latitude, and one to store the longitude. +

        +

        + The following code shows how to add latitude and longitude data to documents in the index, using the addDocument() method from the Java + client library. The latitude and longitude values are expressed in degrees, as floating point numbers. The location used for this example + is Buenos Aires, Argentina: +

        +{% box 'code' %} +
        +lat = -34.70549341022545
+lon = -58.359375
+
+variables = { 
+              0 => lat,
+              1 => lon
+            }
+
+index.document(docid).add(fields, :variables => variables)
+
        + + + +{% endbox %} + +

        Define Distance Scoring Function

        +
        +

        + IndexTank provides a built-in function that you can use to calculate the distance between two points. Two versions are provided, miles() + and km(), for calculating distance in either miles or kilometers. For details on the function syntax, see Scoring Function Formulas. +

        +

        + The following example shows how to use a distance function in the definition of custom scoring function number 5. The inclusion of the + negative sign will cause this function to rank search results from shortest distance to longest. +

        +{% box 'code' %} +
        +index.functions(5, "-miles(d[0], d[1], q[0], q[1])").add
+
        + + + +{% endbox %} + +

        Specify Location as a Query Variable

        +
        +

        + To pass the geolocation of a particular user in a search query, use query variables. The following code shows how to include both the + distance scoring function and the lat/long data as parameters to a query:

        +

        +{% box 'code' %} +
        +index.search(query, 
+             :function => 5, 
+             :var0 => latitude, 
+             :var1 => longitud)
        + + + +{% endbox %} + + +

        More Information

        + + +
        +{% endblock %} diff --git a/storefront/templates/documentation/tutorial-python.html b/storefront/templates/documentation/tutorial-python.html new file mode 100644 index 0000000..d9eb0e3 --- /dev/null +++ b/storefront/templates/documentation/tutorial-python.html @@ -0,0 +1,607 @@ +{% extends "documentation/tutorial-base.html" %} + +{% load custom_tags %} + +{% block language %}Python{% endblock %} + +{% block language_prerequisite %} + Python (version 2.6 or greater) installed on your machine + + (IndexTank can be used with other languages, but in this Tutorial we use Python. If you don't already have Python, you can get it at python.org) + . +{% endblock %} + +{% block system_requirements %} +
      • + The Tutorial has been tested on Linux, Mac OS X, and Windows. +
        • + +{% endblock %} + +{% block download_client %} +
      • +

        Click Python client library. This takes you to
        http://indextank.com/documentation/python-client.

        +
        • +
      • +

        In the Download area, in Stable Version, click one of the links: zip + for Windows or Mac, tgz + for Unix/Linux.

        +
        • +
      • +

        Extract the compressed library to a convenient directory on your computer.

        +
        • +
      • +

        Open a console window and change ( + + cd + ) to the directory where the client library is installed.

        +
        • + +

        + Result: +
        + If you run the command to view the contents of the directory ( + + ls + + on Unix or Mac OS X; + + dir + + on Windows), you should see the file + + indextank_client.py + . +

        +

        + +

        +{% endblock %} + +{% block instantiate_client %} +
          +
        1. +

          While still in the directory where the client library is installed, run the Python interpreter.

          +

          + If you don't know how to use the Python interpreter, see
          http://docs.python.org/tutorial/interpreter.html. +

          +
          +{% box 'code' %} +
          +C:\> python
+
          +{% endbox %} +
          +

          + The Python prompt appears: +

          +
          +{% box 'code' %} +
          +>>>
+
          +{% endbox %} +
          +
          2. +
        2. +

          Import the IndexTank client library to the Python interpreter by typing the following command at the Python prompt.

          +
          +{% box 'code' %} +
          +>>> import indextank.client as itc
+
          +{% endbox %} +
          +

          + + itc + + is a name you give to the imported client so you can refer to it later. +

          +
        3. +

          Instantiate the client by calling + + ApiClient() + . +

          +
          +{% box 'code' %} +
          +>>> api_client = itc.ApiClient('YOUR_API_URL')
+
          +{% endbox %} +
          +

          + For + + YOUR_API_URL + , substitute the URL from Private URL + in your Dashboard (refer to the screen shot at the beginning of the Tutorial if you forgot where to find this). +

          +
          4. +
        +{% endblock %} + +{% block get_handle %} +
        +{% box 'code' %} +
        +>>> test_index = api_client.get_index('test_index')
+
        +{% endbox %} +
        +

        + Here we call the + + get_index() + + method in the client library and pass it the name you assigned when you created the index. +

        +{% endblock %} + +{% block add_documents1 %} +
        +{% box 'code' %} +
        +>>> test_index.add_document('post1', {'text':'I love Bioshock'})
+>>> test_index.add_document('post2', {'text':'Need cheats for Bioshock'})
+>>> test_index.add_document('post3', {'text':'I love Tetris'})
+
        +{% endbox %} +
        +

        + Here we call the + + add_document() + + method in the client library three times to index three posts in the video gamer forum. +

        +{% endblock %} + +{% block search1 %} +
        +{% box 'code' %} +
        +>>> test_index.search('Bioshock')
+
        +{% endbox %} +
        +{% endblock %} + +{% block search1_output %} +
        +{% box 'code' %} +
        +{'matches': 2,
+ 'facets': {}, 
+ 'search_time': '0.070',
+ 'results': [{'docid': 'post2'},{'docid': 'post1'}]}
+
        +{% endbox %} +
        +{% endblock %} + +{% block search2 %} +
        +{% box 'code' %} +
        +>>> test_index.search('love Bioshock')
+
        +{% endbox %} +
        +{% endblock %} + +{% block search2_output %} +
        +{% box 'code' %} +
        +{'matches': 1,
+ 'facets': {}, 
+ 'search_time': '0.005',
+ 'results': [{'docid': 'post1'}]}
+
        +{% endbox %} +
        +{% endblock %} + +{% block search3 %} +
        +{% box 'code' %} +
        +>>> test_index.search('Bioshock OR Tetris')
+
        +{% endbox %} +
        +{% endblock %} + +{% block search3_output %} +
        +{% box 'code' %} +
        +{'matches': 3,
+ 'facets': {}, 
+ 'search_time': '0.007',
+ 'results': [{'docid': 'post3'},{'docid': 'post2'},{'docid': 'post1'}]}
+
        +{% endbox %} +
        +{% endblock %} + +{% block fetch_fields %} +
      • +

        To ask IndexTank to return more than just the doc ID, add the argument + + fetch_fields.

        +
        +{% box 'code' %} +
        +>>> test_index.search('love', fetch_fields=['text'])['results']
+
        +{% endbox %} +
        +{% endblock %} + +{% block fetch_fields_output %} +
        +{% box 'code' %} +
        +[{'text': 'I love Tetris', 'docid': 'post3'},
+ {'text': 'I love Bioshock', 'docid': 'post1'}]
+
        +{% endbox %} +
        +{% endblock %} + +{% block snippet_fields %} +
      • +

        To show portions of the result text with the search term highlighted, use + + snippet_fields.

        +
        +{% box 'code' %} +
        +>>> test_index.search('love', snippet_fields=['text'])['results']
+
        +{% endbox %} +
        +{% endblock %} + +{% block snippet_fields_output %} +
        +{% box 'code' %} +
        +[{'snippet_text': 'I <b>love</b> Tetris', 'docid': 'post3'},
+ {'snippet_text': 'I <b>love</b> Bioshock', 'docid': 'post1'}]
+
        +{% endbox %} +
        +{% endblock %} + +{% block add_fields %} +
        +{% box 'code' %} +
        +>>> test_index.add_document('post1', {'text':'I love Bioshock', 'game':'Bioshock'})
+>>> test_index.add_document('post2', {'text':'Need cheats for Bioshock', 'game':'Bioshock'})
+>>> test_index.add_document('post3', {'text':'I love Tetris', 'game':'Tetris'})
+
        +{% endbox %} +
        +

        + Here we call + + add_document() + + with the same document IDs as before, so IndexTank will overwrite the existing entries in your test index. +

        +{% endblock %} + +{% block search_field %} +

        Now you can search within a particular field. Let's use + + fetch_fields + + again to get some user-friendly output.

        +
        + +{% box 'code' %} +
        +>>> test_index.search('game:Tetris', fetch_fields=['text'])['results']
+
        +{% endbox %} +
        +{% endblock %} + +{% block search_field_output %} +
        + {% box 'code' %} +
        +[{'text': 'I love Tetris', 'docid': 'post3'}]
+
        +{% endbox %} +
        +{% endblock %} + +{% block timestamps %} +
        + {% box 'code' %} +
        +>>> test_index.add_document('newest',{'text': 'New release: Fable III is out','timestamp':1286673129})
+>>> test_index.add_document('not_so_new',{'text': 'New release: GTA III just arrived!','timestamp':1003626729})
+>>> test_index.add_document('oldest',{'text': 'New release: This new game Tetris is awesome!','timestamp':455332329})
+
        +{% endbox %} +
        +{% endblock %} + +{% block scoring1 %} +
        + {% box 'code' %} +
        +>>> test_index.search('New release')
+
        +{% endbox %} +
        +{% endblock %} + +{% block scoring1_output %} +
        + {% box 'code' %} +
        +{'matches': 3,
+ 'facets': {}, 
+ 'search_time': '0.002',
+ 'results': [{'docid': 'newest'},{'docid': 'not_so_new'},{'docid': 'oldest'}]}
+
        +{% endbox %} +
        +{% endblock %} + +{% block scoring2_redefine %} +
        + {% box 'code' %} +
        +>>> test_index.add_function(0,'age')
+
        +{% endbox %} +
        +{% endblock %} + +{% block scoring2_search %} +
        + {% box 'code' %} +
        +>>> test_index.search('New release')
+
        +{% endbox %} +
        +{% endblock %} + +{% block scoring2_output %} +
        + {% box 'code' %} +
        +{'matches': 3,
+ 'facets': {}, 
+ 'search_time': '0.005',
+ 'results': [{'docid': 'oldest'},{'docid': 'not_so_new'},{'docid': 'newest'}]}
+
        +{% endbox %} +
        +{% endblock %} + +{% block add_documents2 %} +
        + {% box 'code' %} +
        +>>> test_index.add_document('post4', {'text': 'When is Duke Nukem Forever coming out? I need my Duke.'})
+>>> test_index.add_document('post5', {'text': 'Duke Nukem is my favorite game. Duke Nukem rules. Duke Nukem is awesome. Here are my favorite Duke Nukem links.'})
+>>> test_index.add_document('post6', {'text': 'People who love Duke Nukem also love our great product!'})
+
        +{% endbox %} +
        +{% endblock %} + +{% block define_function %} +
        + {% box 'code' %} +
        +>>> test_index.add_function(1,'relevance')
+
        +{% endbox %} +
        +{% endblock %} + +{% block search4 %} +
        + {% box 'code' %} +
        +>>> test_index.search('duke', scoring_function=1, fetch_fields=['text'])['results']
+
        +{% endbox %} +
        +{% endblock %} + +{% block search4_output %} +
        + {% box 'code' %} +
        +[{'docid': 'post5',
+'text': 'Duke Nukem is my favorite game. Duke Nukem rules. Duke Nukem is awesome. Here are my favorite Duke Nukem links.'},
+{'docid': 'post4', 'text': 'When is Duke Nukem Forever coming out? I need my Duke.'},
+{'docid': 'post6', 'text': 'People who love Duke Nukem also love our great product!'}]
+
        +{% endbox %} +
        +{% endblock %} + +{% block assign_vars %} +
        + {% box 'code' %} +
        +>>> test_index.add_document('post4', {'text': 'When is Duke Nukem Forever coming out? I need my Duke.'}, variables={0:10, 1:1.0})
+>>> test_index.add_document('post5', {'text': 'Duke Nukem is my favorite game. Duke Nukem rules. Duke Nukem is awesome. Here are my favorite Duke Nukem links.'}, variables={0:1000, 1:0.9})
+>>> test_index.add_document('post6', {'text': 'People who love Duke Nukem also love our great product!'}, variables={0:1, 1:0.05})
+
        +{% endbox %} +
        +{% endblock %} + +{% block function_vars %} +
        + {% box 'code' %} +
        +>>> test_index.add_function(2, 'relevance * log(doc.var[0]) * doc.var[1]')
+
        +{% endbox %} +
        +{% endblock %} + +{% block scoring3 %} +
        + {% box 'code' %} +
        +>>> test_index.search('duke', scoring_function=2, fetch_fields=['text'])['results']
+
        +{% endbox %} +
        +{% endblock %} + +{% block scoring3_output %} +
        + {% box 'code' %} +
        +[{'docid': 'post5', 'text': 'Duke Nukem is my favorite game. Duke Nukem rules. Duke Nukem is awesome. Here are my favorite Duke Nukem links.'},
+ {'docid': 'post4', 'text': 'When is Duke Nukem Forever coming out? I need my Duke.'},
+ {'docid': 'post6', 'text': 'People who love Duke Nukem also love our great product!'}]
+
        +{% endbox %} +
        +{% endblock %} + +{% block update_var %} +
        + {% box 'code' %} +
        +>>> test_index.update_variables('post4',{0:1000000})
+
        +{% endbox %} +
        +{% endblock %} + +{% block scoring4 %} +
        + {% box 'code' %} +
        +>>> test_index.search('duke', scoring_function=2, fetch_fields=['text'])['results']
+
        +{% endbox %} +
        +{% endblock %} + +{% block scoring4_output %} +
        + {% box 'code' %} +
        +[{'docid': 'post4', 'text': 'When is Duke Nukem Forever coming out? I need my Duke.'},
+ {'docid': 'post5', 'text': 'Duke Nukem is my favorite game. Duke Nukem rules. Duke Nukem is awesome. Here are my favorite Duke Nukem links.'},
+ {'docid': 'post6', 'text': 'People who love Duke Nukem also love our great prod]
+
        +{% endbox %} +
        +{% endblock %} + +{% block delete_document %} +
        + {% box 'code' %} +
        +>>> test_index.delete_document('post6')
+
        +{% endbox %} +
        +{% endblock %} + +{% block search5 %} +
        + {% box 'code' %} +
        +>>> test_index.search('duke', scoring_function=2, fetch_fields=['text'])['results']
+
        +{% endbox %} +
        +{% endblock %} + +{% block search5_output %} +
        + {% box 'code' %} +
        +[{'docid': 'post4', 'text': 'When is Duke Nukem Forever coming out? I need my Duke.'},
+ {'docid': 'post5', 'text': 'Duke Nukem is my favorite game. Duke Nukem rules. Duke Nukem is awesome. Here are my favorite Duke Nukem links.'}]
+
        +{% endbox %} +
        +{% endblock %} + +{% block add_doc_var %} +
        +{% box 'code' %} +
        +>>> test_index.add_document('post1', {'text':'I love Bioshock'}, variables={0:115})
+>>> test_index.add_document('post2', {'text':'Need cheats for Bioshock'}, variables={0:2600})
+>>> test_index.add_document('post3', {'text':'I love Tetris'}, variables={0:19500})
+
        +{% endbox %} +
        +{% endblock %} + +{% block scoring5 %} +
        +{% box 'code' %} +
        +>>> test_index.add_function(1, 'relevance / max(1, abs(query.var[0] - doc.var[0]))')
+
        +{% endbox %} +
        +{% endblock %} + +{% block search6 %} +
        +{% box 'code' %} +
        +>>> test_index.search('bioshock', scoring_function=1, fetch_fields=['text'], variables={0: 25})['results']
+
        +{% endbox %} +
        +{% endblock %} + +{% block search6_output %} +
        +{% box 'code' %} +
        +[{'docid': 'post1', 'text': 'I love Bioshock.'},
+ {'docid': 'post2', 'text': 'Need cheats for Bioshock.'}]
+
        +{% endbox %} +
        +{% endblock %} + +{% block search7 %} +
        +{% box 'code' %} +
        +>>> test_index.search('love', scoring_function=1, fetch_fields=['text'], variables={0: 15000})['results']
+
        +{% endbox %} +
        +{% endblock %} + +{% block search7_output %} +
        +{% box 'code' %} +
        +[{'docid': 'post3', 'text': 'I love Tetris.'},
+ {'docid': 'post1', 'text': 'I love Bioshock.'}]
+
        +{% endbox %} +
        +{% endblock %} diff --git a/storefront/templates/documentation/tutorial-scoring-functions.html b/storefront/templates/documentation/tutorial-scoring-functions.html new file mode 100644 index 0000000..da4f04d --- /dev/null +++ b/storefront/templates/documentation/tutorial-scoring-functions.html @@ -0,0 +1,147 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block extrahead %} + +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block right_content %} + {% box 'table' %} + + {% endbox %} + +{% endblock %} + +{% block content_head %}

        Sorting Search Results

        {% endblock %} + +{% block common_content %} +
        + +

        How to use scoring functions to customize the order of search results

        +

        About Sorting Search Results

        +

        Are the most useful documents listed first? This one simple question sums up the field of search result ranking—a field to which considerable engineering brain-power has been devoted in recent years. Starting from the relatively simple matter of matching the search terms entered by the user, a wide variety of criteria can be brought to bear in order to calculate the optimal order in which to display the documents that contain the search terms.

        +

        Why Customize Sorting?

        +

        The IndexTank server applies its own sorting techniques and returns results in order by age (most recent first). In many cases, this default behavior will be sufficient.

        +

        When the requirements of your application, user needs, or business logic demand it, you can customize IndexTank's sorting technique to take control of the order in which search results are displayed. By fine-tuning the search output, you can improve the experience for users and also direct them towards the results you would prefer them to see.

        +

        How It Works

        +

        In a word: mathematics. With every search query, you can pass a scoring function, a mathematical formula that encapsulates your ranking preferences. This function overrides IndexTank's default scoring formula. The formula can be as simple or as complex as you need it to be.

        +

        The implementation involves two parts:

        +
          +
        • + Scoring Function:a custom sorting algorithm is expressed in a formula that is constructed using IndexTank's built-in functions and values. If you have defined custom document variables, you can use them as well. For each document that matches the query's search terms, the scoring function is evaluated, substituting any needed values from that particular document. For details about scoring function syntax, see Scoring Function Formulas. +
          • +
        • + Query:the ID number of a scoring function can be passed as a parameter to any query. You can pass only one scoring function with each query. If needed, you can redefine scoring functions in real time between queries to reflect changing circumstances. +
          • +
        +

        Anatomy of a Scoring Function

        +

        At any given time, your application can have up to six scoring functions +defined. The functions are named with the integers from 0 to 5. +Function 0 is the default and will be applied if no other is specified; +it starts out with an initial definition of-age, which sorts query results +from most recently indexed to least recently indexed (newest to oldest).

        + +
        + Syntax Notes: + Variable and function names are case + sensitive. + The following are all floats: expressions + (except conditions), variable values, +
        + +

        A scoring function is an expression built up from some or all of the +following components:

        +
          +
        • + relevance:a numeric score indicating how closely the document matches the search terms. Calculated by the IndexTank server based on how often each search term appears in the text, and whether the text contains all of the terms. +
          • +
        • + age:a number that tells how fresh the document is. Calculated based on the document's timestamp field, which contains either IndexTank's automatic timestamp (indicating the time when the document was indexed) or a custom timestamp you have applied yourself. +
          • +
        • + Document variables:custom values that you have associated with documents in the index. For example, you might store a count of how many users commented on a document. +
          • +
        • + Query variables:values that are passed in with a query. For example, the user's location. +
          • +
        • + Functions:built-in mathematical and flow control functions provided by IndexTank, including max(), min(), miles(), km(), if(), and more. +
          • +
        • + Operators:+ - * / +
          • +
        + +

        Quick Start: Copy & Paste

        + +

        Tweak the following code snippets for your needs, and you're ready to go

        + +
        + Before You Start: +
          +
        • + Download and instantiate the Java client, if you + have not already done so. +
          • +
        • + Know your index's public URL. You'll need it to + instantiate the client. Find the public URL on + the Dashboard. +
          • +
        + +

        Define the Scoring Function

        +

        The possibilities are limitless, but here are a few ideas to get you started.

        +{% box 'code' %} +
        +// Make function 0 sort most recent first
+index.addFunction(0, "-age");
+
+// Make function 1 sort by textual relevance
+index.addFunction(1, "relevance");
+
+// Make function 2 sort by distance between document and
+// a geographic location passed in the query, sorting
+// from nearest to farthest.
+// doc.var[0] and doc.var[1] have the lat/long of the document.
+// query.var[0] and query.var[1] have the latitude and
+// longitude of an outside location (probably the user's locale).
+index.addFunction(2, "-miles(query.var[0], query.var[1], doc.var[0], doc.var[1])");
+
+// Make function 3 sort by a combination of textual
+// relevance and two document variables, which are
+// given different weights by the use of the log() function
+index.addFunction(3, "relevance * log(doc.var[0]) * doc.var[1]");
+
+// Make function 4 sort by the greater of document variable 0 or 1
+index.addFunction(4, "max(doc.var[0], doc.var[1])");
+
        +{% endbox %} + +

        Pass the Scoring Function in a Query

        +

        To indicate which scoring function to use, pass its ID number as a parameter to the query. This query will use scoring function 2:

        + +{% box 'code' %} +
        +index.search(Query.forString(query).withScoringFunction(2));
+
        +{% endbox %} + + +

        More Information

        + + +
        +{% endblock %} diff --git a/storefront/templates/documentation/tutorial-snippets.html b/storefront/templates/documentation/tutorial-snippets.html new file mode 100644 index 0000000..6e81d51 --- /dev/null +++ b/storefront/templates/documentation/tutorial-snippets.html @@ -0,0 +1,205 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block extrahead %} + + + +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block right_content %} + {% box 'table' %} + + {% endbox %} + +{% endblock %} + +{% block content_head %}

        Snippeting Tutorial

        {% endblock %} + +{% block common_content %} +
        + +

        How to implement snippets, an enhancement to the display of search results

        +

        About Snippeting

        + +

        + To make it easier for users to read search results at a glance, add snippet functionality. When snippets are enabled, a brief portion of the document text is shown along with each search result. +

        + +

        Why Use Snippets?

        +

        Snippets can significantly increase the speed with which users can go from search to action, often providing enough information that the user does not even have to click on the search result link. For example, a list of results for local veterinarians might come with snippets that show each vet's phone number and address, making it easier and faster for the pet owner to quickly place a call to the nearest clinic.

        +

        Snippets can also help the user understand why a particular search is not returning the expected results. With this information, the user can refine the query.

        +

        Snippets are, in fact, so useful and so common that users probably take their presence for granted, and it is difficult to think of a reason for not using them.

        +

        How It Works

        +

        Snippets are selected by the IndexTank server based on the search terms and snippet fields in the query. +

          +
        • + Query: specify one or more document fields from which you want to pull snippets. It's fairly typical to use the text field. If you have defined custom document fields, you can use them as well. +
          • +
        • + Server: The IndexTank server selects a portion of text to return as a snippet from each document field specified + in the "snippet fields" clause of the query. The server selects the portion that contains as many as possible of the + search terms from the query. For example, if the query is looking for the terms "vet" and "cat," a snippet that contians + both those terms will be preferred over one that contains multiple instances of only one of the terms. The server returns + up to 20 words of snippet text with the search terms highlighted. +
          • +
        + +

        Quick Start: Copy & Paste

        +

        Tweak the following code for your needs, and you're ready to go

        + +

        Before you start

        + +
          +
        • +
          +
            +
          • ruby
            • +
          • python
            • +
          • php
            • +
          • java
            • +
          + +
          + {% bg 'work' %} +
          +

          For Ruby environments we provide a gem that handles all the REST calls for you in a very Ruby-fashioned way.

          +
            +
          • + Download the Ruby client + if you have not already done so. +
            • +
          • + Know your index's public URL. You'll need + it to instantiate the client. Find the public + URL on the Dashboard. +
            • +
          +
          + + + + {% endbg %} + +
          • +
        + +

        Set Snippet Fields in Query

        +
        +

        + The following code shows how to request snippets. +

        +{% box 'code' %} +
        +results = index.search(query, 
+                       :fetch => 'title,timestamp', 
+                       :snippet => 'text')
+
+print results['matches'], " results\n"
+results['results'].each {|doc|
+    docid = doc['docid']
+    title = doc['title']
+    timestamp = doc['timestamp']
+    text = doc['snippet_text']
+    print "docid: #{docid}, title: #{title}, timestamp: #{timestamp}, text: #{text}" 
+}
+
        + + + +{% endbox %} + +

        More Information

        +
          +
        • Searching in the IndexTank API reference
          • +
        + +
        +{% endblock %} diff --git a/storefront/templates/documentation/wordpress-plugin.html b/storefront/templates/documentation/wordpress-plugin.html new file mode 100644 index 0000000..12a28a0 --- /dev/null +++ b/storefront/templates/documentation/wordpress-plugin.html @@ -0,0 +1,119 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block extrahead %} + +{% endblock %} + +{% block container_class %}{{ block.super }} documentation{% endblock %} + +{% block right_content %} + + {% box 'chatsimple' %} +

        DOWNLOAD

        + +
        LIVE CHAT NOW
        + {% endbox %} + + {% box 'table' %} + + {% endbox %} +{% endblock %} + + +{% block title %}WordPress Plug-in{% endblock %}endblock %} + +{% block common_content %} +
        + +

        You can use IndexTank to improve the search experience on your Wordpres blog. For that, we have this WordPress plug-in. + Just follow the instructions below and you will have all the power of IndexTank search in your blog.

        + +

        About this plug-in

        +

        + IndexTank's WordPress plug-in will replace your current blog's search with IndexTank's. It will enable you to use your IndexTank index as the search index for your blog. +

        +

        + This plug-in will provide your blog's search with different results sorting functions (time, relevance, number of comments) and a more powerful query language. You will also get InstantSearch, Autocomplete and more accurate search results for your blog.

        + +

        Installing the plug-in

        +

        + The easiest way is to use Wordpress' Plugin Manager. Go to your Wordpress dashboard, and find the Plugins -> Add new menu. Search for 'indextank'. On the results page, find 'IndexTank Search' and install it. +

        +

        + If that doesn't work for your blog, you can try downloading the plugin from the link on the right, and upload it to your wordpress installation. You may need to contact your Wordpress administrator to do this. +

        + +

        Plug-in configuration

        + +
        + NOTE: + Keep in mind that the index you use for the plug-in should be used ONLY for searching your blog's posts. +
        + +

        + By enabling this plug-in, you will change the behaviour of your blog's search box. Typing a query in that box will now perform a search in an IndexTank's index. + This is why you first need to create an index for this purpose. Go to your blog's dashboard, and browse the Tools -> Indextank Searching menu. +

        +

        + On the 'Settings' section, you will see a 'Get one' button. That button will get you and Indextank account, and set up an index. After that, you'll redirected back to the 'Settings' page, but you'll see credentials for your index (API URL and Index name). +

        + +

        + +

        + +

        + Now that you have an account, other sections on the 'Settings' page will be enabled. +

          +
        • Indexing your posts
          • +
        • Reset your index
          • +
        • Look And Feel
          • +
        +
        +

        + +

        + In the "Indexing your posts" section, at the bottom of the configuration page, you will find an "Index all posts" button. Click it. You only need to do this to index your documents for the first time, or to reindex them in case something happened. +

        +

        + There's a checkbox to apply 'the_content' filters to posts. You need to check it only if you have plugins that modify the way content is rendered (like MarkDown). +

        +

        + +

        +

        + Wait for the confirmation message indicating that the blog posts were successfully indexed. The page won't reload. If you get an error message, check the API URL and the Index Name you entered above. Or you can write to support@indextank.com, including your API URL, Index Name and error message. +

        + +

        + You can now test the new search in your blog. Go to your blog's home page and type in the search box. You will find that the autocomplete feature is already enabled. Your blog's search is running with IndexTank! +

        + + +

        Configuring Look and Feel

        + +
        + NOTE: + Customizing your template won't break your blog's look in any way. +
        + +

        +Indextank is compatible with most Wordpress themes out-of-the-box. In order to play nice with your current theme, you need to click the 'Magic' button on the 'Look and Feel' section of the settings page. +

        +

        +It will generate a custom configuration for you, making it possible to render results in a way that completely blends with your theme. +

        + + +
        +{% endblock %} diff --git a/storefront/templates/enter_payment.html b/storefront/templates/enter_payment.html new file mode 100644 index 0000000..a510a58 --- /dev/null +++ b/storefront/templates/enter_payment.html @@ -0,0 +1,53 @@ +{% extends "common-base.html" %} + +{% load custom_tags %} +{% load humanize %} + +{% block title %}Enter Payment Information{% endblock %} + +{% block tryit %}{% endblock %} + + +{% block right_content %} + {% box 'chat' %} +

        HAVE A QUESTION?

        +
        LIVE CHAT NOW
        +

        Our team of experienced and friendly devs will respond almost as quickly as our search API.

        + {% endbox %} +{% endblock %} + +{% block common_content %} + +{% include "includes/chosen_plan.html" %} + +

        Get going in minutes

        + + {% if message %} +
        + {{ message }} +
        + {% endif %} + +
        +
        + {% with payment_form.first_name as field %}{% include 'includes/field.html' %}{% endwith %} + {% with payment_form.last_name as field %}{% include 'includes/field.html' %}{% endwith %} +
        + {% with payment_form.country as field %}{% include 'includes/field.html' %}{% endwith %} +
        + {% with payment_form.state as field %}{% include 'includes/field.html' %}{% endwith %} + {% with payment_form.city as field %}{% include 'includes/field.html' %}{% endwith %} + {% with payment_form.address as field %}{% include 'includes/field.html' %}{% endwith %} + {% with payment_form.zip_code as field %}{% include 'includes/field.html' %}{% endwith %} + + {% with payment_form.credit_card_number as field %}{% include 'includes/field.html' %}{% endwith %} + {% with payment_form.exp_month as field %}{% include 'includes/field.html' %}{% endwith %} +
        + +
        +
        + +{% endblock %} + diff --git a/storefront/templates/forgot_pass.html b/storefront/templates/forgot_pass.html new file mode 100644 index 0000000..9fbfe31 --- /dev/null +++ b/storefront/templates/forgot_pass.html @@ -0,0 +1,34 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block title %}Password Reset{% endblock %}endblock %} + +{% block common_content %} + {% box 'form' %} +

        Enter your Email address

        +

        We'll send you a new password to your inbox

        +
        +
        + {% for f in forgot_form %} +
        + +
        +
        {% for e in f.errors %}{{e}} {% endfor %}
        +
        + {% endfor %} + {% if message %} +
        {{ message }}
        + {% else %} + {% endif %} +
        + +
        +
        + {% endbox %} + +{% endblock %} + diff --git a/storefront/templates/get_started.html b/storefront/templates/get_started.html new file mode 100644 index 0000000..448ad00 --- /dev/null +++ b/storefront/templates/get_started.html @@ -0,0 +1,366 @@ +{% extends "common-base.html" %} +{% load custom_tags %} +{% load humanize %} +{% load mixpanel %} + +{% block extrahead %} + {{ block.super }} + + +{% endblock %} + + +{% block title %}Get Started{% endblock %} + +{% block tryit %}{% endblock %} + + +{% block right_content %} + {% box 'blue_box01' %} +

        + Free for first 100K docs.
        + $49/month for 500k docs.
        + $175/month for 2M docs.
        + Upgrade anytime.
        +

        +
        CHOOSE PLAN
        + {#

        Email us if you have more than 500K docs for special pricing.

        #} + {% endbox %} + {% box 'chat' %} +

        HAVE A QUESTION?

        +
        LIVE CHAT NOW
        +

        Our team of experienced and friendly devs will respond almost as quickly as our search API.

        + {% endbox %} +{% endblock %} + +{% block common_content %} + {% if package.base_price %} + {% include "includes/chosen_plan.html" %} + {% else %} +

        Try it Free in under 2 minutes

        + {% endif %} +
          +
        • +

          1. Get your API urls

          + {% if not request.user.is_authenticated %} +
          +

          Enter your email address to get API Urls:

          +
          +
          +
          + +
          + +
          +
          +
          +
          + {% endif %} +
          +

          Welcome {{ request.user.username }}, your account has been successfully created. {#Please confirm your email address within the next 24hs, otherwise your account will be deleted.#}

          + {% if package.base_price > 0 %} + {% if not user.is_authenticated or request.user.get_profile.account.status == "CREATING" %} +
          + Before you start using your account, you'll need to
          enter your payment information +
          + {% endif %} + {% endif %} +

          Here are the urls you need to access your API:

          + {% bg 'commands' %} +
          +

          Private API url:  {% with request.user.get_profile.account.get_private_apiurl as text %}{% include 'includes/clippy.html' %}{% endwith %}

          + {{ request.user.get_profile.account.get_private_apiurl }} +
          + {% endbg %} + {% bg 'commands' %} +
          +

          Public API url:  {% with request.user.get_profile.account.get_public_apiurl as text %}{% include 'includes/clippy.html' %}{% endwith %}

          + {{ request.user.get_profile.account.get_public_apiurl }} +
          + {% endbg %} +
          +
          • +
        • +

          2. Install Library

          +
          +
            +
          • ruby
            • +
          • python
            • +
          • php
            • +
          • java
            • +
          • other
            • +
          + +
          + {% bg 'work' %} +
          +

          For Ruby environments we provide a gem that handles all the REST calls for you in a very Ruby-fashioned way.

          +

           

          +

          Install our client: gem install indextank {% with "gem install indextank" as text %}{% with "1" as small %}{% with "#999999" as bgcolor %}{% include 'includes/clippy.html' %}{% endwith %}{% endwith %}{% endwith %} +

          +

          Look at the client's source code

          +

          Read the client documentation

          +
          + + + + + {% endbg %} +
          • +
        • +

          3. Test your account

          +

          First step is to instantiate the client

          + +
          +{% box 'commands' %} +
          +require 'rubygems'
+require 'indextank'
+
+api_url = "{% if request.user.is_authenticated %}{{ request.user.get_profile.account.get_private_apiurl }}{% else %}<YOUR API URL HERE>{% endif %}"
+api = IndexTank::Client.new api_url
+
          +{% endbox %} +
          + + + + + + + +
          +

          Next, you need to get an index to start playing, you can create directly with the client.

          + +
          +{% box 'commands' %} +
          +index = api.indexes "MyFirstIndex"
+index.add
+
+while not index.running?
+    sleep 0.5
+end
+
          +{% endbox %} +
          + + + + + + + +
          +

          Now that you have created an index, and it has started, you can index documents and do queries.

          + +
          +{% box 'commands' %} +
          +docid = "MyDoc"
+text = "some sample text"
+index.document(docid).add({ :text => text })
+
+results = index.search "some text"
+
+print results['matches'], " results\n"
+results['results'].each {|doc|
+    docid = doc['docid']
+    print "docid: #{docid}" 
+}
+
          +{% endbox %} +
          + + + + + + + +
          + +
          +

          You have successfully created an index and performed queries. See what else + you can do with the Ruby client, or + try searching over your own test index. +

          + + + + + + + +

          Don't forget to checkout your account's dashboard

          + + +
          • +
        +{% endblock %} + diff --git a/storefront/templates/heroku-dashboard.html b/storefront/templates/heroku-dashboard.html new file mode 100644 index 0000000..0d46e19 --- /dev/null +++ b/storefront/templates/heroku-dashboard.html @@ -0,0 +1,162 @@ +{% extends "common-base.html" %} +{% load custom_tags %} +{% load humanize %} + +{% block extrahead %} + {{ block.super }} + +{% endblock %} + +{% block container_class %}{{ block.super }} dashboard wide{% endblock %} + +{% block title %}My Dashboard{% endblock %} + + +{% block common_content %} +
        +
        + +
        +
        +

        IndexTank Add-on

        +

        + IndexTank is a cloud-based search solution. With it you can have your application's textual content searchable in a highly efficient, scalable and robust platform, without the hassle and cost of building, configuring, maintaining and hosting your own search implementation. + This Heroku Add-on allows you to use IndexTank in your application, adding real-time search with very little effort. +

        +
        + +
        +

        Your IndexTank add-on configuration

        +
        +{% box 'code' %} +
        +api_url = '{{ account.get_private_apiurl }}'
+{% if indexes.0 %}index_name = '{{ indexes.0.name }}'{% endif %}
+
        +{% endbox %} +
        +

        + The api_url is your private key that you can use to talk to the IndexTank service from within your application when it is running locally (not on Heroku).

        + The index_name is the name of your index. You will use this name to operate with your index from within your app code.

        + You'll need to install the IndexTank gem to develop locally. +

        +
        +
        +
        +

        + Installing the IndexTank Gem +

        +
        +{% box 'code' %} +
        +$ gem install indextank
+
        +{% endbox %} +
        +

        + Now you can use the IndexTank client to add documents to your index and perform searches. You can grab the following sample code and use it in your application: +

        +
        +{% box 'code' %} +
        +require 'rubygems'
+require 'indextank'
+
+client = IndexTank::Client.new(ENV['INDEXTANK_API_URL'] || '{{ account.get_private_apiurl }}')
+index = client.indexes('{% if indexes.0 %}{{ indexes.0.name }}{% endif %}')
+
+begin
+    index.document("1").add({ :text => "some text here" })
+    index.document("2").add({ :text => "some other text" })
+    index.document("3").add({ :text => "something else here" })
+
+    results = index.search("some")
+
+    print "#{results['matches']} documents found\\n"
+    results['results'].each { |doc|
+      print "docid: #{doc['docid']}\\n"
+    }
+rescue
+    print "Error: ",$!,"\\n"
+end
+
        +{% endbox %} +
        +

        + You can check the gem specification here. +

        +
        + +
        +
        +

        + Testing your application +

        +

        + Using the code above, you should be able to test your application both locally and running on Heroku. Keep in mind that your tests affect the live index. You can delete and re-create your index anytime to clean it up, either from the code as shown below or from IndexTank's dashboard: +

        +
        +{% box 'code' %} +
        +### Deleting an index
+
+client = IndexTank::Client.new(ENV['INDEXTANK_API_URL'] || '{{ account.get_private_apiurl }}')
+index = client.indexes('{% if indexes.0 %}{{ indexes.0.name }}{% endif %}')
+index.delete()
+
+### Creating a new index
+
+client = IndexTank::Client.new(ENV['INDEXTANK_API_URL'] || '{{ account.get_private_apiurl }}')
+index = client.indexes('{% if indexes.0 %}{{ indexes.0.name }}{% endif %}')
+index.add()
+
+print "Waiting for index to be ready"
+while not index.running?
+    print "."
+    STDOUT.flush
+    sleep 0.5
+end
+print "\\n"
+STDOUT.flush
+
        +{% endbox %} +
        +
        + +
        +
        +

        + Pushing to Heroku +

        +

        + You should be able to use IndexTank from within your application running on Heroku after the usual push commands: +

        +
        +{% box 'code' %} +
        +$ git commit
+$ git push heroku master
+
        +{% endbox %} +
        +
        +
        +
        +
        +{% endblock %} diff --git a/storefront/templates/home.html b/storefront/templates/home.html new file mode 100644 index 0000000..ca4b544 --- /dev/null +++ b/storefront/templates/home.html @@ -0,0 +1,187 @@ +{% extends "base.html" %} + +{% load custom_tags %} +{% load messages %} + +{% block content_head %} +

        Easily add powerful search to your site Good search is good business.

        +{% endblock %} + + +{% block content_body %} +
        +
        +
        +
        +
          +
        • +

          PROVEN FULL-TEXT SEARCH API

          +
          • +
        • Truly real-time: instant updates without reindexing
          • +
        • Geo & Social aware: use location, votes, ratings or comments
          • +
        • Works with Ruby, Rails, Python, Java, PHP, .NET & more!
          • +
        +
          +
        • +

          CUSTOM SEARCH THAT YOU CONTROL

          +
          • +
        • You control how to sort and score results
          • +
        • “Fuzzy”, Autocomplete, Facets for how users really search
          • +
        • Highlights & Snippets quickly shows search results relevance
          • +
        +
          +
        • +

          EASY, FAST & HOSTED

          +
          • +
        • Scalable from a personal blog to hundreds of millions of documents! (try Reddit)
          • +
        • Free up 100K documents
          • +
        • Easier than SQL, SOLR/Lucene & Sphinx.
          • +
        +
        +
        +
        +
        +
        +
        +
        +
        + {% if request.user.is_authenticated %} +

        WELCOME BACK

        +

        Check out your indexes

        +
        My Dashboard
        + {% else %} +

        NEW SIGN-UPS ARE CURRENTLY DISABLED

        + {#

        No new accounts will be created for IndexTank, but there are plans for continuing the service...

        #} +
        LEARN MORE
        + {% endif %} + {% comment %} + + {% endcomment %} +
        +
        +
        +
        +
        +
        + {% include 'includes/testimonial-banner.html' %} +
        +
        +
        +
        + + +
        +
        +
        +

        INDEXTANK IN ACTION

        + +
        +
        +
        +
        +
        +
        + +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +

        Featured Demo

        + tv icon +
        +
        +
        +
        +
        + +
        + +
        +
        +
        +
        +

        See IndexTank in use on the next Generation TV + Network

        +
        +
        +
        +
        +
        + +
        +
        +
        + + +
        +
        +
        +
          +
        • +

          DEVELOPER CONTEST

          +

          + New Contest: IndexTank / 80legs Crawlathon! + by Diego on June 16 +

          +

          + IndexTank / Factual contest results! + by Diego on April 26 +

          +

          +

          Support from our core engineering team is a click away via live chat or email.

          +
          • +
        • +

          WHAT’S NEW

          +

          New IRC channel on Freenode! + Come chat with our community on irc.freenode.net and join the channel #indextank. +

          +

          We've got stemming (beta!) + We're on the last steps of adding stemming. If you know what stemming is, and you're willing to give it a try (knowing that there might be a few bugs) contact us and we will enable this feature for your index. +

          +
          • +
        • + {% if blog_posts %} +

          BLOG

          + {% for post in blog_posts %} +

          {{post.title}} + by {{post.author}} on {{post.date}}

          + {% endfor %} +

          Read more

          + {% endif %} +
          • +
        +
        +
        +
        + + + +{% endblock %} diff --git a/storefront/templates/includes/chosen_plan.html b/storefront/templates/includes/chosen_plan.html new file mode 100644 index 0000000..529515f --- /dev/null +++ b/storefront/templates/includes/chosen_plan.html @@ -0,0 +1,12 @@ +{% load humanize %} + +

        Chosen plan: {{ package.name }}

        + +
        +

        +    -   Maximum index size: {{ package.index_max_size|intcomma }} documents.
        +    -   First 30 days are free! And you can cancel at anytime.
        +    -   Total price: ${{ package.base_price|floatformat:2 }} / month +

        +
        + diff --git a/storefront/templates/includes/clippy.html b/storefront/templates/includes/clippy.html new file mode 100644 index 0000000..846f4da --- /dev/null +++ b/storefront/templates/includes/clippy.html @@ -0,0 +1,22 @@ +{% load custom_tags %} + + + + + + + + + + diff --git a/storefront/templates/includes/dashboard-info.html b/storefront/templates/includes/dashboard-info.html new file mode 100644 index 0000000..1268f16 --- /dev/null +++ b/storefront/templates/includes/dashboard-info.html @@ -0,0 +1,41 @@ +{% load humanize %} + +
        +
        Private URL   + {% with account.get_private_apiurl as text %}{% include 'includes/clippy.html' %}{% endwith %} +
        +
        +
        + {{ account.get_private_apiurl }} +
        +

        + This is the base url for API calls. The easiest way to access our API is instantiating one + of our clients with this URL. +

        + {% comment %} +
        Public URL   + {% with account.get_public_apiurl as text %}{% include 'includes/clippy.html' %}{% endwith %} +
        +
        +
        + {{ account.get_public_apiurl }} +
        +

        + This URL is used for autocomplete API calls. It can be used in public HTML because it + doesn't grant access for altering or querying your indexes. +

        + {% endcomment %} +
        + +
        + +
        +
        Current Plan : {{ account.package.name }} (${{ account.package.base_price|floatformat:2 }}/month)
        +
        +
          +
        • Up to {{ account.package.max_indexes|apnumber }} index{{ account.package.max_indexes|pluralize:"es" }}.
          • +
        • Up to {{ account.package.index_max_size|intcomma }} documents.
          • +
        • Up to {{ account.package.searches_per_day|intcomma }} daily queries.
          • +
        • To upgrade your account, please contact us.
          • +
        +
        \ No newline at end of file diff --git a/storefront/templates/includes/empty-inputs.js b/storefront/templates/includes/empty-inputs.js new file mode 100644 index 0000000..45c0ff6 --- /dev/null +++ b/storefront/templates/includes/empty-inputs.js @@ -0,0 +1,29 @@ +$(function() { + var elems = $('input.empty'); + elems.blur(function() { + var e = $(this); + if (this.value == '') { + e.addClass('empty'); + this.value = this.getAttribute('emptyvalue'); + if (this.type == 'password') { + this.setAttribute('ispass', true); + this.type = 'text'; + } + } + }); + elems.focus(function() { + var e = $(this); + if (e.hasClass('empty')) { + e.removeClass('empty'); + this.value = ''; + if (this.getAttribute('ispass')) { + this.type = 'password'; + } + } + }); + elems.removeClass('empty'); + elems.blur(); + $('form').submit(function() { + $('input.empty').val(''); + }); +}); \ No newline at end of file diff --git a/storefront/templates/includes/field.html b/storefront/templates/includes/field.html new file mode 100644 index 0000000..4ebd769 --- /dev/null +++ b/storefront/templates/includes/field.html @@ -0,0 +1,7 @@ +
        + +
        +
        {% for e in field.errors %}{{e}} {% endfor %}
        +
        \ No newline at end of file diff --git a/storefront/templates/includes/footer.html b/storefront/templates/includes/footer.html new file mode 100644 index 0000000..945c073 --- /dev/null +++ b/storefront/templates/includes/footer.html @@ -0,0 +1,53 @@ + + + + + +
          +
        • Web Analytics
          • +
        + + + + + diff --git a/storefront/templates/includes/head.html b/storefront/templates/includes/head.html new file mode 100644 index 0000000..a509323 --- /dev/null +++ b/storefront/templates/includes/head.html @@ -0,0 +1,101 @@ +{% load custom_tags %} + + + + + + + + +{% if provisioner == 'heroku' %} + +{% endif %} + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/storefront/templates/includes/header.html b/storefront/templates/includes/header.html new file mode 100644 index 0000000..42f88c3 --- /dev/null +++ b/storefront/templates/includes/header.html @@ -0,0 +1,3 @@ +{% load custom_tags %} + +

        IndexTank - Custom search you control

        diff --git a/storefront/templates/includes/heroku-header.html b/storefront/templates/includes/heroku-header.html new file mode 100644 index 0000000..d791a56 --- /dev/null +++ b/storefront/templates/includes/heroku-header.html @@ -0,0 +1,19 @@ +
        + + + +
        diff --git a/storefront/templates/includes/histogram.html b/storefront/templates/includes/histogram.html new file mode 100644 index 0000000..bbad9f0 --- /dev/null +++ b/storefront/templates/includes/histogram.html @@ -0,0 +1,45 @@ + diff --git a/storefront/templates/includes/loggedin.html b/storefront/templates/includes/loggedin.html new file mode 100644 index 0000000..f1a620e --- /dev/null +++ b/storefront/templates/includes/loggedin.html @@ -0,0 +1,6 @@ + diff --git a/storefront/templates/includes/login_area.html b/storefront/templates/includes/login_area.html new file mode 100644 index 0000000..9152a94 --- /dev/null +++ b/storefront/templates/includes/login_area.html @@ -0,0 +1,18 @@ +{% load custom_tags %} + +{% if provisioner %} +
        + +
        +{% else %} +
        + {% include 'includes/signin-box.html' %} +
        +
        + {% include 'includes/loggedin.html' %} +
        +{% endif %} diff --git a/storefront/templates/includes/menu.html b/storefront/templates/includes/menu.html new file mode 100644 index 0000000..441c56c --- /dev/null +++ b/storefront/templates/includes/menu.html @@ -0,0 +1,9 @@ + diff --git a/storefront/templates/includes/olark.js b/storefront/templates/includes/olark.js new file mode 100644 index 0000000..3335de8 --- /dev/null +++ b/storefront/templates/includes/olark.js @@ -0,0 +1 @@ +/**/ \ No newline at end of file diff --git a/storefront/templates/includes/search.html b/storefront/templates/includes/search.html new file mode 100644 index 0000000..c4bcb18 --- /dev/null +++ b/storefront/templates/includes/search.html @@ -0,0 +1,14 @@ + +
        +
        + +
        + +
        +
        +
        + diff --git a/storefront/templates/includes/signin-box.html b/storefront/templates/includes/signin-box.html new file mode 100644 index 0000000..e53d9ff --- /dev/null +++ b/storefront/templates/includes/signin-box.html @@ -0,0 +1,13 @@ + + diff --git a/storefront/templates/includes/staff-banner.html b/storefront/templates/includes/staff-banner.html new file mode 100644 index 0000000..c39a147 --- /dev/null +++ b/storefront/templates/includes/staff-banner.html @@ -0,0 +1,102 @@ +{% load custom_tags %} + +{% with 8|random as p %} + {% if p == 1 or showall %} + + {% endif %} + {% if p == 2 or showall %} + + {% endif %} + {% if p == 3 or showall %} + + {% endif %} + {% if p == 4 or showall %} + + {% endif %} + {% if p == 5 or showall %} + + {% endif %} + {% if p == 6 or showall %} + + {% endif %} + {% if p == 7 or showall %} + + {% endif %} + {% if p == 8 or showall %} + + {% endif %} + {% if p == 0 %} + + {% endif %} + {% if p == 0 %} +     +    {% endif %} +    {% if p == 0 %} +     +    {% endif %} +    {% if p == 0 %} +     +    {% endif %} +    {% if p == 0 %} +     +    {% endif %} +    {% if p == 0 %} +     +    {% endif %} +{% endwith %} diff --git a/storefront/templates/includes/testimonial-banner.html b/storefront/templates/includes/testimonial-banner.html new file mode 100644 index 0000000..a052046 --- /dev/null +++ b/storefront/templates/includes/testimonial-banner.html @@ -0,0 +1,84 @@ +{% load custom_tags %} + +{% with 17|random as p %} + {% if p == 1 or showall %} +

        Search works so much better than if we’d built it.
        + —David King, Reddit

        + {% endif %} + {% if p == 2 or showall %} +

        We’re not just getting the product itself and hosting like we were before. We’re getting access to a team of experts.
        + —David King, Reddit

        + {% endif %} + {% if p == 3 or showall %} +

        Lightning-fast search dropped into #cmyk courtesy of @indextank ;)
        + —Patric M., @719dotcom

        + {% endif %} + {% if p == 4 or showall %} +

        I just switched from Solr to IndexTank.com and MY GOD is it beautiful.
        + —Jack C., @jackdanger

        + {% endif %} + {% if p == 5 or showall %} +

        If you try doing geo stuff Solr is just broken. With #indextank you provide a sorting function like "relevance / distance"
        + —Jack C., @jackdanger

        + {% endif %} + {% if p == 6 or showall %} +

        Hey I just implemented IT in our site autocomplete and search feature and it's much more faster than before. Awesome product! :)
        + —Pierre-Luc Brunet, ashpy.com

        + {% endif %} + + {% if p == 7 or showall %} +

        Let's not forget @indextank while we're praising service providers. You guys rock!
        + —phraseup*, phraseup*

        + {% endif %} + + {% if p == 8 or showall %} +

        Just came across @IndexTank. Look sick. No more ThinkingSphinx for me!
        + —Bill Palmer, @billpalmertv

        + {% endif %} + + {% if p == 9 or showall %} +

        We are using @indextank for our websites search. Just got direct support from the founder. #GreatProductGreatPeople
        + —Jason S., @peregrine

        + {% endif %} + + {% if p == 10 or showall %} +

        @loniszczuk Helped me out today. @indextank is the coolest text search api around. Their customer service is top notch. #twothumbsup
        + —Leon Strachan, @leonstrachan

        + {% endif %} + + {% if p == 11 or showall %} +

        very impressed with the quick and helpful support @indextank, thanks @dbuthay
        + —Adam Maddox, @adammaddox

        + {% endif %} + + {% if p == 12 or showall %} +

        Great blog post by a happy and passionate @IndexTank User: All startups should use IndexTank for searching http://k9.vc/r5ML9J
        + —ManuKumar, @ManuKumar

        + {% endif %} + + {% if p == 13 or showall %} +

        @dbasch @IndexTank cheers. We generally use Solr in our search systems but thismight be useful for wordpress solutions
        + —Liam Sheerin, @Liam_Sheerin

        + {% endif %} + + {% if p == 14 or showall %} +

        checking out @indextank on @akuzemchak's recommendation. very cool hosted website search with a ton of options & powerful API
        + —chris busse, @busse

        + {% endif %} + + {% if p == 15 or showall %} +

        trying http://indextank.com a hosted search API for fulltext search. works like a charm! @indextank
        + —Claudio Semeraro, @keepitterron

        + {% endif %} + + {% if p == 16 or showall %} +

        Thanks for your support, @indextank
        + —Getaround, @Getaround

        + {% endif %} + + {% if p == 17 or showall %} +

        Very impressed with @indextank - easy full-text search api and friendly support - indextank.com
        + —David Baldwin, @baldwindavid

        + {% endif %} + +{% endwith %} diff --git a/storefront/templates/includes/twitter-banner.html b/storefront/templates/includes/twitter-banner.html new file mode 100644 index 0000000..d58d3f2 --- /dev/null +++ b/storefront/templates/includes/twitter-banner.html @@ -0,0 +1,35 @@ +{% load custom_tags %} + +{% with 5|random as p %} +
        + +

        + {% if p == 1 or showall %} + treeder: + Just tried @IndexTank, color me impressed! - Hosted Real-Time Search http://bit.ly/g9cInn + on Monday, February 14 via web + {% endif %} + {% if p == 2 or showall %} + itsemil: + @IndexTank saves the day(for pilarhq) again. This is the second time I tweet about them. IndexTank, amazing support, scaled real-time search + on Monday, February 14 via web + {% endif %} + {% if p == 3 or showall %} + jamesthigpen: + I played with solr this morning and the value proposition of indextank became incredibly obvious. + on Saturday, January 29 via Twitter for iPhone + {% endif %} + {% if p == 4 or showall %} + Geletka: + The folks at indextank are a class act. If you have a rails site on heroku, you should check out their product. + on Friday, February 18 via Twitter for iPad + {% endif %} + {% if p == 5 or showall %} + howardpyle: + Really digging IndexTank (search API) and their app contest entries. Check out this one to search dev subreddits: http://www.proggitftw.com/ + on Thursday, February 17 via HootSuite + {% endif %} +

        +         +
        +{% endwith %} diff --git a/storefront/templates/insights.html b/storefront/templates/insights.html new file mode 100644 index 0000000..dd82bc8 --- /dev/null +++ b/storefront/templates/insights.html @@ -0,0 +1,156 @@ +{% load custom_tags %} +{% load humanize %} + +
        +

        Content Insights

        +
        +{% if insights.doc_avgs or insights.doc_by_time %} +
        +

        Last update

        +
          +
        • {{ insights_update.doc_avgs|timesince }} ago
          • +
        • at {{ insights_update.doc_avgs|date:"g:i a, F j" }}
          • +
        +
        +

        Index Size

        +
          +
        • {{ index.current_docs_number|intcomma }} documents
          • +
        • {{ insights.doc_avgs.sentenceCount|intcomma }} sentences
          • +
        • {{ insights.doc_avgs.wordCount|intcomma }} words
          • +
        • {{ insights.doc_avgs.charCount|intcomma }} characters
          • +
        +
        +

        Average document

        +
          +
        • {{ insights.doc_avgs.sentencesPerDoc|floatformat:2 }} sentences
          • +
        • {{ insights.doc_avgs.wordsPerDoc|floatformat:2 }} words
          • +
        • {{ insights.doc_avgs.documentSize|floatformat:2 }} characters
          • +
        +
        + +
        +

        Documents over time (based on document timestamps)

        +
        + {% with insights.doc_by_time as data %} + {% with "#doc_by_time_"|add:index.code as target %} + {% with "1" as time %} + {% include 'includes/histogram.html' %} + {% endwith %} + {% endwith %} + {% endwith %} + +
        +

        Document size distribution (in chars)

        +
        + {% with insights.doc_avgs.sizeHistogram as data %} + {% with "#sizeHistogram_"|add:index.code as target %} + {% include 'includes/histogram.html' %} + {% endwith %} + {% endwith %} +
        +{% else %} +
        + No insights have been generated yet for this index. +
        +

        Insights are generated every couple of hours, come back later to check if yours are done.

        +{% endif %} + +{% comment %} + + + + + +
        + +
        + + + + + {% if index.is_ready %} + + {% else %} + + {% endif %} + + + + + + +
        + {{ index.name }} insights + < dashboardmanageRunningStarting
        + {% if insights.doc_avgs or insights.doc_by_time %} + {# charCount, wordCount, sentenceCount, wordsPerDoc, sentencesPerDoc, documentSize, sizeHistogram #} +
        +
        Index created on
        +
          + {{ index.creation_time|date:"g a, F j Y" }} +
        +
        Insights last updated at
        +
          + {{ insights_update.doc_avgs|date:"g:i a, F j" }} +
        +
        +
        +
        Index Size
        +
          +
        • {{ index.current_docs_number|intcomma }} documents
          • +
        • {{ insights.doc_avgs.sentenceCount|intcomma }} sentences
          • +
        • {{ insights.doc_avgs.wordCount|intcomma }} words
          • +
        • {{ insights.doc_avgs.charCount|intcomma }} characters
          • +
        +
        +
        +
        Average document
        +
          +
        • {{ insights.doc_avgs.sentencesPerDoc|floatformat:2 }} sentences
          • +
        • {{ insights.doc_avgs.wordsPerDoc|floatformat:2 }} words
          • +
        • {{ insights.doc_avgs.documentSize|floatformat:2 }} characters
          • +
        +
        +
        +
        +
        Documents over time (based on document timestamps)
        +
        + {% with insights.doc_by_time as data %} + {% with "#doc_by_time" as target %} + {% with "1" as time %} + {% include 'includes/histogram.html' %} + {% endwith %} + {% endwith %} + {% endwith %} +
        +
        +
        Document size distribution (in chars)
        +
        + {% with insights.doc_avgs.sizeHistogram as data %} + {% with "#sizeHistogram" as target %} + {% include 'includes/histogram.html' %} + {% endwith %} + {% endwith %} +
        +
        + {% else %} +
        +
      • + No insights have been generated for this index
        + Insights are generated every couple of hours, come back later to check if yours are done +
        • +
        + +

        Current index size: {{ index.current_docs_number|intcomma }} documents

        +         + {% endif %} +
        + + {% include 'includes/dashboard-info.html' %} + +
        + {% include 'includes/staff-banner.html' %} +
        + +{% endblock %} +{% endcomment %} \ No newline at end of file diff --git a/storefront/templates/instruments.html b/storefront/templates/instruments.html new file mode 100644 index 0000000..a78597f --- /dev/null +++ b/storefront/templates/instruments.html @@ -0,0 +1,100 @@ +{% load custom_tags %} +{% load analytical %} + + + {% analytical_head_top %} + + + + + + + + + + + + + + + + {% analytical_head_bottom %} + + + + {% analytical_body_top %} +
        +
        +
        +
        + +
        +
        +
        +
        + {% analytical_body_bottom %} + + diff --git a/storefront/templates/layout.html b/storefront/templates/layout.html new file mode 100644 index 0000000..35ce32f --- /dev/null +++ b/storefront/templates/layout.html @@ -0,0 +1,73 @@ + +{% load custom_tags %} +{% load messages %} +{% load analytical %} + + + + {% analytical_head_top %} + {% block head %} + + {% block page_title %}{% endblock %} + {% endblock %} + {% analytical_head_bottom %} + + + + {% analytical_body_top %} + + {% block body %} + +
        + +
        +
        + +
        +
        + + +
        +
        + +
        + {% block login_area %}{% endblock %} +
        + + +
        + {% block content_head %}{% endblock %} +
        + + +
        +
        +
        + + + {% block content_body %}{% endblock %} + + + +
        + +
        + +
        + + {% endblock %} + {% analytical_body_bottom %} + + + +{% comment %} + {% block messages %}{% endblock %} +{% endcomment %} diff --git a/storefront/templates/login.html b/storefront/templates/login.html new file mode 100644 index 0000000..3e87519 --- /dev/null +++ b/storefront/templates/login.html @@ -0,0 +1,33 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block title %}Log in{% endblock %}endblock %} + +{% block common_content %} + {% box 'form' %} +

        Log in to IndexTank

        +
        +
        + {% for f in login_form %} +
        + +
        +
        {% for e in f.errors %}{{e}} {% endfor %}
        +
        + {% endfor %} + {% if login_message %} +
        {{ login_message }}
        + {% else %} + {% endif %} +
        + +
        +
        + {% endbox %} +{% endblock %} + diff --git a/storefront/templates/manage_get_doc.html b/storefront/templates/manage_get_doc.html new file mode 100644 index 0000000..ca772d6 --- /dev/null +++ b/storefront/templates/manage_get_doc.html @@ -0,0 +1,33 @@ + +

        DOC ID

        +
          +
        • +
          {{docid}}
          +
          • +
        +

        FIELDS

        +
          + {% for k,v in doc.fields.items %} +
        • {{k}}
          • +
        • +
          {{v}}
          +
          • + {% endfor %} +
        +

        VARIABLES

        +
          + {% for k,v in doc.variables.items %} +
        • {{k}}: {{v}}
          • + {% empty %} +
        • - None -
          • + {% endfor %} +
        +

        CATEGORIES

        +
          + {% for k,v in doc.categories.items %} +
        • {{k}}: {{v}}
          • + {% empty %} +
        • - None -
          • + {% endfor %} +
        +         \ No newline at end of file diff --git a/storefront/templates/manage_index.html b/storefront/templates/manage_index.html new file mode 100644 index 0000000..d727ff8 --- /dev/null +++ b/storefront/templates/manage_index.html @@ -0,0 +1,134 @@ +{% load humanize %} +{% load custom_tags %} + +
        +

        Scoring Functions

        +
        + Documentation is available here.
        +Examples: + 
        relevance * max(1, log(doc.var[2]))
        (sorts documents considering the textual relevance of the document times the natural logarithm of the third var in the document, or 1 if it's lower)         + 
        miles(query.var[0], query.var[1], doc.var[0], doc.var[1])
        (sorts documents considering the distance between doc and a point passed in the query)
        +
        + {% for func in functions %} +
        +
        +
        Function #{{ func.name }}
        +
        + +
        +
        + + + +
        +
        +
        +
        + {% endfor %} + +
        + +

        Public search API

        + +

        + Current status: + + + Enabled - + [Disable now] + + + + Disabled - + [Enable now] + +

        + + +

        Deleting the index

        + +
        + + If you need to, you can completely delete your current index. This action cannot be undone. +
        + + + + diff --git a/storefront/templates/manage_inspect.html b/storefront/templates/manage_inspect.html new file mode 100644 index 0000000..a85f596 --- /dev/null +++ b/storefront/templates/manage_inspect.html @@ -0,0 +1,141 @@ +{% load humanize %} +{% load custom_tags %} + +
        +

        Search your index

        +
        + +
        + + + + + + + + +
        Query + + Func # + + + +
        +
        + +
        + + + + + + + + + + +
        +
        + DOC ID +
        + + +
        + + + diff --git a/storefront/templates/new-index.html b/storefront/templates/new-index.html new file mode 100644 index 0000000..e4560c8 --- /dev/null +++ b/storefront/templates/new-index.html @@ -0,0 +1,34 @@ +{% extends "common-base.html" %} +{% load humanize %} +{% load custom_tags %} + +{% block title %}Create a new index{% endblock %}endblock %} + +{% block common_content %} + {% box 'form' %} +

        Creating an index

        +

        Choose the name that will indentify your new index in the dashboard and the API. Only digits, letters and "_" are allowed.

        +
        +
        + {% for f in form %} +
        + +
        +
        {% for e in f.errors %}{{e}} {% endfor %}
        +
        + {% endfor %} + {% if message %} +
        {{ message }}
        + {% endif %} +
        + +
        +
        + {% endbox %} +{% endblock %} + diff --git a/storefront/templates/pricing.html b/storefront/templates/pricing.html new file mode 100644 index 0000000..59e5ca6 --- /dev/null +++ b/storefront/templates/pricing.html @@ -0,0 +1,178 @@ +{% extends "base.html" %} +{% load custom_tags %} +{% load humanize %} + +{% block extrahead %} + {{ block.super }} + + + +{% endblock %} + + +{% block title %}Pricing Free for the first 100k docs, upgrade any time.{% endblock %} + +{% block tryit %}{% endblock %} + +{% block content_body %} +
        +
        +
        + {% box 'largebox' %} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {% comment %} + + + {% if user.is_authenticated %} + {% if user.get_profile.account.package.base_price > 0 %} + + + + + {% else %} + + + + + {% endif %} + {% else %} + + + + + {% endif %} + + {% endcomment %} +
        Starter Free $0/monthPlus $49/monthPro $175/monthCustom contact us
        Core FeaturesYesYesYesYes
        Documents100K500K2MCustom
        Unlimited queries *YesYesYesYes
        Requires logoYesNoNoNo
        Contact usContact usContact usContact usContact usContact usContact usContact usContact us
        + {% endbox %} +
        + {##} + + +
        + +
        +
        +
        + +
        +
        +
        +

        JOIN THE GROWING LIST OF TOP COMPANIES TRUSTING INDEXTANK

        +
        + + + + + +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +

        Questions

        +
          +
        • +

          Can I try it out without having to enter my credit card?

          +
          • +
        • +

          Yes. Our free account just requires you to enter an email address.

          +
          • +
        • +

          Will you help me get started?

          +
          • +
        • +

          Absolutely! Try our tutorials for the quickest way to get up to speed. + Every plan gets our great customer support from our own team of developers. + We’ve been working on search for years, and want to share our experience with + you the moment you need help. Plus, we also know we can learn from you as well, + so please be in touch.

          +
          • +
        • +

          Can I upgrade my account?

          +
          • +
        • +

          Yes. If you have a free account you can upgrade it to a paid plan automatically. + If you want to upgrade or downgrade an already paying account, please contact us and we'll help you.

          +
          • +
        • +

          Can I cancel any time I want?

          +
          • +
        • +

          Yes. You can cancel your account at any time without any prior notification.

          +
          • +
        • +

          How do I cancel my service?

          +
          • +
        • +

          Send us an email telling us you want to cancel, and we’ll take care of it.

          +
          • +
        • +

          When will you charge me?

          +
          • +
        • +

          We charge for the paid plans every month in advance, and we provide the first + month free of charge. This means that if you sign up, for example, on January 15th, + you will not be charged until February 15th, and that will pay for the period from + February 15th until March 14th.

          +
          • +
        +
        +
        +
        + {% box 'chat' %} +

        HAVE A QUESTION?

        +
        LIVE CHAT NOW
        +

        Our team of experienced and friendly devs will respond almost as quickly as our search API.

        + {% endbox %} +
        +
        +
        +
        + + +{% endblock %} + + diff --git a/storefront/templates/quotes.html b/storefront/templates/quotes.html new file mode 100644 index 0000000..50e8eee --- /dev/null +++ b/storefront/templates/quotes.html @@ -0,0 +1,14 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block title %}Quotes{% endblock %}endblock %} + +{% block common_content %} + {% with "1" as showall %} +

        From twitter

        + {% include 'includes/twitter-banner.html' %} +

        From contestants

        + {% include 'includes/staff-banner.html' %} + {% endwith %} +{% endblock %} + diff --git a/storefront/templates/score_functions.html b/storefront/templates/score_functions.html new file mode 100644 index 0000000..8486b19 --- /dev/null +++ b/storefront/templates/score_functions.html @@ -0,0 +1,72 @@ +{% extends "common-base.html" %} +{% load humanize %} +{% load custom_tags %} + + +{% block title %}Scoring Functions{% endblock %}endblock %} + +{% block common-content %} + +

        + Manage your scoring functions. +

        + {% if message %} +
        + {{ message }} +
        + {% endif %} + +
        +
        + + + + +

        + {% for function in functions %} +

        +
        Formula #{{ function.name }}:

        {% if function.definition %}{{ function.definition }}{% else %}[undefined]{% endif %}
        + +
        Edit
        +         + +
        Remove
        +         + +
        +
        + + + {% endfor %} +

        + +{% endblock %} + diff --git a/storefront/templates/search_results.html b/storefront/templates/search_results.html new file mode 100644 index 0000000..a8a8ad3 --- /dev/null +++ b/storefront/templates/search_results.html @@ -0,0 +1,58 @@ +{% extends "common-base.html" %} +{% load custom_tags %} +{% load humanize %} + +{% block extrahead %} + {{ block.super }} + + +{% endblock %} + +{% block title %}Search IndexTank.com {% endblock %} + +{% block common_content %} +
        +
        +
        +
        +{% endblock %} diff --git a/storefront/templates/sidebar-base.html b/storefront/templates/sidebar-base.html new file mode 100644 index 0000000..6e6519d --- /dev/null +++ b/storefront/templates/sidebar-base.html @@ -0,0 +1,19 @@ +{% extends "base.html" %} + +{% load messages %} + +{% block content_body %} +
        +
        +
        +
        +
        + {% render_messages messages %} + {% block common_content %} + {% endblock %} +
        +
        +
        +
        +
        +{% endblock %} diff --git a/storefront/templates/sidebar-short-base.html b/storefront/templates/sidebar-short-base.html new file mode 100644 index 0000000..fd1a314 --- /dev/null +++ b/storefront/templates/sidebar-short-base.html @@ -0,0 +1,23 @@ +{% extends "base.html" %} + +{% load messages %} + +{% block content %} +
        +
        + {% block presidebar %}{% endblock %} +
        + {% block sidebar_content %}{% endblock %} +
        +
        + {% block postsidebar %}{% endblock %} +
        +
        + {% render_messages messages %} + {% block common_content %}{% endblock %} +
        +
        +
        +
        +
        +{% endblock %} diff --git a/storefront/templates/sign_up.html b/storefront/templates/sign_up.html new file mode 100644 index 0000000..beffa84 --- /dev/null +++ b/storefront/templates/sign_up.html @@ -0,0 +1,46 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block common_content %} + {% if invitation %} +

        This is an invitation to create a BETA account!

        + {% else %} +

        Create an account and get your API key!

        + {% endif %} + +
        +

        You need an email and a password to get started

        + + +
        + {% if message %} +
        + {{ message }} +
        + {% endif %} + + + {% for f in sign_up_form %} + + + + + {% endfor %} + + + + +
        {{ f.label_tag }}{{ f }} +
        {% for e in f.errors %}{{e}} {% endfor %}
        +
        +
        + +
        Next step >>
        +         +
        + +
        +
        +
        +{% endblock %} + diff --git a/storefront/templates/special_offer.html b/storefront/templates/special_offer.html new file mode 100644 index 0000000..c04c087 --- /dev/null +++ b/storefront/templates/special_offer.html @@ -0,0 +1,102 @@ +{% extends "common-base.html" %} +{% load custom_tags %} +{% load humanize %} + +{% block bodyclasses %}{{ block.super }} get_started step{{ step }}{% endblock %} +{% block common_content %} + +

        + Primary Messaging
        optional subtitle

        + +
        +
        +

        Secondary messaging

        +

        Secondary messaging

        +
        +

        get started now

        + $0/month +
        +

        Secondary messaging

        +

        Secondary messaging

        +
         
        +
        +
        + +
        +
        +
        +
        +

        What's included?

        +
        + + Core Features +
        +
        + + Documents = 5,000 +
        +
        + + Storage = 10MB +
        +
        + + Queries = 500/day +
        +
        + + Geolocation +
        +
        + + Faceting +
        +
        +
        +
        +
        + +
        +
        Want more power? checkout our complete pricing plans.
        +
        +

        Questions?

        +
        +

        + Can I try it out without having to enter my credit card?
        + Yes. Our free account just requires you to enter an email address and choose a password. +

        +

        + Will you help me get started?
        + Absolutely! Try our tutorials for the quickest way to get up to speed. Every plan gets our great customer support from our own team of developers. We’ve been working on search for years, and want to share our experience with you the moment you need help. Plus, we also know we can learn from you as well, so please be in touch. +

        +

        + How do I upgrade from the Free package to the Starter package?
        + Contact us via email . +

        +
        +
        +

        + Can I switch plans?
        + Sure. We’re working on automating this process, but in the meantime, contact us and we’ll do it for you. +

        +

        + Can I cancel any time I want?
        + Yes. You can cancel your account at any time without any prior notification. +

        +

        + How do I cancel my service?
        + Send us an email telling us you want to cancel, and we’ll take care of it. +

        +

        + When will you charge me?
        + We charge for the paid plans every month in advance, and we provide the first month free of charge. This means that if you sign up, for example, on January 15th, you will not be charged until February 15th, and that will pay for the period from February 15th until March 14th. +

        +
        +
        + + {% include 'includes/staff-banner.html' %} +
        + + +{% endblock %} + diff --git a/storefront/templates/used_invite.html b/storefront/templates/used_invite.html new file mode 100644 index 0000000..62924a2 --- /dev/null +++ b/storefront/templates/used_invite.html @@ -0,0 +1,11 @@ +{% extends "common-base.html" %} +{% load custom_tags %} + +{% block common_content %} +

        The invitation has already been used up

        + + +
        +

        Please, log in with your already generated credentials.

        +{% endblock %} + diff --git a/storefront/templates/we_are_down.html b/storefront/templates/we_are_down.html new file mode 100644 index 0000000..bcef4b7 --- /dev/null +++ b/storefront/templates/we_are_down.html @@ -0,0 +1,68 @@ +{% load custom_tags %} + + + + IndexTank is down + + + + + + + + + + + + + + + + + + + + + + +
        + +
        +
        + +
        +
        +
        +

        We are down

        +

        Amazon is currently experiencing degradation on their service. They are working to resolve it.

        +

        Once we get access to our data and backups we'll work as fast as possible to restore our service. We apologize for the inconvenience.

        +
        +
        + + diff --git a/storefront/templates/why_us.html b/storefront/templates/why_us.html new file mode 100644 index 0000000..f237da1 --- /dev/null +++ b/storefront/templates/why_us.html @@ -0,0 +1,211 @@ +{% extends "base.html" %} +{% load custom_tags %} +{% load humanize %} + +{% block title %}Why Us{% endblock %} + +{% block content_body %} +
        +
        +
        +

        COMPARE OUR FEATURES

        +
        +
          +
        • image
          • +
        • image
          • + {% comment %} +
        • image
          • +
        • image
          • + {% endcomment %} +
        +
        +
        +
        +
        +
        +
        placeholder +

        After a bout with infamously disappointing search, Reddit found IndexTank, which easily handles the company’s huge index, delivers real-time results, and has helped re-polish their reputation.

        +
        “If we had unlimited programming resources, we could have built something great, but it was much easier to outsource, saving us the time, hassle and headache.â€
        — Jeremy Edberg, Reddit
        +
          +
        • IndexTank delivers the right search results to 30% more users, Saves 20-30 hours a month.
          • +
        • They have complete control so the results and filters to sort by are just the way the users like it.
          • +
        • Search results so fast they wanted to display the speed.
          • +
        • User Generate Votes are part of search results and relevance in real-time.
          • +
        +
        +
        +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +

        PROVEN FULL-TEXT SEARCH API

        +
          +
        • +

          True Real-time

          +
          • +
        • +

          Indextank is built for the needs of modern web sites where content is rapidly changing. With IndexTank, + your index gets updated instantly. Our engines are optimized to update relevance factors frequently with + zero fragmentation penalty.

          +
          • +
        • +

          Geo & Social aware

          +
          • +
        • +

          Use locations, votes, ratings or comments to improve the quality of your search results. These factors + can be used to boost relevant documents and to restrict the results to a radius of a geographic location.

          +
          • +
        • +

          Libraries and Tutorials

          +
          • +
        • +

          Use the service with our official libraries for several platforms. We provide clients for + Ruby, + Python, + PHP and + Java + as well as plug-ins for different platforms. + The community also contributes several alternatives to these and other languages. Besides, our Rest API is + super simple so you can develop your own client with very little effort. +

          +
          • +
        +
        +
        +

        CUSTOM SEARCH THAT YOU CONTROL

        +
          +
        • +

          Sorting & Scoring

          +
          • +
        • +

          In IndexTank, results ordering is defined by a custom sorting algorithm + that is expressed in a formula constructed using IndexTank's built-in functions and values. + These allow to incorporate the textual relevance of the result, the age of the documents, + geographic distances and any custom factors defined in the documents.

          +
          • +
        • +

          Highlights & Snippets

          +
          • +
        • +

          Get relevant snippets with your search results with no extra effort, showing your users the + reason why those results are relevant and allowing them to understand what they're doing and + how to refine their query if necessary.

          +
          • +
        • +

          Faceting

          +
          • +
        • +

          Simply define the categories to which your documents belong and automatically get + faceting functionality. Your users will + be able to refine their queries to specific categories while seeing before hand the number of + documents in each category.

          +
          • +
        • +

          "Fuzzy" Search

          +
          • +
        • +

          Sometimes it is useful to present results that wouldn't normally match the user's query but + could potentially be relevant. For this, IndexTank has different options such as OR queries, + text stemming and "Did you mean" suggestions.

          +
          • +
        +
        +
        +

        EASY, FAST & HOSTED

        +
          +
        • +

          Scalable

          +
          • +
        • +

          Run for free up to 100K documents and don't worry about scaling. Our service seamlessly + supports indexes of tens of millions of documents.

          +
          • +
        • +

          Fast results

          +
          • +
        • +

          IndexTank was designed to be super fast because we know speed is a key factor to + provide a good search service.

          +
          • +
        • +

          Easy to use

          +
          • +
        • +

          IndexTank API is very simple and is designed for simplicity, you should be able to get + up and running very quickly.

          +
          • +
        +
        +
        +
        +
        +
        +
        +
        +
        +

        HAVE A QUESTION?

        +
        LIVE CHAT NOW
        +

        Our team of experienced and friendly devs will respond almost as quickly as our search API.

        +
        +
        +
        +
        +
        +
        +

        TESTIMONIALS

        +
          +

        • “We’re not just getting the product itself and hosting like we were before. + We’re getting access to a team of experts.” —David King, Reddit

          +
          • +
        • +

          “Lightning-fast search dropped into #cmyk courtesy of @indextank ;)” + —Patric M @719dotcom

          +
          • +
        • +

          “I just switched from Solr to IndexTank.com + and MY GOD is it beautiful.”—Jack C + @jackdanger

          +
          • +
        +
        +
        +

        PRESS

        +
        +
        +
        +
        +
        +
        +{% endblock %} + diff --git a/storefront/templatetags/__init__.py b/storefront/templatetags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/storefront/templatetags/analytical.py b/storefront/templatetags/analytical.py new file mode 100644 index 0000000..3899adb --- /dev/null +++ b/storefront/templatetags/analytical.py @@ -0,0 +1,81 @@ +""" +Analytical template tags and filters. +""" + +from __future__ import absolute_import + +import logging + +from django import template +from django.template import Node, TemplateSyntaxError +from django.utils.importlib import import_module +from templatetags.utils import AnalyticalException + + +TAG_LOCATIONS = ['head_top', 'head_bottom', 'body_top', 'body_bottom'] +TAG_POSITIONS = ['first', None, 'last'] +TAG_MODULES = [ + 'storefront.clicky', + 'storefront.mixpanel', + 'storefront.google_analytics', +] +''' + 'storefront.olark', + 'analytical.chartbeat', + 'analytical.crazy_egg', + 'analytical.gosquared', + 'analytical.hubspot', + 'analytical.kiss_insights', + 'analytical.kiss_metrics', + 'analytical.optimizely', + 'analytical.performable', + 'analytical.reinvigorate', + 'analytical.woopra', +''' + + +logger = logging.getLogger(__name__) +register = template.Library() + + +def _location_tag(location): + def analytical_tag(parser, token): + bits = token.split_contents() + if len(bits) > 1: + raise TemplateSyntaxError("'%s' tag takes no arguments" % bits[0]) + return AnalyticalNode(location) + return analytical_tag + +for loc in TAG_LOCATIONS: + register.tag('analytical_%s' % loc, _location_tag(loc)) + + +class AnalyticalNode(Node): + def __init__(self, location): + self.nodes = [node_cls() for node_cls in template_nodes[location]] + + def render(self, context): + return "".join([node.render(context) for node in self.nodes]) + + +def _load_template_nodes(): + template_nodes = dict((l, dict((p, []) for p in TAG_POSITIONS)) + for l in TAG_LOCATIONS) + def add_node_cls(location, node, position=None): + template_nodes[location][position].append(node) + for path in TAG_MODULES: + module = _import_tag_module(path) + try: + module.contribute_to_analytical(add_node_cls) + except AnalyticalException, e: + logger.debug("not loading tags from '%s': %s", path, e) + for location in TAG_LOCATIONS: + template_nodes[location] = sum((template_nodes[location][p] + for p in TAG_POSITIONS), []) + return template_nodes + +def _import_tag_module(path): + app_name, lib_name = path.rsplit('.', 1) + return import_module("%s.templatetags.%s" % (app_name, lib_name)) + +template_nodes = _load_template_nodes() diff --git a/storefront/templatetags/clicky.py b/storefront/templatetags/clicky.py new file mode 100644 index 0000000..246186b --- /dev/null +++ b/storefront/templatetags/clicky.py @@ -0,0 +1,77 @@ +""" +Clicky template tags and filters. +""" + +from __future__ import absolute_import + +import re + +from django.template import Library, Node, TemplateSyntaxError +from django.utils import simplejson + +from templatetags.utils import get_identity, is_internal_ip, disable_html, \ + get_required_setting + + +SITE_ID_RE = re.compile(r'^\d+$') +TRACKING_CODE = """ + + +""" + + +register = Library() + + +@register.tag +def clicky(parser, token): + """ + Clicky tracking template tag. + + Renders Javascript code to track page visits. You must supply + your Clicky Site ID (as a string) in the ``CLICKY_SITE_ID`` + setting. + """ + bits = token.split_contents() + if len(bits) > 1: + raise TemplateSyntaxError("'%s' takes no arguments" % bits[0]) + return ClickyNode() + +class ClickyNode(Node): + def __init__(self): + self.site_id = get_required_setting('CLICKY_SITE_ID', SITE_ID_RE, + "must be a (string containing) a number") + + def render(self, context): + custom = {} + for dict_ in context: + for var, val in dict_.items(): + if var.startswith('clicky_'): + custom[var[7:]] = val + if 'username' not in custom.get('session', {}): + identity = get_identity(context, 'clicky') + if identity is not None: + custom.setdefault('session', {})['username'] = identity + + html = TRACKING_CODE % {'site_id': self.site_id, + 'custom': simplejson.dumps(custom)} + if is_internal_ip(context, 'CLICKY'): + html = disable_html(html, 'Clicky') + return html + + +def contribute_to_analytical(add_node): + ClickyNode() # ensure properly configured + add_node('body_bottom', ClickyNode) diff --git a/storefront/templatetags/custom_tags.py b/storefront/templatetags/custom_tags.py new file mode 100644 index 0000000..6e30f86 --- /dev/null +++ b/storefront/templatetags/custom_tags.py @@ -0,0 +1,70 @@ +from django import template +from django.template import Node + +from django.conf import settings +import random + +register = template.Library() + +def var_tag_compiler(params, defaults, name, node_class, parser, token): + "Returns a template.Node subclass." + bits = token.split_contents()[1:] + return node_class(map(parser.compile_filter, bits)) + +def simple_var_tag(func): + params, xx, xxx, defaults = template.getargspec(func) + + class SimpleNode(Node): + def __init__(self, vars_to_resolve): + self.vars_to_resolve = vars_to_resolve + + def render(self, context): + resolved_vars = [var.resolve(context, True) for var in self.vars_to_resolve] + return func(*resolved_vars) + + compile_func = template.curry(var_tag_compiler, params, defaults, getattr(func, "_decorated_function", func).__name__, SimpleNode) + compile_func.__doc__ = func.__doc__ + register.tag(getattr(func, "_decorated_function", func).__name__, compile_func) + return func + +@simple_var_tag +def static(*parts): + path = ''.join(parts) + urls = settings.STATIC_URLS + size = len(urls) + h = hash(path) % size + if h < 0: + h += size + return urls[h] + '/' + path + +@register.filter(name='random') +def rand(value): + return random.randint(1,value) + +@register.filter(name='concat') +def concat(value, arg): + return str(value) + str(arg) + +@register.filter(name='get') +def doget(value,arg): + return dict(value).get(arg) or '' + +@register.filter(name='range') +def rangefilter(value): + return xrange(int(value)) + +@register.simple_tag +def box(kind): + return '
        ' % (kind,kind,kind,kind,kind) + +@register.simple_tag +def endbox(): + return '
        ' + +@register.simple_tag +def bg(kind): + return '
        ' % (kind,kind,kind,kind) + +@register.simple_tag +def endbg(): + return '
        ' diff --git a/storefront/templatetags/google_analytics.py b/storefront/templatetags/google_analytics.py new file mode 100644 index 0000000..1981b74 --- /dev/null +++ b/storefront/templatetags/google_analytics.py @@ -0,0 +1,86 @@ +""" +Google Analytics template tags and filters. +""" + +from __future__ import absolute_import + +import re + +from django.template import Library, Node, TemplateSyntaxError + +from templatetags.utils import is_internal_ip, disable_html, get_required_setting + +SCOPE_VISITOR = 1 +SCOPE_SESSION = 2 +SCOPE_PAGE = 3 + +PROPERTY_ID_RE = re.compile(r'^UA-\d+-\d+$') +SETUP_CODE = """ + +""" +CUSTOM_VAR_CODE = "_gaq.push(['_setCustomVar', %(index)s, '%(name)s', " \ + "'%(value)s', %(scope)s]);" + + +register = Library() + + +@register.tag +def google_analytics(parser, token): + """ + Google Analytics tracking template tag. + + Renders Javascript code to track page visits. You must supply + your website property ID (as a string) in the + ``GOOGLE_ANALYTICS_PROPERTY_ID`` setting. + """ + bits = token.split_contents() + if len(bits) > 1: + raise TemplateSyntaxError("'%s' takes no arguments" % bits[0]) + return GoogleAnalyticsNode() + +class GoogleAnalyticsNode(Node): + def __init__(self): + self.property_id = get_required_setting( + 'GOOGLE_ANALYTICS_PROPERTY_ID', PROPERTY_ID_RE, + "must be a string looking like 'UA-XXXXXX-Y'") + + def render(self, context): + commands = self._get_custom_var_commands(context) + html = SETUP_CODE % {'property_id': self.property_id, + 'commands': " ".join(commands)} + if is_internal_ip(context, 'GOOGLE_ANALYTICS'): + html = disable_html(html, 'Google Analytics') + return html + + def _get_custom_var_commands(self, context): + values = (context.get('google_analytics_var%s' % i) + for i in range(1, 6)) + vars = [(i, v) for i, v in enumerate(values, 1) if v is not None] + commands = [] + for index, var in vars: + name = var[0] + value = var[1] + try: + scope = var[2] + except IndexError: + scope = SCOPE_PAGE + commands.append(CUSTOM_VAR_CODE % locals()) + return commands + + +def contribute_to_analytical(add_node): + GoogleAnalyticsNode() # ensure properly configured + add_node('head_bottom', GoogleAnalyticsNode) diff --git a/storefront/templatetags/macros.py b/storefront/templatetags/macros.py new file mode 100644 index 0000000..2e827c3 --- /dev/null +++ b/storefront/templatetags/macros.py @@ -0,0 +1,169 @@ +from django import template +from django.template import TemplateSyntaxError + +register = template.Library() + +""" + The MacroRoot node (= %enablemacros% tag) functions quite similar to + the ExtendsNode from django.template.loader_tags. It will capture + everything that follows, and thus should be one of the first tags in + the template. Because %extends% also needs to be the first, if you are + using template inheritance, use %extends_with_macros% instead. + + This whole procedure is necessary because otherwise we would have no + possiblity to access the blocktag referenced by a %repeat% (we could + do it for %macro%, but not for %block%, at least not without patching + the django source). + + So what we do is add a custom attribute to the parser object and store + a reference to the MacroRoot node there, which %repeat% object will + later be able to access when they need to find a block. + + Apart from that, the node doesn't do much, except rendering it's childs. +""" +class MacroRoot(template.Node): + def __init__(self, nodelist=[]): + self.nodelist = nodelist + + def render(self, context): + return self.nodelist.render(context) + + def find(self, block_name, parent_nodelist=None): + # parent_nodelist is internally for recusion, start with root nodelist + if parent_nodelist is None: parent_nodelist = self.nodelist + + from django.template.loader_tags import BlockNode + for node in parent_nodelist: + if isinstance(node, (MacroNode, BlockNode)): + if node.name == block_name: + return node + if hasattr(node, 'nodelist'): + result = self.find(block_name, node.nodelist) + if result: + return result + return None # nothing found + +def do_enablemacros(parser, token): + # check that there are no arguments + bits = token.split_contents() + if len(bits) != 1: + raise TemplateSyntaxError, "'%s' takes no arguments" % bits[0] + # create the Node object now, so we can assign it to the parser + # before we continue with our call to parse(). this enables repeat + # tags that follow later to already enforce at the parsing stage + # that macros are correctly enabled. + parser._macro_root = MacroRoot() + # capture the rest of the template + nodelist = parser.parse() + if nodelist.get_nodes_by_type(MacroRoot): + raise TemplateSyntaxError, "'%s' cannot appear more than once in the same template" % bits[0] + # update the nodelist on the previously created MacroRoot node and + # return it. + parser._macro_root.nodelist = nodelist + return parser._macro_root + +def do_extends_with_macros(parser, token): + from django.template.loader_tags import do_extends + # parse it as an ExtendsNode, but also create a fake MacroRoot node + # and add it to the parser, like we do in do_enablemacros(). + parser._macro_root = MacroRoot() + extendsnode = do_extends(parser, token) + parser._macro_root.nodelist = extendsnode.nodelist + return extendsnode + +""" + %macro% is pretty much exactly like a %block%. Both can be repeated, but + the macro does not output it's content by itself, but *only* if it is + called via a %repeat% tag. +""" + +from django.template.loader_tags import BlockNode, do_block + +class MacroNode(BlockNode): + def render(self, context): + return '' + + # the render that actually works + def repeat(self, context): + return super(MacroNode, self).render(context) + +def do_macro(parser, token): + # let the block parse itself + result = do_block(parser, token) + # "upgrade" the BlockNode to a MacroNode and return it. Yes, I was not + # completely comfortable with it either at first, but Google says it's ok. + result.__class__ = MacroNode + return result + +""" + This (the %repeast%) is the heart of the macro system. It will try to + find the specified %macro% or %block% tag and render it with the most + up-to-date context, including any number of additional parameters passed + to the repeat-tag itself. +""" +class RepeatNode(template.Node): + def __init__(self, block_name, macro_root, extra_context): + self.block_name = block_name + self.macro_root = macro_root + self.extra_context = extra_context + + def render(self, context): + block = self.macro_root.find(self.block_name) + if not block: + # apparently we are not supposed to raise exceptions at rendering + # stage, but this is serious, and we cannot do it while parsing. + # once again, it comes down to being able to support repeating of + # standard blocks. If we would only support our own %macro% tags, + # we would not need the whole %enablemacros% stuff and could do + # things differently. + raise TemplateSyntaxError, "cannot repeat '%s': block or macro not found" % self.block_name + else: + # resolve extra context variables + resolved_context = {} + for key, value in self.extra_context.items(): + resolved_context[key] = value.resolve(context) + # render the block with the new context + context.update(resolved_context) + if isinstance(block, MacroNode): + result = block.repeat(context) + else: + result = block.render(context) + context.pop() + return result + +def do_repeat(parser, token): + # Stolen from django.templatetags.i18n.BlockTranslateParser + # Parses something like "with x as y, i as j", and + # returns it as a context dict. + class RepeatTagParser(template.TokenParser): + def top(self): + extra_context = {} + # first tag is the blockname + try: block_name = self.tag() + except TemplateSyntaxError: + raise TemplateSyntaxError("'%s' requires a block or macro name" % self.tagname) + # read param bindings + while self.more(): + tag = self.tag() + if tag == 'with' or tag == 'and': + value = self.value() + if self.tag() != 'as': + raise TemplateSyntaxError, "variable bindings in %s must be 'with value as variable'" % self.tagname + extra_context[self.tag()] = parser.compile_filter(value) + else: + raise TemplateSyntaxError, "unknown subtag %s for '%s' found" % (tag, self.tagname) + return self.tagname, block_name, extra_context + + # parse arguments + (tag_name, block_name, extra_context) = \ + RepeatTagParser(token.contents).top() + # return as a RepeatNode + if not hasattr(parser, '_macro_root'): + raise TemplateSyntaxError, "'%s' requires macros to be enabled first" % tag_name + return RepeatNode(block_name, parser._macro_root, extra_context) + +# register all our tags +register.tag('repeat', do_repeat) +register.tag('macro', do_macro) +register.tag('enablemacros', do_enablemacros) +register.tag('extends_with_macros', do_extends_with_macros) \ No newline at end of file diff --git a/storefront/templatetags/messages.py b/storefront/templatetags/messages.py new file mode 100644 index 0000000..673f5fc --- /dev/null +++ b/storefront/templatetags/messages.py @@ -0,0 +1,44 @@ +from django import template +register = template.Library() + +class MessagesNode(template.Node): + """ Outputs grouped Django Messages Framework messages in separate + lists sorted by level. """ + + def __init__(self, messages): + self.messages = messages + + def render(self, context): + try: + messages = context[self.messages] + + # Make a dictionary of messages grouped by tag, sorted by level. + grouped = {} + for m in messages: + # Add message + if (m.level, m.tags) in grouped: + grouped[(m.level, m.tags)].append(m.message) + else: + grouped[(m.level, m.tags)] = [m.message] + + # Create a list of messages for each tag. + out_str = '' + for level, tag in sorted(grouped.iterkeys()): + out_str += '
        \n' % tag + for m in grouped[(level, tag)]: + out_str += '
      • %s
        • ' % (m) + out_str += '
      \n
      \n' + + return out_str + + except KeyError: + return '' + +@register.tag(name='render_messages') +def render_messages(parser, token): + parts = token.split_contents() + if len(parts) != 2: + raise template.TemplateSyntaxError("%r tag requires a single argument" + % token.contents.split()[0]) + return MessagesNode(parts[1]) + diff --git a/storefront/templatetags/mixpanel.py b/storefront/templatetags/mixpanel.py new file mode 100644 index 0000000..10cb2d0 --- /dev/null +++ b/storefront/templatetags/mixpanel.py @@ -0,0 +1,87 @@ +""" +Mixpanel template tags and filters. +""" + +from __future__ import absolute_import + +import re + +from django import template +from django.template import Library, Node, TemplateSyntaxError +from django.utils import simplejson + +from templatetags.utils import is_internal_ip, disable_html, get_identity, get_required_setting + + +MIXPANEL_API_TOKEN_RE = re.compile(r'^[0-9a-f]{32}$') +TRACKING_CODE = """ + +""" +IDENTIFY_CODE = "mpq.push(['identify', '%s']); mpq.push(['name_tag', '%s']);" +EVENT_CODE = "mpq.push(['track', '%(name)s', %(properties)s]);" +SUPER_PROPERTY_CODE = "mpq.push(['register', %(super_properties)s, 'all', '14']);" + +#register = Library() +register = template.Library() + + +@register.simple_tag +def mixpanel_log_event(in_script, eventName, properties): + if properties: + code = 'mpq.push(["track", "%s", %s])' % (eventName, properties) + else: + code = 'mpq.push("track", "%s")' % (eventName) + if not in_script: + code = '' + return code + +@register.tag +def mixpanel(parser, token): + """ + Mixpanel tracking template tag. + + Renders Javascript code to track page visits. You must supply + your Mixpanel token in the ``MIXPANEL_API_TOKEN`` setting. + """ + bits = token.split_contents() + if len(bits) > 1: + raise TemplateSyntaxError("'%s' takes no arguments" % bits[0]) + return MixpanelNode() + +class MixpanelNode(Node): + def __init__(self): + self.token = get_required_setting( + 'MIXPANEL_API_TOKEN', MIXPANEL_API_TOKEN_RE, + "must be a string containing a 32-digit hexadecimal number") + + def render(self, context): + commands = [] + data = context.get('mixpanel') + commands.append(SUPER_PROPERTY_CODE % (data)) + for name, properties in data.get('events'): + print 'name=%s, prop=%s' % (name, properties) + commands.append(EVENT_CODE % {'name': name, 'properties': simplejson.dumps(properties)}) + + identity = get_identity(context, 'mixpanel') + if identity is not None: + commands.append(IDENTIFY_CODE % (identity, identity)) + + + html = TRACKING_CODE % {'token': self.token, 'commands': " ".join(commands)} + if is_internal_ip(context, 'MIXPANEL'): + html = disable_html(html, 'Mixpanel') + return html + + +def contribute_to_analytical(add_node): + MixpanelNode() # ensure properly configured + add_node('head_bottom', MixpanelNode) diff --git a/storefront/templatetags/utils-analytical.py b/storefront/templatetags/utils-analytical.py new file mode 100644 index 0000000..202b699 --- /dev/null +++ b/storefront/templatetags/utils-analytical.py @@ -0,0 +1,124 @@ +""" +Utility function for django-analytical. +""" + +from django.conf import settings + + +HTML_COMMENT = "" + + +def get_required_setting(setting, value_re, invalid_msg): + """ + Return a constant from ``django.conf.settings``. The `setting` + argument is the constant name, the `value_re` argument is a regular + expression used to validate the setting value and the `invalid_msg` + argument is used as exception message if the value is not valid. + """ + try: + value = getattr(settings, setting) + except AttributeError: + raise AnalyticalException("%s setting: not found" % setting) + value = str(value) + if not value_re.search(value): + raise AnalyticalException("%s setting: %s: '%s'" + % (setting, invalid_msg, value)) + return value + + +def get_user_from_context(context): + """ + Get the user instance from the template context, if possible. + + If the context does not contain a `request` or `user` attribute, + `None` is returned. + """ + try: + return context['user'] + except KeyError: + pass + try: + request = context['request'] + return request.user + except (KeyError, AttributeError): + pass + return None + + +def get_identity(context, prefix=None, identity_func=None, user=None): + """ + Get the identity of a logged in user from a template context. + + The `prefix` argument is used to provide different identities to + different analytics services. The `identity_func` argument is a + function that returns the identity of the user; by default the + identity is the username. + """ + if prefix is not None: + try: + return context['%s_identity' % prefix] + except KeyError: + pass + try: + return context['analytical_identity'] + except KeyError: + pass + if getattr(settings, 'ANALYTICAL_AUTO_IDENTIFY', True): + try: + if user is None: + user = get_user_from_context(context) + if user.is_authenticated(): + if identity_func is not None: + return identity_func(user) + else: + return user.username + except (KeyError, AttributeError): + pass + return None + + +def is_internal_ip(context, prefix=None): + """ + Return whether the visitor is coming from an internal IP address, + based on information from the template context. + + The prefix is used to allow different analytics services to have + different notions of internal addresses. + """ + try: + request = context['request'] + remote_ip = request.META.get('HTTP_X_FORWARDED_FOR', '') + if not remote_ip: + remote_ip = request.META.get('REMOTE_ADDR', '') + if not remote_ip: + return False + + internal_ips = '' + if prefix is not None: + internal_ips = getattr(settings, '%s_INTERNAL_IPS' % prefix, '') + if not internal_ips: + internal_ips = getattr(settings, 'ANALYTICAL_INTERNAL_IPS', '') + if not internal_ips: + internal_ips = getattr(settings, 'INTERNAL_IPS', '') + + return remote_ip in internal_ips + except (KeyError, AttributeError): + return False + + +def disable_html(html, service): + """ + Disable HTML code by commenting it out. + + The `service` argument is used to display a friendly message. + """ + return HTML_COMMENT % {'html': html, 'service': service} + + +class AnalyticalException(Exception): + """ + Raised when an exception occurs in any django-analytical code that should + be silenced in templates. + """ + silent_variable_failure = True diff --git a/storefront/templatetags/utils.py b/storefront/templatetags/utils.py new file mode 100644 index 0000000..202b699 --- /dev/null +++ b/storefront/templatetags/utils.py @@ -0,0 +1,124 @@ +""" +Utility function for django-analytical. +""" + +from django.conf import settings + + +HTML_COMMENT = "" + + +def get_required_setting(setting, value_re, invalid_msg): + """ + Return a constant from ``django.conf.settings``. The `setting` + argument is the constant name, the `value_re` argument is a regular + expression used to validate the setting value and the `invalid_msg` + argument is used as exception message if the value is not valid. + """ + try: + value = getattr(settings, setting) + except AttributeError: + raise AnalyticalException("%s setting: not found" % setting) + value = str(value) + if not value_re.search(value): + raise AnalyticalException("%s setting: %s: '%s'" + % (setting, invalid_msg, value)) + return value + + +def get_user_from_context(context): + """ + Get the user instance from the template context, if possible. + + If the context does not contain a `request` or `user` attribute, + `None` is returned. + """ + try: + return context['user'] + except KeyError: + pass + try: + request = context['request'] + return request.user + except (KeyError, AttributeError): + pass + return None + + +def get_identity(context, prefix=None, identity_func=None, user=None): + """ + Get the identity of a logged in user from a template context. + + The `prefix` argument is used to provide different identities to + different analytics services. The `identity_func` argument is a + function that returns the identity of the user; by default the + identity is the username. + """ + if prefix is not None: + try: + return context['%s_identity' % prefix] + except KeyError: + pass + try: + return context['analytical_identity'] + except KeyError: + pass + if getattr(settings, 'ANALYTICAL_AUTO_IDENTIFY', True): + try: + if user is None: + user = get_user_from_context(context) + if user.is_authenticated(): + if identity_func is not None: + return identity_func(user) + else: + return user.username + except (KeyError, AttributeError): + pass + return None + + +def is_internal_ip(context, prefix=None): + """ + Return whether the visitor is coming from an internal IP address, + based on information from the template context. + + The prefix is used to allow different analytics services to have + different notions of internal addresses. + """ + try: + request = context['request'] + remote_ip = request.META.get('HTTP_X_FORWARDED_FOR', '') + if not remote_ip: + remote_ip = request.META.get('REMOTE_ADDR', '') + if not remote_ip: + return False + + internal_ips = '' + if prefix is not None: + internal_ips = getattr(settings, '%s_INTERNAL_IPS' % prefix, '') + if not internal_ips: + internal_ips = getattr(settings, 'ANALYTICAL_INTERNAL_IPS', '') + if not internal_ips: + internal_ips = getattr(settings, 'INTERNAL_IPS', '') + + return remote_ip in internal_ips + except (KeyError, AttributeError): + return False + + +def disable_html(html, service): + """ + Disable HTML code by commenting it out. + + The `service` argument is used to display a friendly message. + """ + return HTML_COMMENT % {'html': html, 'service': service} + + +class AnalyticalException(Exception): + """ + Raised when an exception occurs in any django-analytical code that should + be silenced in templates. + """ + silent_variable_failure = True diff --git a/storefront/thrift/TSCons.py b/storefront/thrift/TSCons.py new file mode 100644 index 0000000..2404625 --- /dev/null +++ b/storefront/thrift/TSCons.py @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from os import path +from SCons.Builder import Builder + +def scons_env(env, add=''): + opath = path.dirname(path.abspath('$TARGET')) + lstr = 'thrift --gen cpp -o ' + opath + ' ' + add + ' $SOURCE' + cppbuild = Builder(action = lstr) + env.Append(BUILDERS = {'ThriftCpp' : cppbuild}) + +def gen_cpp(env, dir, file): + scons_env(env) + suffixes = ['_types.h', '_types.cpp'] + targets = map(lambda s: 'gen-cpp/' + file + s, suffixes) + return env.ThriftCpp(targets, dir+file+'.thrift') diff --git a/storefront/thrift/TSerialization.py b/storefront/thrift/TSerialization.py new file mode 100644 index 0000000..b19f98a --- /dev/null +++ b/storefront/thrift/TSerialization.py @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from protocol import TBinaryProtocol +from transport import TTransport + +def serialize(thrift_object, protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()): + transport = TTransport.TMemoryBuffer() + protocol = protocol_factory.getProtocol(transport) + thrift_object.write(protocol) + return transport.getvalue() + +def deserialize(base, buf, protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()): + transport = TTransport.TMemoryBuffer(buf) + protocol = protocol_factory.getProtocol(transport) + base.read(protocol) + return base + diff --git a/storefront/thrift/Thrift.py b/storefront/thrift/Thrift.py new file mode 100644 index 0000000..91728a7 --- /dev/null +++ b/storefront/thrift/Thrift.py @@ -0,0 +1,133 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys + +class TType: + STOP = 0 + VOID = 1 + BOOL = 2 + BYTE = 3 + I08 = 3 + DOUBLE = 4 + I16 = 6 + I32 = 8 + I64 = 10 + STRING = 11 + UTF7 = 11 + STRUCT = 12 + MAP = 13 + SET = 14 + LIST = 15 + UTF8 = 16 + UTF16 = 17 + +class TMessageType: + CALL = 1 + REPLY = 2 + EXCEPTION = 3 + ONEWAY = 4 + +class TProcessor: + + """Base class for procsessor, which works on two streams.""" + + def process(iprot, oprot): + pass + +class TException(Exception): + + """Base class for all thrift exceptions.""" + + # BaseException.message is deprecated in Python v[2.6,3.0) + if (2,6,0) <= sys.version_info < (3,0): + def _get_message(self): + return self._message + def _set_message(self, message): + self._message = message + message = property(_get_message, _set_message) + + def __init__(self, message=None): + Exception.__init__(self, message) + self.message = message + +class TApplicationException(TException): + + """Application level thrift exceptions.""" + + UNKNOWN = 0 + UNKNOWN_METHOD = 1 + INVALID_MESSAGE_TYPE = 2 + WRONG_METHOD_NAME = 3 + BAD_SEQUENCE_ID = 4 + MISSING_RESULT = 5 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + + def __str__(self): + if self.message: + return self.message + elif self.type == self.UNKNOWN_METHOD: + return 'Unknown method' + elif self.type == self.INVALID_MESSAGE_TYPE: + return 'Invalid message type' + elif self.type == self.WRONG_METHOD_NAME: + return 'Wrong method name' + elif self.type == self.BAD_SEQUENCE_ID: + return 'Bad sequence ID' + elif self.type == self.MISSING_RESULT: + return 'Missing result' + else: + return 'Default (unknown) TApplicationException' + + def read(self, iprot): + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.message = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.type = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + oprot.writeStructBegin('TApplicationException') + if self.message != None: + oprot.writeFieldBegin('message', TType.STRING, 1) + oprot.writeString(self.message) + oprot.writeFieldEnd() + if self.type != None: + oprot.writeFieldBegin('type', TType.I32, 2) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() diff --git a/storefront/thrift/__init__.py b/storefront/thrift/__init__.py new file mode 100644 index 0000000..48d659c --- /dev/null +++ b/storefront/thrift/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['Thrift', 'TSCons'] diff --git a/storefront/thrift/protocol/TBinaryProtocol.py b/storefront/thrift/protocol/TBinaryProtocol.py new file mode 100644 index 0000000..50c6aa8 --- /dev/null +++ b/storefront/thrift/protocol/TBinaryProtocol.py @@ -0,0 +1,259 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TProtocol import * +from struct import pack, unpack + +class TBinaryProtocol(TProtocolBase): + + """Binary implementation of the Thrift protocol driver.""" + + # NastyHaxx. Python 2.4+ on 32-bit machines forces hex constants to be + # positive, converting this into a long. If we hardcode the int value + # instead it'll stay in 32 bit-land. + + # VERSION_MASK = 0xffff0000 + VERSION_MASK = -65536 + + # VERSION_1 = 0x80010000 + VERSION_1 = -2147418112 + + TYPE_MASK = 0x000000ff + + def __init__(self, trans, strictRead=False, strictWrite=True): + TProtocolBase.__init__(self, trans) + self.strictRead = strictRead + self.strictWrite = strictWrite + + def writeMessageBegin(self, name, type, seqid): + if self.strictWrite: + self.writeI32(TBinaryProtocol.VERSION_1 | type) + self.writeString(name) + self.writeI32(seqid) + else: + self.writeString(name) + self.writeByte(type) + self.writeI32(seqid) + + def writeMessageEnd(self): + pass + + def writeStructBegin(self, name): + pass + + def writeStructEnd(self): + pass + + def writeFieldBegin(self, name, type, id): + self.writeByte(type) + self.writeI16(id) + + def writeFieldEnd(self): + pass + + def writeFieldStop(self): + self.writeByte(TType.STOP); + + def writeMapBegin(self, ktype, vtype, size): + self.writeByte(ktype) + self.writeByte(vtype) + self.writeI32(size) + + def writeMapEnd(self): + pass + + def writeListBegin(self, etype, size): + self.writeByte(etype) + self.writeI32(size) + + def writeListEnd(self): + pass + + def writeSetBegin(self, etype, size): + self.writeByte(etype) + self.writeI32(size) + + def writeSetEnd(self): + pass + + def writeBool(self, bool): + if bool: + self.writeByte(1) + else: + self.writeByte(0) + + def writeByte(self, byte): + buff = pack("!b", byte) + self.trans.write(buff) + + def writeI16(self, i16): + buff = pack("!h", i16) + self.trans.write(buff) + + def writeI32(self, i32): + buff = pack("!i", i32) + self.trans.write(buff) + + def writeI64(self, i64): + buff = pack("!q", i64) + self.trans.write(buff) + + def writeDouble(self, dub): + buff = pack("!d", dub) + self.trans.write(buff) + + def writeString(self, str): + self.writeI32(len(str)) + self.trans.write(str) + + def readMessageBegin(self): + sz = self.readI32() + if sz < 0: + version = sz & TBinaryProtocol.VERSION_MASK + if version != TBinaryProtocol.VERSION_1: + raise TProtocolException(type=TProtocolException.BAD_VERSION, message='Bad version in readMessageBegin: %d' % (sz)) + type = sz & TBinaryProtocol.TYPE_MASK + name = self.readString() + seqid = self.readI32() + else: + if self.strictRead: + raise TProtocolException(type=TProtocolException.BAD_VERSION, message='No protocol version header') + name = self.trans.readAll(sz) + type = self.readByte() + seqid = self.readI32() + return (name, type, seqid) + + def readMessageEnd(self): + pass + + def readStructBegin(self): + pass + + def readStructEnd(self): + pass + + def readFieldBegin(self): + type = self.readByte() + if type == TType.STOP: + return (None, type, 0) + id = self.readI16() + return (None, type, id) + + def readFieldEnd(self): + pass + + def readMapBegin(self): + ktype = self.readByte() + vtype = self.readByte() + size = self.readI32() + return (ktype, vtype, size) + + def readMapEnd(self): + pass + + def readListBegin(self): + etype = self.readByte() + size = self.readI32() + return (etype, size) + + def readListEnd(self): + pass + + def readSetBegin(self): + etype = self.readByte() + size = self.readI32() + return (etype, size) + + def readSetEnd(self): + pass + + def readBool(self): + byte = self.readByte() + if byte == 0: + return False + return True + + def readByte(self): + buff = self.trans.readAll(1) + val, = unpack('!b', buff) + return val + + def readI16(self): + buff = self.trans.readAll(2) + val, = unpack('!h', buff) + return val + + def readI32(self): + buff = self.trans.readAll(4) + val, = unpack('!i', buff) + return val + + def readI64(self): + buff = self.trans.readAll(8) + val, = unpack('!q', buff) + return val + + def readDouble(self): + buff = self.trans.readAll(8) + val, = unpack('!d', buff) + return val + + def readString(self): + len = self.readI32() + str = self.trans.readAll(len) + return str + + +class TBinaryProtocolFactory: + def __init__(self, strictRead=False, strictWrite=True): + self.strictRead = strictRead + self.strictWrite = strictWrite + + def getProtocol(self, trans): + prot = TBinaryProtocol(trans, self.strictRead, self.strictWrite) + return prot + + +class TBinaryProtocolAccelerated(TBinaryProtocol): + + """C-Accelerated version of TBinaryProtocol. + + This class does not override any of TBinaryProtocol's methods, + but the generated code recognizes it directly and will call into + our C module to do the encoding, bypassing this object entirely. + We inherit from TBinaryProtocol so that the normal TBinaryProtocol + encoding can happen if the fastbinary module doesn't work for some + reason. (TODO(dreiss): Make this happen sanely in more cases.) + + In order to take advantage of the C module, just use + TBinaryProtocolAccelerated instead of TBinaryProtocol. + + NOTE: This code was contributed by an external developer. + The internal Thrift team has reviewed and tested it, + but we cannot guarantee that it is production-ready. + Please feel free to report bugs and/or success stories + to the public mailing list. + """ + + pass + + +class TBinaryProtocolAcceleratedFactory: + def getProtocol(self, trans): + return TBinaryProtocolAccelerated(trans) diff --git a/storefront/thrift/protocol/TCompactProtocol.py b/storefront/thrift/protocol/TCompactProtocol.py new file mode 100644 index 0000000..fbc156a --- /dev/null +++ b/storefront/thrift/protocol/TCompactProtocol.py @@ -0,0 +1,368 @@ +from TProtocol import * +from struct import pack, unpack + +__all__ = ['TCompactProtocol', 'TCompactProtocolFactory'] + +CLEAR = 0 +FIELD_WRITE = 1 +VALUE_WRITE = 2 +CONTAINER_WRITE = 3 +BOOL_WRITE = 4 +FIELD_READ = 5 +CONTAINER_READ = 6 +VALUE_READ = 7 +BOOL_READ = 8 + +def make_helper(v_from, container): + def helper(func): + def nested(self, *args, **kwargs): + assert self.state in (v_from, container), (self.state, v_from, container) + return func(self, *args, **kwargs) + return nested + return helper +writer = make_helper(VALUE_WRITE, CONTAINER_WRITE) +reader = make_helper(VALUE_READ, CONTAINER_READ) + +def makeZigZag(n, bits): + return (n << 1) ^ (n >> (bits - 1)) + +def fromZigZag(n): + return (n >> 1) ^ -(n & 1) + +def writeVarint(trans, n): + out = [] + while True: + if n & ~0x7f == 0: + out.append(n) + break + else: + out.append((n & 0xff) | 0x80) + n = n >> 7 + trans.write(''.join(map(chr, out))) + +def readVarint(trans): + result = 0 + shift = 0 + while True: + x = trans.readAll(1) + byte = ord(x) + result |= (byte & 0x7f) << shift + if byte >> 7 == 0: + return result + shift += 7 + +class CompactType: + TRUE = 1 + FALSE = 2 + BYTE = 0x03 + I16 = 0x04 + I32 = 0x05 + I64 = 0x06 + DOUBLE = 0x07 + BINARY = 0x08 + LIST = 0x09 + SET = 0x0A + MAP = 0x0B + STRUCT = 0x0C + +CTYPES = {TType.BOOL: CompactType.TRUE, # used for collection + TType.BYTE: CompactType.BYTE, + TType.I16: CompactType.I16, + TType.I32: CompactType.I32, + TType.I64: CompactType.I64, + TType.DOUBLE: CompactType.DOUBLE, + TType.STRING: CompactType.BINARY, + TType.STRUCT: CompactType.STRUCT, + TType.LIST: CompactType.LIST, + TType.SET: CompactType.SET, + TType.MAP: CompactType.MAP, + } + +TTYPES = {} +for k, v in CTYPES.items(): + TTYPES[v] = k +TTYPES[CompactType.FALSE] = TType.BOOL +del k +del v + +class TCompactProtocol(TProtocolBase): + "Compact implementation of the Thrift protocol driver." + + PROTOCOL_ID = 0x82 + VERSION = 1 + VERSION_MASK = 0x1f + TYPE_MASK = 0xe0 + TYPE_SHIFT_AMOUNT = 5 + + def __init__(self, trans): + TProtocolBase.__init__(self, trans) + self.state = CLEAR + self.__last_fid = 0 + self.__bool_fid = None + self.__bool_value = None + self.__structs = [] + self.__containers = [] + + def __writeVarint(self, n): + writeVarint(self.trans, n) + + def writeMessageBegin(self, name, type, seqid): + assert self.state == CLEAR + self.__writeUByte(self.PROTOCOL_ID) + self.__writeUByte(self.VERSION | (type << self.TYPE_SHIFT_AMOUNT)) + self.__writeVarint(seqid) + self.__writeString(name) + self.state = VALUE_WRITE + + def writeMessageEnd(self): + assert self.state == VALUE_WRITE + self.state = CLEAR + + def writeStructBegin(self, name): + assert self.state in (CLEAR, CONTAINER_WRITE, VALUE_WRITE), self.state + self.__structs.append((self.state, self.__last_fid)) + self.state = FIELD_WRITE + self.__last_fid = 0 + + def writeStructEnd(self): + assert self.state == FIELD_WRITE + self.state, self.__last_fid = self.__structs.pop() + + def writeFieldStop(self): + self.__writeByte(0) + + def __writeFieldHeader(self, type, fid): + delta = fid - self.__last_fid + if 0 < delta <= 15: + self.__writeUByte(delta << 4 | type) + else: + self.__writeByte(type) + self.__writeI16(fid) + self.__last_fid = fid + + def writeFieldBegin(self, name, type, fid): + assert self.state == FIELD_WRITE, self.state + if type == TType.BOOL: + self.state = BOOL_WRITE + self.__bool_fid = fid + else: + self.state = VALUE_WRITE + self.__writeFieldHeader(CTYPES[type], fid) + + def writeFieldEnd(self): + assert self.state in (VALUE_WRITE, BOOL_WRITE), self.state + self.state = FIELD_WRITE + + def __writeUByte(self, byte): + self.trans.write(pack('!B', byte)) + + def __writeByte(self, byte): + self.trans.write(pack('!b', byte)) + + def __writeI16(self, i16): + self.__writeVarint(makeZigZag(i16, 16)) + + def __writeSize(self, i32): + self.__writeVarint(i32) + + def writeCollectionBegin(self, etype, size): + assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state + if size <= 14: + self.__writeUByte(size << 4 | CTYPES[etype]) + else: + self.__writeUByte(0xf0 | CTYPES[etype]) + self.__writeSize(size) + self.__containers.append(self.state) + self.state = CONTAINER_WRITE + writeSetBegin = writeCollectionBegin + writeListBegin = writeCollectionBegin + + def writeMapBegin(self, ktype, vtype, size): + assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state + if size == 0: + self.__writeByte(0) + else: + self.__writeSize(size) + self.__writeUByte(CTYPES[ktype] << 4 | CTYPES[vtype]) + self.__containers.append(self.state) + self.state = CONTAINER_WRITE + + def writeCollectionEnd(self): + assert self.state == CONTAINER_WRITE, self.state + self.state = self.__containers.pop() + writeMapEnd = writeCollectionEnd + writeSetEnd = writeCollectionEnd + writeListEnd = writeCollectionEnd + + def writeBool(self, bool): + if self.state == BOOL_WRITE: + self.__writeFieldHeader(types[bool], self.__bool_fid) + elif self.state == CONTAINER_WRITE: + self.__writeByte(int(bool)) + else: + raise AssertetionError, "Invalid state in compact protocol" + + writeByte = writer(__writeByte) + writeI16 = writer(__writeI16) + + @writer + def writeI32(self, i32): + self.__writeVarint(makeZigZag(i32, 32)) + + @writer + def writeI64(self, i64): + self.__writeVarint(makeZigZag(i64, 64)) + + @writer + def writeDouble(self, dub): + self.trans.write(pack('!d', dub)) + + def __writeString(self, s): + self.__writeSize(len(s)) + self.trans.write(s) + writeString = writer(__writeString) + + def readFieldBegin(self): + assert self.state == FIELD_READ, self.state + type = self.__readUByte() + if type & 0x0f == TType.STOP: + return (None, 0, 0) + delta = type >> 4 + if delta == 0: + fid = self.__readI16() + else: + fid = self.__last_fid + delta + self.__last_fid = fid + type = type & 0x0f + if type == CompactType.TRUE: + self.state = BOOL_READ + self.__bool_value = True + elif type == CompactType.FALSE: + self.state = BOOL_READ + self.__bool_value = False + else: + self.state = VALUE_READ + return (None, self.__getTType(type), fid) + + def readFieldEnd(self): + assert self.state in (VALUE_READ, BOOL_READ), self.state + self.state = FIELD_READ + + def __readUByte(self): + result, = unpack('!B', self.trans.readAll(1)) + return result + + def __readByte(self): + result, = unpack('!b', self.trans.readAll(1)) + return result + + def __readVarint(self): + return readVarint(self.trans) + + def __readZigZag(self): + return fromZigZag(self.__readVarint()) + + def __readSize(self): + result = self.__readVarint() + if result < 0: + raise TException("Length < 0") + return result + + def readMessageBegin(self): + assert self.state == CLEAR + proto_id = self.__readUByte() + if proto_id != self.PROTOCOL_ID: + raise TProtocolException(TProtocolException.BAD_VERSION, + 'Bad protocol id in the message: %d' % proto_id) + ver_type = self.__readUByte() + type = (ver_type & self.TYPE_MASK) >> self.TYPE_SHIFT_AMOUNT + version = ver_type & self.VERSION_MASK + if version != self.VERSION: + raise TProtocolException(TProtocolException.BAD_VERSION, + 'Bad version: %d (expect %d)' % (version, self.VERSION)) + seqid = self.__readVarint() + name = self.__readString() + return (name, type, seqid) + + def readMessageEnd(self): + assert self.state == VALUE_READ + assert len(self.__structs) == 0 + self.state = CLEAR + + def readStructBegin(self): + assert self.state in (CLEAR, CONTAINER_READ, VALUE_READ), self.state + self.__structs.append((self.state, self.__last_fid)) + self.state = FIELD_READ + self.__last_fid = 0 + + def readStructEnd(self): + assert self.state == FIELD_READ + self.state, self.__last_fid = self.__structs.pop() + + def readCollectionBegin(self): + assert self.state in (VALUE_READ, CONTAINER_READ), self.state + size_type = self.__readUByte() + size = size_type >> 4 + type = self.__getTType(size_type) + if size == 15: + size = self.__readSize() + self.__containers.append(self.state) + self.state = CONTAINER_READ + return type, size + readSetBegin = readCollectionBegin + readListBegin = readCollectionBegin + + def readMapBegin(self): + assert self.state in (VALUE_READ, CONTAINER_READ), self.state + size = self.__readSize() + types = 0 + if size > 0: + types = self.__readUByte() + vtype = self.__getTType(types) + ktype = self.__getTType(types >> 4) + self.__containers.append(self.state) + self.state = CONTAINER_READ + return (ktype, vtype, size) + + def readCollectionEnd(self): + assert self.state == CONTAINER_READ, self.state + self.state = self.__containers.pop() + readSetEnd = readCollectionEnd + readListEnd = readCollectionEnd + readMapEnd = readCollectionEnd + + def readBool(self): + if self.state == BOOL_READ: + return self.__bool_value + elif self.state == CONTAINER_READ: + return bool(self.__readByte()) + else: + raise AssertionError, "Invalid state in compact protocol: %d" % self.state + + readByte = reader(__readByte) + __readI16 = __readZigZag + readI16 = reader(__readZigZag) + readI32 = reader(__readZigZag) + readI64 = reader(__readZigZag) + + @reader + def readDouble(self): + buff = self.trans.readAll(8) + val, = unpack('!d', buff) + return val + + def __readString(self): + len = self.__readSize() + return self.trans.readAll(len) + readString = reader(__readString) + + def __getTType(self, byte): + return TTYPES[byte & 0x0f] + + +class TCompactProtocolFactory: + def __init__(self): + pass + + def getProtocol(self, trans): + return TCompactProtocol(trans) diff --git a/storefront/thrift/protocol/TProtocol.py b/storefront/thrift/protocol/TProtocol.py new file mode 100644 index 0000000..be3cb14 --- /dev/null +++ b/storefront/thrift/protocol/TProtocol.py @@ -0,0 +1,205 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from thrift.Thrift import * + +class TProtocolException(TException): + + """Custom Protocol Exception class""" + + UNKNOWN = 0 + INVALID_DATA = 1 + NEGATIVE_SIZE = 2 + SIZE_LIMIT = 3 + BAD_VERSION = 4 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + +class TProtocolBase: + + """Base class for Thrift protocol driver.""" + + def __init__(self, trans): + self.trans = trans + + def writeMessageBegin(self, name, type, seqid): + pass + + def writeMessageEnd(self): + pass + + def writeStructBegin(self, name): + pass + + def writeStructEnd(self): + pass + + def writeFieldBegin(self, name, type, id): + pass + + def writeFieldEnd(self): + pass + + def writeFieldStop(self): + pass + + def writeMapBegin(self, ktype, vtype, size): + pass + + def writeMapEnd(self): + pass + + def writeListBegin(self, etype, size): + pass + + def writeListEnd(self): + pass + + def writeSetBegin(self, etype, size): + pass + + def writeSetEnd(self): + pass + + def writeBool(self, bool): + pass + + def writeByte(self, byte): + pass + + def writeI16(self, i16): + pass + + def writeI32(self, i32): + pass + + def writeI64(self, i64): + pass + + def writeDouble(self, dub): + pass + + def writeString(self, str): + pass + + def readMessageBegin(self): + pass + + def readMessageEnd(self): + pass + + def readStructBegin(self): + pass + + def readStructEnd(self): + pass + + def readFieldBegin(self): + pass + + def readFieldEnd(self): + pass + + def readMapBegin(self): + pass + + def readMapEnd(self): + pass + + def readListBegin(self): + pass + + def readListEnd(self): + pass + + def readSetBegin(self): + pass + + def readSetEnd(self): + pass + + def readBool(self): + pass + + def readByte(self): + pass + + def readI16(self): + pass + + def readI32(self): + pass + + def readI64(self): + pass + + def readDouble(self): + pass + + def readString(self): + pass + + def skip(self, type): + if type == TType.STOP: + return + elif type == TType.BOOL: + self.readBool() + elif type == TType.BYTE: + self.readByte() + elif type == TType.I16: + self.readI16() + elif type == TType.I32: + self.readI32() + elif type == TType.I64: + self.readI64() + elif type == TType.DOUBLE: + self.readDouble() + elif type == TType.STRING: + self.readString() + elif type == TType.STRUCT: + name = self.readStructBegin() + while True: + (name, type, id) = self.readFieldBegin() + if type == TType.STOP: + break + self.skip(type) + self.readFieldEnd() + self.readStructEnd() + elif type == TType.MAP: + (ktype, vtype, size) = self.readMapBegin() + for i in range(size): + self.skip(ktype) + self.skip(vtype) + self.readMapEnd() + elif type == TType.SET: + (etype, size) = self.readSetBegin() + for i in range(size): + self.skip(etype) + self.readSetEnd() + elif type == TType.LIST: + (etype, size) = self.readListBegin() + for i in range(size): + self.skip(etype) + self.readListEnd() + +class TProtocolFactory: + def getProtocol(self, trans): + pass diff --git a/storefront/thrift/protocol/__init__.py b/storefront/thrift/protocol/__init__.py new file mode 100644 index 0000000..01bfe18 --- /dev/null +++ b/storefront/thrift/protocol/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TProtocol', 'TBinaryProtocol', 'fastbinary'] diff --git a/storefront/thrift/protocol/fastbinary.c b/storefront/thrift/protocol/fastbinary.c new file mode 100644 index 0000000..67b215a --- /dev/null +++ b/storefront/thrift/protocol/fastbinary.c @@ -0,0 +1,1203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include "cStringIO.h" +#include +#include +#include + +/* Fix endianness issues on Solaris */ +#if defined (__SVR4) && defined (__sun) + #if defined(__i386) && !defined(__i386__) + #define __i386__ + #endif + + #ifndef BIG_ENDIAN + #define BIG_ENDIAN (4321) + #endif + #ifndef LITTLE_ENDIAN + #define LITTLE_ENDIAN (1234) + #endif + + /* I386 is LE, even on Solaris */ + #if !defined(BYTE_ORDER) && defined(__i386__) + #define BYTE_ORDER LITTLE_ENDIAN + #endif +#endif + +// TODO(dreiss): defval appears to be unused. Look into removing it. +// TODO(dreiss): Make parse_spec_args recursive, and cache the output +// permanently in the object. (Malloc and orphan.) +// TODO(dreiss): Why do we need cStringIO for reading, why not just char*? +// Can cStringIO let us work with a BufferedTransport? +// TODO(dreiss): Don't ignore the rv from cwrite (maybe). + +/* ====== BEGIN UTILITIES ====== */ + +#define INIT_OUTBUF_SIZE 128 + +// Stolen out of TProtocol.h. +// It would be a huge pain to have both get this from one place. +typedef enum TType { + T_STOP = 0, + T_VOID = 1, + T_BOOL = 2, + T_BYTE = 3, + T_I08 = 3, + T_I16 = 6, + T_I32 = 8, + T_U64 = 9, + T_I64 = 10, + T_DOUBLE = 4, + T_STRING = 11, + T_UTF7 = 11, + T_STRUCT = 12, + T_MAP = 13, + T_SET = 14, + T_LIST = 15, + T_UTF8 = 16, + T_UTF16 = 17 +} TType; + +#ifndef __BYTE_ORDER +# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN) +# define __BYTE_ORDER BYTE_ORDER +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __BIG_ENDIAN BIG_ENDIAN +# else +# error "Cannot determine endianness" +# endif +#endif + +// Same comment as the enum. Sorry. +#if __BYTE_ORDER == __BIG_ENDIAN +# define ntohll(n) (n) +# define htonll(n) (n) +#elif __BYTE_ORDER == __LITTLE_ENDIAN +# if defined(__GNUC__) && defined(__GLIBC__) +# include +# define ntohll(n) bswap_64(n) +# define htonll(n) bswap_64(n) +# else /* GNUC & GLIBC */ +# define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) ) +# define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) ) +# endif /* GNUC & GLIBC */ +#else /* __BYTE_ORDER */ +# error "Can't define htonll or ntohll!" +#endif + +// Doing a benchmark shows that interning actually makes a difference, amazingly. +#define INTERN_STRING(value) _intern_ ## value + +#define INT_CONV_ERROR_OCCURRED(v) ( ((v) == -1) && PyErr_Occurred() ) +#define CHECK_RANGE(v, min, max) ( ((v) <= (max)) && ((v) >= (min)) ) + +// Py_ssize_t was not defined before Python 2.5 +#if (PY_VERSION_HEX < 0x02050000) +typedef int Py_ssize_t; +#endif + +/** + * A cache of the spec_args for a set or list, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + TType element_type; + PyObject* typeargs; +} SetListTypeArgs; + +/** + * A cache of the spec_args for a map, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + TType ktag; + TType vtag; + PyObject* ktypeargs; + PyObject* vtypeargs; +} MapTypeArgs; + +/** + * A cache of the spec_args for a struct, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + PyObject* klass; + PyObject* spec; +} StructTypeArgs; + +/** + * A cache of the item spec from a struct specification, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + int tag; + TType type; + PyObject* attrname; + PyObject* typeargs; + PyObject* defval; +} StructItemSpec; + +/** + * A cache of the two key attributes of a CReadableTransport, + * so we don't have to keep calling PyObject_GetAttr. + */ +typedef struct { + PyObject* stringiobuf; + PyObject* refill_callable; +} DecodeBuffer; + +/** Pointer to interned string to speed up attribute lookup. */ +static PyObject* INTERN_STRING(cstringio_buf); +/** Pointer to interned string to speed up attribute lookup. */ +static PyObject* INTERN_STRING(cstringio_refill); + +static inline bool +check_ssize_t_32(Py_ssize_t len) { + // error from getting the int + if (INT_CONV_ERROR_OCCURRED(len)) { + return false; + } + if (!CHECK_RANGE(len, 0, INT32_MAX)) { + PyErr_SetString(PyExc_OverflowError, "string size out of range"); + return false; + } + return true; +} + +static inline bool +parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) { + long val = PyInt_AsLong(o); + + if (INT_CONV_ERROR_OCCURRED(val)) { + return false; + } + if (!CHECK_RANGE(val, min, max)) { + PyErr_SetString(PyExc_OverflowError, "int out of range"); + return false; + } + + *ret = (int32_t) val; + return true; +} + + +/* --- FUNCTIONS TO PARSE STRUCT SPECIFICATOINS --- */ + +static bool +parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 2) { + PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args"); + return false; + } + + dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->element_type)) { + return false; + } + + dest->typeargs = PyTuple_GET_ITEM(typeargs, 1); + + return true; +} + +static bool +parse_map_args(MapTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 4) { + PyErr_SetString(PyExc_TypeError, "expecting 4 arguments for typeargs to map"); + return false; + } + + dest->ktag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->ktag)) { + return false; + } + + dest->vtag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2)); + if (INT_CONV_ERROR_OCCURRED(dest->vtag)) { + return false; + } + + dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1); + dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3); + + return true; +} + +static bool +parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 2) { + PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for struct args"); + return false; + } + + dest->klass = PyTuple_GET_ITEM(typeargs, 0); + dest->spec = PyTuple_GET_ITEM(typeargs, 1); + + return true; +} + +static int +parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) { + + // i'd like to use ParseArgs here, but it seems to be a bottleneck. + if (PyTuple_Size(spec_tuple) != 5) { + PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for spec tuple"); + return false; + } + + dest->tag = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->tag)) { + return false; + } + + dest->type = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1)); + if (INT_CONV_ERROR_OCCURRED(dest->type)) { + return false; + } + + dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2); + dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3); + dest->defval = PyTuple_GET_ITEM(spec_tuple, 4); + return true; +} + +/* ====== END UTILITIES ====== */ + + +/* ====== BEGIN WRITING FUNCTIONS ====== */ + +/* --- LOW-LEVEL WRITING FUNCTIONS --- */ + +static void writeByte(PyObject* outbuf, int8_t val) { + int8_t net = val; + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int8_t)); +} + +static void writeI16(PyObject* outbuf, int16_t val) { + int16_t net = (int16_t)htons(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int16_t)); +} + +static void writeI32(PyObject* outbuf, int32_t val) { + int32_t net = (int32_t)htonl(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int32_t)); +} + +static void writeI64(PyObject* outbuf, int64_t val) { + int64_t net = (int64_t)htonll(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int64_t)); +} + +static void writeDouble(PyObject* outbuf, double dub) { + // Unfortunately, bitwise_cast doesn't work in C. Bad C! + union { + double f; + int64_t t; + } transfer; + transfer.f = dub; + writeI64(outbuf, transfer.t); +} + + +/* --- MAIN RECURSIVE OUTPUT FUCNTION -- */ + +static int +output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) { + /* + * Refcounting Strategy: + * + * We assume that elements of the thrift_spec tuple are not going to be + * mutated, so we don't ref count those at all. Other than that, we try to + * keep a reference to all the user-created objects while we work with them. + * output_val assumes that a reference is already held. The *caller* is + * responsible for handling references + */ + + switch (type) { + + case T_BOOL: { + int v = PyObject_IsTrue(value); + if (v == -1) { + return false; + } + + writeByte(output, (int8_t) v); + break; + } + case T_I08: { + int32_t val; + + if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) { + return false; + } + + writeByte(output, (int8_t) val); + break; + } + case T_I16: { + int32_t val; + + if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) { + return false; + } + + writeI16(output, (int16_t) val); + break; + } + case T_I32: { + int32_t val; + + if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) { + return false; + } + + writeI32(output, val); + break; + } + case T_I64: { + int64_t nval = PyLong_AsLongLong(value); + + if (INT_CONV_ERROR_OCCURRED(nval)) { + return false; + } + + if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) { + PyErr_SetString(PyExc_OverflowError, "int out of range"); + return false; + } + + writeI64(output, nval); + break; + } + + case T_DOUBLE: { + double nval = PyFloat_AsDouble(value); + if (nval == -1.0 && PyErr_Occurred()) { + return false; + } + + writeDouble(output, nval); + break; + } + + case T_STRING: { + Py_ssize_t len = PyString_Size(value); + + if (!check_ssize_t_32(len)) { + return false; + } + + writeI32(output, (int32_t) len); + PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len); + break; + } + + case T_LIST: + case T_SET: { + Py_ssize_t len; + SetListTypeArgs parsedargs; + PyObject *item; + PyObject *iterator; + + if (!parse_set_list_args(&parsedargs, typeargs)) { + return false; + } + + len = PyObject_Length(value); + + if (!check_ssize_t_32(len)) { + return false; + } + + writeByte(output, parsedargs.element_type); + writeI32(output, (int32_t) len); + + iterator = PyObject_GetIter(value); + if (iterator == NULL) { + return false; + } + + while ((item = PyIter_Next(iterator))) { + if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) { + Py_DECREF(item); + Py_DECREF(iterator); + return false; + } + Py_DECREF(item); + } + + Py_DECREF(iterator); + + if (PyErr_Occurred()) { + return false; + } + + break; + } + + case T_MAP: { + PyObject *k, *v; + Py_ssize_t pos = 0; + Py_ssize_t len; + + MapTypeArgs parsedargs; + + len = PyDict_Size(value); + if (!check_ssize_t_32(len)) { + return false; + } + + if (!parse_map_args(&parsedargs, typeargs)) { + return false; + } + + writeByte(output, parsedargs.ktag); + writeByte(output, parsedargs.vtag); + writeI32(output, len); + + // TODO(bmaurer): should support any mapping, not just dicts + while (PyDict_Next(value, &pos, &k, &v)) { + // TODO(dreiss): Think hard about whether these INCREFs actually + // turn any unsafe scenarios into safe scenarios. + Py_INCREF(k); + Py_INCREF(v); + + if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs) + || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) { + Py_DECREF(k); + Py_DECREF(v); + return false; + } + Py_DECREF(k); + Py_DECREF(v); + } + break; + } + + // TODO(dreiss): Consider breaking this out as a function + // the way we did for decode_struct. + case T_STRUCT: { + StructTypeArgs parsedargs; + Py_ssize_t nspec; + Py_ssize_t i; + + if (!parse_struct_args(&parsedargs, typeargs)) { + return false; + } + + nspec = PyTuple_Size(parsedargs.spec); + + if (nspec == -1) { + return false; + } + + for (i = 0; i < nspec; i++) { + StructItemSpec parsedspec; + PyObject* spec_tuple; + PyObject* instval = NULL; + + spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i); + if (spec_tuple == Py_None) { + continue; + } + + if (!parse_struct_item_spec (&parsedspec, spec_tuple)) { + return false; + } + + instval = PyObject_GetAttr(value, parsedspec.attrname); + + if (!instval) { + return false; + } + + if (instval == Py_None) { + Py_DECREF(instval); + continue; + } + + writeByte(output, (int8_t) parsedspec.type); + writeI16(output, parsedspec.tag); + + if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) { + Py_DECREF(instval); + return false; + } + + Py_DECREF(instval); + } + + writeByte(output, (int8_t)T_STOP); + break; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return false; + + } + + return true; +} + + +/* --- TOP-LEVEL WRAPPER FOR OUTPUT -- */ + +static PyObject * +encode_binary(PyObject *self, PyObject *args) { + PyObject* enc_obj; + PyObject* type_args; + PyObject* buf; + PyObject* ret = NULL; + + if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) { + return NULL; + } + + buf = PycStringIO->NewOutput(INIT_OUTBUF_SIZE); + if (output_val(buf, enc_obj, T_STRUCT, type_args)) { + ret = PycStringIO->cgetvalue(buf); + } + + Py_DECREF(buf); + return ret; +} + +/* ====== END WRITING FUNCTIONS ====== */ + + +/* ====== BEGIN READING FUNCTIONS ====== */ + +/* --- LOW-LEVEL READING FUNCTIONS --- */ + +static void +free_decodebuf(DecodeBuffer* d) { + Py_XDECREF(d->stringiobuf); + Py_XDECREF(d->refill_callable); +} + +static bool +decode_buffer_from_obj(DecodeBuffer* dest, PyObject* obj) { + dest->stringiobuf = PyObject_GetAttr(obj, INTERN_STRING(cstringio_buf)); + if (!dest->stringiobuf) { + return false; + } + + if (!PycStringIO_InputCheck(dest->stringiobuf)) { + free_decodebuf(dest); + PyErr_SetString(PyExc_TypeError, "expecting stringio input"); + return false; + } + + dest->refill_callable = PyObject_GetAttr(obj, INTERN_STRING(cstringio_refill)); + + if(!dest->refill_callable) { + free_decodebuf(dest); + return false; + } + + if (!PyCallable_Check(dest->refill_callable)) { + free_decodebuf(dest); + PyErr_SetString(PyExc_TypeError, "expecting callable"); + return false; + } + + return true; +} + +static bool readBytes(DecodeBuffer* input, char** output, int len) { + int read; + + // TODO(dreiss): Don't fear the malloc. Think about taking a copy of + // the partial read instead of forcing the transport + // to prepend it to its buffer. + + read = PycStringIO->cread(input->stringiobuf, output, len); + + if (read == len) { + return true; + } else if (read == -1) { + return false; + } else { + PyObject* newiobuf; + + // using building functions as this is a rare codepath + newiobuf = PyObject_CallFunction( + input->refill_callable, "s#i", *output, read, len, NULL); + if (newiobuf == NULL) { + return false; + } + + // must do this *AFTER* the call so that we don't deref the io buffer + Py_CLEAR(input->stringiobuf); + input->stringiobuf = newiobuf; + + read = PycStringIO->cread(input->stringiobuf, output, len); + + if (read == len) { + return true; + } else if (read == -1) { + return false; + } else { + // TODO(dreiss): This could be a valid code path for big binary blobs. + PyErr_SetString(PyExc_TypeError, + "refill claimed to have refilled the buffer, but didn't!!"); + return false; + } + } +} + +static int8_t readByte(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int8_t))) { + return -1; + } + + return *(int8_t*) buf; +} + +static int16_t readI16(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int16_t))) { + return -1; + } + + return (int16_t) ntohs(*(int16_t*) buf); +} + +static int32_t readI32(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int32_t))) { + return -1; + } + return (int32_t) ntohl(*(int32_t*) buf); +} + + +static int64_t readI64(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int64_t))) { + return -1; + } + + return (int64_t) ntohll(*(int64_t*) buf); +} + +static double readDouble(DecodeBuffer* input) { + union { + int64_t f; + double t; + } transfer; + + transfer.f = readI64(input); + if (transfer.f == -1) { + return -1; + } + return transfer.t; +} + +static bool +checkTypeByte(DecodeBuffer* input, TType expected) { + TType got = readByte(input); + if (INT_CONV_ERROR_OCCURRED(got)) { + return false; + } + + if (expected != got) { + PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field"); + return false; + } + return true; +} + +static bool +skip(DecodeBuffer* input, TType type) { +#define SKIPBYTES(n) \ + do { \ + if (!readBytes(input, &dummy_buf, (n))) { \ + return false; \ + } \ + } while(0) + + char* dummy_buf; + + switch (type) { + + case T_BOOL: + case T_I08: SKIPBYTES(1); break; + case T_I16: SKIPBYTES(2); break; + case T_I32: SKIPBYTES(4); break; + case T_I64: + case T_DOUBLE: SKIPBYTES(8); break; + + case T_STRING: { + // TODO(dreiss): Find out if these check_ssize_t32s are really necessary. + int len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + SKIPBYTES(len); + break; + } + + case T_LIST: + case T_SET: { + TType etype; + int len, i; + + etype = readByte(input); + if (etype == -1) { + return false; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + for (i = 0; i < len; i++) { + if (!skip(input, etype)) { + return false; + } + } + break; + } + + case T_MAP: { + TType ktype, vtype; + int len, i; + + ktype = readByte(input); + if (ktype == -1) { + return false; + } + + vtype = readByte(input); + if (vtype == -1) { + return false; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + for (i = 0; i < len; i++) { + if (!(skip(input, ktype) && skip(input, vtype))) { + return false; + } + } + break; + } + + case T_STRUCT: { + while (true) { + TType type; + + type = readByte(input); + if (type == -1) { + return false; + } + + if (type == T_STOP) + break; + + SKIPBYTES(2); // tag + if (!skip(input, type)) { + return false; + } + } + break; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return false; + + } + + return true; + +#undef SKIPBYTES +} + + +/* --- HELPER FUNCTION FOR DECODE_VAL --- */ + +static PyObject* +decode_val(DecodeBuffer* input, TType type, PyObject* typeargs); + +static bool +decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) { + int spec_seq_len = PyTuple_Size(spec_seq); + if (spec_seq_len == -1) { + return false; + } + + while (true) { + TType type; + int16_t tag; + PyObject* item_spec; + PyObject* fieldval = NULL; + StructItemSpec parsedspec; + + type = readByte(input); + if (type == -1) { + return false; + } + if (type == T_STOP) { + break; + } + tag = readI16(input); + if (INT_CONV_ERROR_OCCURRED(tag)) { + return false; + } + if (tag >= 0 && tag < spec_seq_len) { + item_spec = PyTuple_GET_ITEM(spec_seq, tag); + } else { + item_spec = Py_None; + } + + if (item_spec == Py_None) { + if (!skip(input, type)) { + return false; + } else { + continue; + } + } + + if (!parse_struct_item_spec(&parsedspec, item_spec)) { + return false; + } + if (parsedspec.type != type) { + if (!skip(input, type)) { + PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading and can't be skipped"); + return false; + } else { + continue; + } + } + + fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs); + if (fieldval == NULL) { + return false; + } + + if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) { + Py_DECREF(fieldval); + return false; + } + Py_DECREF(fieldval); + } + return true; +} + + +/* --- MAIN RECURSIVE INPUT FUCNTION --- */ + +// Returns a new reference. +static PyObject* +decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) { + switch (type) { + + case T_BOOL: { + int8_t v = readByte(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + + switch (v) { + case 0: Py_RETURN_FALSE; + case 1: Py_RETURN_TRUE; + // Don't laugh. This is a potentially serious issue. + default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL; + } + break; + } + case T_I08: { + int8_t v = readByte(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + + return PyInt_FromLong(v); + } + case T_I16: { + int16_t v = readI16(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + return PyInt_FromLong(v); + } + case T_I32: { + int32_t v = readI32(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + return PyInt_FromLong(v); + } + + case T_I64: { + int64_t v = readI64(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + // TODO(dreiss): Find out if we can take this fastpath always when + // sizeof(long) == sizeof(long long). + if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) { + return PyInt_FromLong((long) v); + } + + return PyLong_FromLongLong(v); + } + + case T_DOUBLE: { + double v = readDouble(input); + if (v == -1.0 && PyErr_Occurred()) { + return false; + } + return PyFloat_FromDouble(v); + } + + case T_STRING: { + Py_ssize_t len = readI32(input); + char* buf; + if (!readBytes(input, &buf, len)) { + return NULL; + } + + return PyString_FromStringAndSize(buf, len); + } + + case T_LIST: + case T_SET: { + SetListTypeArgs parsedargs; + int32_t len; + PyObject* ret = NULL; + int i; + + if (!parse_set_list_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!checkTypeByte(input, parsedargs.element_type)) { + return NULL; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return NULL; + } + + ret = PyList_New(len); + if (!ret) { + return NULL; + } + + for (i = 0; i < len; i++) { + PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs); + if (!item) { + Py_DECREF(ret); + return NULL; + } + PyList_SET_ITEM(ret, i, item); + } + + // TODO(dreiss): Consider biting the bullet and making two separate cases + // for list and set, avoiding this post facto conversion. + if (type == T_SET) { + PyObject* setret; +#if (PY_VERSION_HEX < 0x02050000) + // hack needed for older versions + setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL); +#else + // official version + setret = PySet_New(ret); +#endif + Py_DECREF(ret); + return setret; + } + return ret; + } + + case T_MAP: { + int32_t len; + int i; + MapTypeArgs parsedargs; + PyObject* ret = NULL; + + if (!parse_map_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!checkTypeByte(input, parsedargs.ktag)) { + return NULL; + } + if (!checkTypeByte(input, parsedargs.vtag)) { + return NULL; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + ret = PyDict_New(); + if (!ret) { + goto error; + } + + for (i = 0; i < len; i++) { + PyObject* k = NULL; + PyObject* v = NULL; + k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs); + if (k == NULL) { + goto loop_error; + } + v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs); + if (v == NULL) { + goto loop_error; + } + if (PyDict_SetItem(ret, k, v) == -1) { + goto loop_error; + } + + Py_DECREF(k); + Py_DECREF(v); + continue; + + // Yuck! Destructors, anyone? + loop_error: + Py_XDECREF(k); + Py_XDECREF(v); + goto error; + } + + return ret; + + error: + Py_XDECREF(ret); + return NULL; + } + + case T_STRUCT: { + StructTypeArgs parsedargs; + if (!parse_struct_args(&parsedargs, typeargs)) { + return NULL; + } + + PyObject* ret = PyObject_CallObject(parsedargs.klass, NULL); + if (!ret) { + return NULL; + } + + if (!decode_struct(input, ret, parsedargs.spec)) { + Py_DECREF(ret); + return NULL; + } + + return ret; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return NULL; + } +} + + +/* --- TOP-LEVEL WRAPPER FOR INPUT -- */ + +static PyObject* +decode_binary(PyObject *self, PyObject *args) { + PyObject* output_obj = NULL; + PyObject* transport = NULL; + PyObject* typeargs = NULL; + StructTypeArgs parsedargs; + DecodeBuffer input = {}; + + if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) { + return NULL; + } + + if (!parse_struct_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!decode_buffer_from_obj(&input, transport)) { + return NULL; + } + + if (!decode_struct(&input, output_obj, parsedargs.spec)) { + free_decodebuf(&input); + return NULL; + } + + free_decodebuf(&input); + + Py_RETURN_NONE; +} + +/* ====== END READING FUNCTIONS ====== */ + + +/* -- PYTHON MODULE SETUP STUFF --- */ + +static PyMethodDef ThriftFastBinaryMethods[] = { + + {"encode_binary", encode_binary, METH_VARARGS, ""}, + {"decode_binary", decode_binary, METH_VARARGS, ""}, + + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +PyMODINIT_FUNC +initfastbinary(void) { +#define INIT_INTERN_STRING(value) \ + do { \ + INTERN_STRING(value) = PyString_InternFromString(#value); \ + if(!INTERN_STRING(value)) return; \ + } while(0) + + INIT_INTERN_STRING(cstringio_buf); + INIT_INTERN_STRING(cstringio_refill); +#undef INIT_INTERN_STRING + + PycString_IMPORT; + if (PycStringIO == NULL) return; + + (void) Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods); +} diff --git a/storefront/thrift/server/THttpServer.py b/storefront/thrift/server/THttpServer.py new file mode 100644 index 0000000..3047d9c --- /dev/null +++ b/storefront/thrift/server/THttpServer.py @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import BaseHTTPServer + +from thrift.server import TServer +from thrift.transport import TTransport + +class ResponseException(Exception): + """Allows handlers to override the HTTP response + + Normally, THttpServer always sends a 200 response. If a handler wants + to override this behavior (e.g., to simulate a misconfigured or + overloaded web server during testing), it can raise a ResponseException. + The function passed to the constructor will be called with the + RequestHandler as its only argument. + """ + def __init__(self, handler): + self.handler = handler + + +class THttpServer(TServer.TServer): + """A simple HTTP-based Thrift server + + This class is not very performant, but it is useful (for example) for + acting as a mock version of an Apache-based PHP Thrift endpoint.""" + + def __init__(self, processor, server_address, + inputProtocolFactory, outputProtocolFactory = None, + server_class = BaseHTTPServer.HTTPServer): + """Set up protocol factories and HTTP server. + + See BaseHTTPServer for server_address. + See TServer for protocol factories.""" + + if outputProtocolFactory is None: + outputProtocolFactory = inputProtocolFactory + + TServer.TServer.__init__(self, processor, None, None, None, + inputProtocolFactory, outputProtocolFactory) + + thttpserver = self + + class RequestHander(BaseHTTPServer.BaseHTTPRequestHandler): + def do_POST(self): + # Don't care about the request path. + itrans = TTransport.TFileObjectTransport(self.rfile) + otrans = TTransport.TFileObjectTransport(self.wfile) + itrans = TTransport.TBufferedTransport(itrans, int(self.headers['Content-Length'])) + otrans = TTransport.TMemoryBuffer() + iprot = thttpserver.inputProtocolFactory.getProtocol(itrans) + oprot = thttpserver.outputProtocolFactory.getProtocol(otrans) + try: + thttpserver.processor.process(iprot, oprot) + except ResponseException, exn: + exn.handler(self) + else: + self.send_response(200) + self.send_header("content-type", "application/x-thrift") + self.end_headers() + self.wfile.write(otrans.getvalue()) + + self.httpd = server_class(server_address, RequestHander) + + def serve(self): + self.httpd.serve_forever() diff --git a/storefront/thrift/server/TNonblockingServer.py b/storefront/thrift/server/TNonblockingServer.py new file mode 100644 index 0000000..ea348a0 --- /dev/null +++ b/storefront/thrift/server/TNonblockingServer.py @@ -0,0 +1,310 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +"""Implementation of non-blocking server. + +The main idea of the server is reciving and sending requests +only from main thread. + +It also makes thread pool server in tasks terms, not connections. +""" +import threading +import socket +import Queue +import select +import struct +import logging + +from thrift.transport import TTransport +from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory + +__all__ = ['TNonblockingServer'] + +class Worker(threading.Thread): + """Worker is a small helper to process incoming connection.""" + def __init__(self, queue): + threading.Thread.__init__(self) + self.queue = queue + + def run(self): + """Process queries from task queue, stop if processor is None.""" + while True: + try: + processor, iprot, oprot, otrans, callback = self.queue.get() + if processor is None: + break + processor.process(iprot, oprot) + callback(True, otrans.getvalue()) + except Exception: + logging.exception("Exception while processing request") + callback(False, '') + +WAIT_LEN = 0 +WAIT_MESSAGE = 1 +WAIT_PROCESS = 2 +SEND_ANSWER = 3 +CLOSED = 4 + +def locked(func): + "Decorator which locks self.lock." + def nested(self, *args, **kwargs): + self.lock.acquire() + try: + return func(self, *args, **kwargs) + finally: + self.lock.release() + return nested + +def socket_exception(func): + "Decorator close object on socket.error." + def read(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except socket.error: + self.close() + return read + +class Connection: + """Basic class is represented connection. + + It can be in state: + WAIT_LEN --- connection is reading request len. + WAIT_MESSAGE --- connection is reading request. + WAIT_PROCESS --- connection has just read whole request and + waits for call ready routine. + SEND_ANSWER --- connection is sending answer string (including length + of answer). + CLOSED --- socket was closed and connection should be deleted. + """ + def __init__(self, new_socket, wake_up): + self.socket = new_socket + self.socket.setblocking(False) + self.status = WAIT_LEN + self.len = 0 + self.message = '' + self.lock = threading.Lock() + self.wake_up = wake_up + + def _read_len(self): + """Reads length of request. + + It's really paranoic routine and it may be replaced by + self.socket.recv(4).""" + read = self.socket.recv(4 - len(self.message)) + if len(read) == 0: + # if we read 0 bytes and self.message is empty, it means client close + # connection + if len(self.message) != 0: + logging.error("can't read frame size from socket") + self.close() + return + self.message += read + if len(self.message) == 4: + self.len, = struct.unpack('!i', self.message) + if self.len < 0: + logging.error("negative frame size, it seems client"\ + " doesn't use FramedTransport") + self.close() + elif self.len == 0: + logging.error("empty frame, it's really strange") + self.close() + else: + self.message = '' + self.status = WAIT_MESSAGE + + @socket_exception + def read(self): + """Reads data from stream and switch state.""" + assert self.status in (WAIT_LEN, WAIT_MESSAGE) + if self.status == WAIT_LEN: + self._read_len() + # go back to the main loop here for simplicity instead of + # falling through, even though there is a good chance that + # the message is already available + elif self.status == WAIT_MESSAGE: + read = self.socket.recv(self.len - len(self.message)) + if len(read) == 0: + logging.error("can't read frame from socket (get %d of %d bytes)" % + (len(self.message), self.len)) + self.close() + return + self.message += read + if len(self.message) == self.len: + self.status = WAIT_PROCESS + + @socket_exception + def write(self): + """Writes data from socket and switch state.""" + assert self.status == SEND_ANSWER + sent = self.socket.send(self.message) + if sent == len(self.message): + self.status = WAIT_LEN + self.message = '' + self.len = 0 + else: + self.message = self.message[sent:] + + @locked + def ready(self, all_ok, message): + """Callback function for switching state and waking up main thread. + + This function is the only function witch can be called asynchronous. + + The ready can switch Connection to three states: + WAIT_LEN if request was oneway. + SEND_ANSWER if request was processed in normal way. + CLOSED if request throws unexpected exception. + + The one wakes up main thread. + """ + assert self.status == WAIT_PROCESS + if not all_ok: + self.close() + self.wake_up() + return + self.len = '' + if len(message) == 0: + # it was a oneway request, do not write answer + self.message = '' + self.status = WAIT_LEN + else: + self.message = struct.pack('!i', len(message)) + message + self.status = SEND_ANSWER + self.wake_up() + + @locked + def is_writeable(self): + "Returns True if connection should be added to write list of select." + return self.status == SEND_ANSWER + + # it's not necessary, but... + @locked + def is_readable(self): + "Returns True if connection should be added to read list of select." + return self.status in (WAIT_LEN, WAIT_MESSAGE) + + @locked + def is_closed(self): + "Returns True if connection is closed." + return self.status == CLOSED + + def fileno(self): + "Returns the file descriptor of the associated socket." + return self.socket.fileno() + + def close(self): + "Closes connection" + self.status = CLOSED + self.socket.close() + +class TNonblockingServer: + """Non-blocking server.""" + def __init__(self, processor, lsocket, inputProtocolFactory=None, + outputProtocolFactory=None, threads=10): + self.processor = processor + self.socket = lsocket + self.in_protocol = inputProtocolFactory or TBinaryProtocolFactory() + self.out_protocol = outputProtocolFactory or self.in_protocol + self.threads = int(threads) + self.clients = {} + self.tasks = Queue.Queue() + self._read, self._write = socket.socketpair() + self.prepared = False + + def setNumThreads(self, num): + """Set the number of worker threads that should be created.""" + # implement ThreadPool interface + assert not self.prepared, "You can't change number of threads for working server" + self.threads = num + + def prepare(self): + """Prepares server for serve requests.""" + self.socket.listen() + for _ in xrange(self.threads): + thread = Worker(self.tasks) + thread.setDaemon(True) + thread.start() + self.prepared = True + + def wake_up(self): + """Wake up main thread. + + The server usualy waits in select call in we should terminate one. + The simplest way is using socketpair. + + Select always wait to read from the first socket of socketpair. + + In this case, we can just write anything to the second socket from + socketpair.""" + self._write.send('1') + + def _select(self): + """Does select on open connections.""" + readable = [self.socket.handle.fileno(), self._read.fileno()] + writable = [] + for i, connection in self.clients.items(): + if connection.is_readable(): + readable.append(connection.fileno()) + if connection.is_writeable(): + writable.append(connection.fileno()) + if connection.is_closed(): + del self.clients[i] + return select.select(readable, writable, readable) + + def handle(self): + """Handle requests. + + WARNING! You must call prepare BEFORE calling handle. + """ + assert self.prepared, "You have to call prepare before handle" + rset, wset, xset = self._select() + for readable in rset: + if readable == self._read.fileno(): + # don't care i just need to clean readable flag + self._read.recv(1024) + elif readable == self.socket.handle.fileno(): + client = self.socket.accept().handle + self.clients[client.fileno()] = Connection(client, self.wake_up) + else: + connection = self.clients[readable] + connection.read() + if connection.status == WAIT_PROCESS: + itransport = TTransport.TMemoryBuffer(connection.message) + otransport = TTransport.TMemoryBuffer() + iprot = self.in_protocol.getProtocol(itransport) + oprot = self.out_protocol.getProtocol(otransport) + self.tasks.put([self.processor, iprot, oprot, + otransport, connection.ready]) + for writeable in wset: + self.clients[writeable].write() + for oob in xset: + self.clients[oob].close() + del self.clients[oob] + + def close(self): + """Closes the server.""" + for _ in xrange(self.threads): + self.tasks.put([None, None, None, None, None]) + self.socket.close() + self.prepared = False + + def serve(self): + """Serve forever.""" + self.prepare() + while True: + self.handle() diff --git a/storefront/thrift/server/TServer.py b/storefront/thrift/server/TServer.py new file mode 100644 index 0000000..8456e2d --- /dev/null +++ b/storefront/thrift/server/TServer.py @@ -0,0 +1,274 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import logging +import sys +import os +import traceback +import threading +import Queue + +from thrift.Thrift import TProcessor +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol + +class TServer: + + """Base interface for a server, which must have a serve method.""" + + """ 3 constructors for all servers: + 1) (processor, serverTransport) + 2) (processor, serverTransport, transportFactory, protocolFactory) + 3) (processor, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory)""" + def __init__(self, *args): + if (len(args) == 2): + self.__initArgs__(args[0], args[1], + TTransport.TTransportFactoryBase(), + TTransport.TTransportFactoryBase(), + TBinaryProtocol.TBinaryProtocolFactory(), + TBinaryProtocol.TBinaryProtocolFactory()) + elif (len(args) == 4): + self.__initArgs__(args[0], args[1], args[2], args[2], args[3], args[3]) + elif (len(args) == 6): + self.__initArgs__(args[0], args[1], args[2], args[3], args[4], args[5]) + + def __initArgs__(self, processor, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory): + self.processor = processor + self.serverTransport = serverTransport + self.inputTransportFactory = inputTransportFactory + self.outputTransportFactory = outputTransportFactory + self.inputProtocolFactory = inputProtocolFactory + self.outputProtocolFactory = outputProtocolFactory + + def serve(self): + pass + +class TSimpleServer(TServer): + + """Simple single-threaded server that just pumps around one transport.""" + + def __init__(self, *args): + TServer.__init__(self, *args) + + def serve(self): + self.serverTransport.listen() + while True: + client = self.serverTransport.accept() + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + +class TThreadedServer(TServer): + + """Threaded server that spawns a new thread per each connection.""" + + def __init__(self, *args, **kwargs): + TServer.__init__(self, *args) + self.daemon = kwargs.get("daemon", False) + + def serve(self): + self.serverTransport.listen() + while True: + try: + client = self.serverTransport.accept() + t = threading.Thread(target = self.handle, args=(client,)) + t.setDaemon(self.daemon) + t.start() + except KeyboardInterrupt: + raise + except Exception, x: + logging.exception(x) + + def handle(self, client): + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + +class TThreadPoolServer(TServer): + + """Server with a fixed size pool of threads which service requests.""" + + def __init__(self, *args, **kwargs): + TServer.__init__(self, *args) + self.clients = Queue.Queue() + self.threads = 10 + self.daemon = kwargs.get("daemon", False) + + def setNumThreads(self, num): + """Set the number of worker threads that should be created""" + self.threads = num + + def serveThread(self): + """Loop around getting clients from the shared queue and process them.""" + while True: + try: + client = self.clients.get() + self.serveClient(client) + except Exception, x: + logging.exception(x) + + def serveClient(self, client): + """Process input/output from a client for as long as possible""" + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + + def serve(self): + """Start a fixed number of worker threads and put client into a queue""" + for i in range(self.threads): + try: + t = threading.Thread(target = self.serveThread) + t.setDaemon(self.daemon) + t.start() + except Exception, x: + logging.exception(x) + + # Pump the socket for clients + self.serverTransport.listen() + while True: + try: + client = self.serverTransport.accept() + self.clients.put(client) + except Exception, x: + logging.exception(x) + + +class TForkingServer(TServer): + + """A Thrift server that forks a new process for each request""" + """ + This is more scalable than the threaded server as it does not cause + GIL contention. + + Note that this has different semantics from the threading server. + Specifically, updates to shared variables will no longer be shared. + It will also not work on windows. + + This code is heavily inspired by SocketServer.ForkingMixIn in the + Python stdlib. + """ + + def __init__(self, *args): + TServer.__init__(self, *args) + self.children = [] + + def serve(self): + def try_close(file): + try: + file.close() + except IOError, e: + logging.warning(e, exc_info=True) + + + self.serverTransport.listen() + while True: + client = self.serverTransport.accept() + try: + pid = os.fork() + + if pid: # parent + # add before collect, otherwise you race w/ waitpid + self.children.append(pid) + self.collect_children() + + # Parent must close socket or the connection may not get + # closed promptly + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + try_close(itrans) + try_close(otrans) + else: + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + + ecode = 0 + try: + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, e: + logging.exception(e) + ecode = 1 + finally: + try_close(itrans) + try_close(otrans) + + os._exit(ecode) + + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + + def collect_children(self): + while self.children: + try: + pid, status = os.waitpid(0, os.WNOHANG) + except os.error: + pid = None + + if pid: + self.children.remove(pid) + else: + break + + diff --git a/storefront/thrift/server/__init__.py b/storefront/thrift/server/__init__.py new file mode 100644 index 0000000..1bf6e25 --- /dev/null +++ b/storefront/thrift/server/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TServer', 'TNonblockingServer'] diff --git a/storefront/thrift/transport/THttpClient.py b/storefront/thrift/transport/THttpClient.py new file mode 100644 index 0000000..5026978 --- /dev/null +++ b/storefront/thrift/transport/THttpClient.py @@ -0,0 +1,126 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TTransport import * +from cStringIO import StringIO + +import urlparse +import httplib +import warnings +import socket + +class THttpClient(TTransportBase): + + """Http implementation of TTransport base.""" + + def __init__(self, uri_or_host, port=None, path=None): + """THttpClient supports two different types constructor parameters. + + THttpClient(host, port, path) - deprecated + THttpClient(uri) + + Only the second supports https.""" + + if port is not None: + warnings.warn("Please use the THttpClient('http://host:port/path') syntax", DeprecationWarning, stacklevel=2) + self.host = uri_or_host + self.port = port + assert path + self.path = path + self.scheme = 'http' + else: + parsed = urlparse.urlparse(uri_or_host) + self.scheme = parsed.scheme + assert self.scheme in ('http', 'https') + if self.scheme == 'http': + self.port = parsed.port or httplib.HTTP_PORT + elif self.scheme == 'https': + self.port = parsed.port or httplib.HTTPS_PORT + self.host = parsed.hostname + self.path = parsed.path + if parsed.query: + self.path += '?%s' % parsed.query + self.__wbuf = StringIO() + self.__http = None + self.__timeout = None + + def open(self): + if self.scheme == 'http': + self.__http = httplib.HTTP(self.host, self.port) + else: + self.__http = httplib.HTTPS(self.host, self.port) + + def close(self): + self.__http.close() + self.__http = None + + def isOpen(self): + return self.__http != None + + def setTimeout(self, ms): + if not hasattr(socket, 'getdefaulttimeout'): + raise NotImplementedError + + if ms is None: + self.__timeout = None + else: + self.__timeout = ms/1000.0 + + def read(self, sz): + return self.__http.file.read(sz) + + def write(self, buf): + self.__wbuf.write(buf) + + def __withTimeout(f): + def _f(*args, **kwargs): + orig_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(args[0].__timeout) + result = f(*args, **kwargs) + socket.setdefaulttimeout(orig_timeout) + return result + return _f + + def flush(self): + if self.isOpen(): + self.close() + self.open(); + + # Pull data out of buffer + data = self.__wbuf.getvalue() + self.__wbuf = StringIO() + + # HTTP request + self.__http.putrequest('POST', self.path) + + # Write headers + self.__http.putheader('Host', self.host) + self.__http.putheader('Content-Type', 'application/x-thrift') + self.__http.putheader('Content-Length', str(len(data))) + self.__http.endheaders() + + # Write payload + self.__http.send(data) + + # Get reply to flush the request + self.code, self.message, self.headers = self.__http.getreply() + + # Decorate if we know how to timeout + if hasattr(socket, 'getdefaulttimeout'): + flush = __withTimeout(flush) diff --git a/storefront/thrift/transport/TSocket.py b/storefront/thrift/transport/TSocket.py new file mode 100644 index 0000000..d77e358 --- /dev/null +++ b/storefront/thrift/transport/TSocket.py @@ -0,0 +1,163 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TTransport import * +import os +import errno +import socket +import sys + +class TSocketBase(TTransportBase): + def _resolveAddr(self): + if self._unix_socket is not None: + return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, self._unix_socket)] + else: + return socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE | socket.AI_ADDRCONFIG) + + def close(self): + if self.handle: + self.handle.close() + self.handle = None + +class TSocket(TSocketBase): + """Socket implementation of TTransport base.""" + + def __init__(self, host='localhost', port=9090, unix_socket=None): + """Initialize a TSocket + + @param host(str) The host to connect to. + @param port(int) The (TCP) port to connect to. + @param unix_socket(str) The filename of a unix socket to connect to. + (host and port will be ignored.) + """ + + self.host = host + self.port = port + self.handle = None + self._unix_socket = unix_socket + self._timeout = None + + def setHandle(self, h): + self.handle = h + + def isOpen(self): + return self.handle != None + + def setTimeout(self, ms): + if ms is None: + self._timeout = None + else: + self._timeout = ms/1000.0 + + if (self.handle != None): + self.handle.settimeout(self._timeout) + + def open(self): + try: + res0 = self._resolveAddr() + for res in res0: + self.handle = socket.socket(res[0], res[1]) + self.handle.settimeout(self._timeout) + try: + self.handle.connect(res[4]) + except socket.error, e: + if res is not res0[-1]: + continue + else: + raise e + break + except socket.error, e: + if self._unix_socket: + message = 'Could not connect to socket %s' % self._unix_socket + else: + message = 'Could not connect to %s:%d' % (self.host, self.port) + raise TTransportException(type=TTransportException.NOT_OPEN, message=message) + + def read(self, sz): + try: + buff = self.handle.recv(sz) + except socket.error, e: + if (e.args[0] == errno.ECONNRESET and + (sys.platform == 'darwin' or sys.platform.startswith('freebsd'))): + # freebsd and Mach don't follow POSIX semantic of recv + # and fail with ECONNRESET if peer performed shutdown. + # See corresponding comment and code in TSocket::read() + # in lib/cpp/src/transport/TSocket.cpp. + self.close() + # Trigger the check to raise the END_OF_FILE exception below. + buff = '' + else: + raise + if len(buff) == 0: + raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket read 0 bytes') + return buff + + def write(self, buff): + if not self.handle: + raise TTransportException(type=TTransportException.NOT_OPEN, message='Transport not open') + sent = 0 + have = len(buff) + while sent < have: + plus = self.handle.send(buff) + if plus == 0: + raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket sent 0 bytes') + sent += plus + buff = buff[plus:] + + def flush(self): + pass + +class TServerSocket(TSocketBase, TServerTransportBase): + """Socket implementation of TServerTransport base.""" + + def __init__(self, port=9090, unix_socket=None): + self.host = None + self.port = port + self._unix_socket = unix_socket + self.handle = None + + def listen(self): + res0 = self._resolveAddr() + for res in res0: + if res[0] is socket.AF_INET6 or res is res0[-1]: + break + + # We need remove the old unix socket if the file exists and + # nobody is listening on it. + if self._unix_socket: + tmp = socket.socket(res[0], res[1]) + try: + tmp.connect(res[4]) + except socket.error, err: + eno, message = err.args + if eno == errno.ECONNREFUSED: + os.unlink(res[4]) + + self.handle = socket.socket(res[0], res[1]) + self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(self.handle, 'set_timeout'): + self.handle.set_timeout(None) + self.handle.bind(res[4]) + self.handle.listen(128) + + def accept(self): + client, addr = self.handle.accept() + result = TSocket() + result.setHandle(client) + return result diff --git a/storefront/thrift/transport/TTransport.py b/storefront/thrift/transport/TTransport.py new file mode 100644 index 0000000..12e51a9 --- /dev/null +++ b/storefront/thrift/transport/TTransport.py @@ -0,0 +1,331 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from cStringIO import StringIO +from struct import pack,unpack +from thrift.Thrift import TException + +class TTransportException(TException): + + """Custom Transport Exception class""" + + UNKNOWN = 0 + NOT_OPEN = 1 + ALREADY_OPEN = 2 + TIMED_OUT = 3 + END_OF_FILE = 4 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + +class TTransportBase: + + """Base class for Thrift transport layer.""" + + def isOpen(self): + pass + + def open(self): + pass + + def close(self): + pass + + def read(self, sz): + pass + + def readAll(self, sz): + buff = '' + have = 0 + while (have < sz): + chunk = self.read(sz-have) + have += len(chunk) + buff += chunk + + if len(chunk) == 0: + raise EOFError() + + return buff + + def write(self, buf): + pass + + def flush(self): + pass + +# This class should be thought of as an interface. +class CReadableTransport: + """base class for transports that are readable from C""" + + # TODO(dreiss): Think about changing this interface to allow us to use + # a (Python, not c) StringIO instead, because it allows + # you to write after reading. + + # NOTE: This is a classic class, so properties will NOT work + # correctly for setting. + @property + def cstringio_buf(self): + """A cStringIO buffer that contains the current chunk we are reading.""" + pass + + def cstringio_refill(self, partialread, reqlen): + """Refills cstringio_buf. + + Returns the currently used buffer (which can but need not be the same as + the old cstringio_buf). partialread is what the C code has read from the + buffer, and should be inserted into the buffer before any more reads. The + return value must be a new, not borrowed reference. Something along the + lines of self._buf should be fine. + + If reqlen bytes can't be read, throw EOFError. + """ + pass + +class TServerTransportBase: + + """Base class for Thrift server transports.""" + + def listen(self): + pass + + def accept(self): + pass + + def close(self): + pass + +class TTransportFactoryBase: + + """Base class for a Transport Factory""" + + def getTransport(self, trans): + return trans + +class TBufferedTransportFactory: + + """Factory transport that builds buffered transports""" + + def getTransport(self, trans): + buffered = TBufferedTransport(trans) + return buffered + + +class TBufferedTransport(TTransportBase,CReadableTransport): + + """Class that wraps another transport and buffers its I/O. + + The implementation uses a (configurable) fixed-size read buffer + but buffers all writes until a flush is performed. + """ + + DEFAULT_BUFFER = 4096 + + def __init__(self, trans, rbuf_size = DEFAULT_BUFFER): + self.__trans = trans + self.__wbuf = StringIO() + self.__rbuf = StringIO("") + self.__rbuf_size = rbuf_size + + def isOpen(self): + return self.__trans.isOpen() + + def open(self): + return self.__trans.open() + + def close(self): + return self.__trans.close() + + def read(self, sz): + ret = self.__rbuf.read(sz) + if len(ret) != 0: + return ret + + self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size))) + return self.__rbuf.read(sz) + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + out = self.__wbuf.getvalue() + # reset wbuf before write/flush to preserve state on underlying failure + self.__wbuf = StringIO() + self.__trans.write(out) + self.__trans.flush() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self.__rbuf + + def cstringio_refill(self, partialread, reqlen): + retstring = partialread + if reqlen < self.__rbuf_size: + # try to make a read of as much as we can. + retstring += self.__trans.read(self.__rbuf_size) + + # but make sure we do read reqlen bytes. + if len(retstring) < reqlen: + retstring += self.__trans.readAll(reqlen - len(retstring)) + + self.__rbuf = StringIO(retstring) + return self.__rbuf + +class TMemoryBuffer(TTransportBase, CReadableTransport): + """Wraps a cStringIO object as a TTransport. + + NOTE: Unlike the C++ version of this class, you cannot write to it + then immediately read from it. If you want to read from a + TMemoryBuffer, you must either pass a string to the constructor. + TODO(dreiss): Make this work like the C++ version. + """ + + def __init__(self, value=None): + """value -- a value to read from for stringio + + If value is set, this will be a transport for reading, + otherwise, it is for writing""" + if value is not None: + self._buffer = StringIO(value) + else: + self._buffer = StringIO() + + def isOpen(self): + return not self._buffer.closed + + def open(self): + pass + + def close(self): + self._buffer.close() + + def read(self, sz): + return self._buffer.read(sz) + + def write(self, buf): + self._buffer.write(buf) + + def flush(self): + pass + + def getvalue(self): + return self._buffer.getvalue() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self._buffer + + def cstringio_refill(self, partialread, reqlen): + # only one shot at reading... + raise EOFError() + +class TFramedTransportFactory: + + """Factory transport that builds framed transports""" + + def getTransport(self, trans): + framed = TFramedTransport(trans) + return framed + + +class TFramedTransport(TTransportBase, CReadableTransport): + + """Class that wraps another transport and frames its I/O when writing.""" + + def __init__(self, trans,): + self.__trans = trans + self.__rbuf = StringIO() + self.__wbuf = StringIO() + + def isOpen(self): + return self.__trans.isOpen() + + def open(self): + return self.__trans.open() + + def close(self): + return self.__trans.close() + + def read(self, sz): + ret = self.__rbuf.read(sz) + if len(ret) != 0: + return ret + + self.readFrame() + return self.__rbuf.read(sz) + + def readFrame(self): + buff = self.__trans.readAll(4) + sz, = unpack('!i', buff) + self.__rbuf = StringIO(self.__trans.readAll(sz)) + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + wout = self.__wbuf.getvalue() + wsz = len(wout) + # reset wbuf before write/flush to preserve state on underlying failure + self.__wbuf = StringIO() + # N.B.: Doing this string concatenation is WAY cheaper than making + # two separate calls to the underlying socket object. Socket writes in + # Python turn out to be REALLY expensive, but it seems to do a pretty + # good job of managing string buffer operations without excessive copies + buf = pack("!i", wsz) + wout + self.__trans.write(buf) + self.__trans.flush() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self.__rbuf + + def cstringio_refill(self, prefix, reqlen): + # self.__rbuf will already be empty here because fastbinary doesn't + # ask for a refill until the previous buffer is empty. Therefore, + # we can start reading new frames immediately. + while len(prefix) < reqlen: + self.readFrame() + prefix += self.__rbuf.getvalue() + self.__rbuf = StringIO(prefix) + return self.__rbuf + + +class TFileObjectTransport(TTransportBase): + """Wraps a file-like object to make it work as a Thrift transport.""" + + def __init__(self, fileobj): + self.fileobj = fileobj + + def isOpen(self): + return True + + def close(self): + self.fileobj.close() + + def read(self, sz): + return self.fileobj.read(sz) + + def write(self, buf): + self.fileobj.write(buf) + + def flush(self): + self.fileobj.flush() diff --git a/storefront/thrift/transport/TTwisted.py b/storefront/thrift/transport/TTwisted.py new file mode 100644 index 0000000..b6dcb4e --- /dev/null +++ b/storefront/thrift/transport/TTwisted.py @@ -0,0 +1,219 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from zope.interface import implements, Interface, Attribute +from twisted.internet.protocol import Protocol, ServerFactory, ClientFactory, \ + connectionDone +from twisted.internet import defer +from twisted.protocols import basic +from twisted.python import log +from twisted.web import server, resource, http + +from thrift.transport import TTransport +from cStringIO import StringIO + + +class TMessageSenderTransport(TTransport.TTransportBase): + + def __init__(self): + self.__wbuf = StringIO() + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + msg = self.__wbuf.getvalue() + self.__wbuf = StringIO() + self.sendMessage(msg) + + def sendMessage(self, message): + raise NotImplementedError + + +class TCallbackTransport(TMessageSenderTransport): + + def __init__(self, func): + TMessageSenderTransport.__init__(self) + self.func = func + + def sendMessage(self, message): + self.func(message) + + +class ThriftClientProtocol(basic.Int32StringReceiver): + + MAX_LENGTH = 2 ** 31 - 1 + + def __init__(self, client_class, iprot_factory, oprot_factory=None): + self._client_class = client_class + self._iprot_factory = iprot_factory + if oprot_factory is None: + self._oprot_factory = iprot_factory + else: + self._oprot_factory = oprot_factory + + self.recv_map = {} + self.started = defer.Deferred() + + def dispatch(self, msg): + self.sendString(msg) + + def connectionMade(self): + tmo = TCallbackTransport(self.dispatch) + self.client = self._client_class(tmo, self._oprot_factory) + self.started.callback(self.client) + + def connectionLost(self, reason=connectionDone): + for k,v in self.client._reqs.iteritems(): + tex = TTransport.TTransportException( + type=TTransport.TTransportException.END_OF_FILE, + message='Connection closed') + v.errback(tex) + + def stringReceived(self, frame): + tr = TTransport.TMemoryBuffer(frame) + iprot = self._iprot_factory.getProtocol(tr) + (fname, mtype, rseqid) = iprot.readMessageBegin() + + try: + method = self.recv_map[fname] + except KeyError: + method = getattr(self.client, 'recv_' + fname) + self.recv_map[fname] = method + + method(iprot, mtype, rseqid) + + +class ThriftServerProtocol(basic.Int32StringReceiver): + + MAX_LENGTH = 2 ** 31 - 1 + + def dispatch(self, msg): + self.sendString(msg) + + def processError(self, error): + self.transport.loseConnection() + + def processOk(self, _, tmo): + msg = tmo.getvalue() + + if len(msg) > 0: + self.dispatch(msg) + + def stringReceived(self, frame): + tmi = TTransport.TMemoryBuffer(frame) + tmo = TTransport.TMemoryBuffer() + + iprot = self.factory.iprot_factory.getProtocol(tmi) + oprot = self.factory.oprot_factory.getProtocol(tmo) + + d = self.factory.processor.process(iprot, oprot) + d.addCallbacks(self.processOk, self.processError, + callbackArgs=(tmo,)) + + +class IThriftServerFactory(Interface): + + processor = Attribute("Thrift processor") + + iprot_factory = Attribute("Input protocol factory") + + oprot_factory = Attribute("Output protocol factory") + + +class IThriftClientFactory(Interface): + + client_class = Attribute("Thrift client class") + + iprot_factory = Attribute("Input protocol factory") + + oprot_factory = Attribute("Output protocol factory") + + +class ThriftServerFactory(ServerFactory): + + implements(IThriftServerFactory) + + protocol = ThriftServerProtocol + + def __init__(self, processor, iprot_factory, oprot_factory=None): + self.processor = processor + self.iprot_factory = iprot_factory + if oprot_factory is None: + self.oprot_factory = iprot_factory + else: + self.oprot_factory = oprot_factory + + +class ThriftClientFactory(ClientFactory): + + implements(IThriftClientFactory) + + protocol = ThriftClientProtocol + + def __init__(self, client_class, iprot_factory, oprot_factory=None): + self.client_class = client_class + self.iprot_factory = iprot_factory + if oprot_factory is None: + self.oprot_factory = iprot_factory + else: + self.oprot_factory = oprot_factory + + def buildProtocol(self, addr): + p = self.protocol(self.client_class, self.iprot_factory, + self.oprot_factory) + p.factory = self + return p + + +class ThriftResource(resource.Resource): + + allowedMethods = ('POST',) + + def __init__(self, processor, inputProtocolFactory, + outputProtocolFactory=None): + resource.Resource.__init__(self) + self.inputProtocolFactory = inputProtocolFactory + if outputProtocolFactory is None: + self.outputProtocolFactory = inputProtocolFactory + else: + self.outputProtocolFactory = outputProtocolFactory + self.processor = processor + + def getChild(self, path, request): + return self + + def _cbProcess(self, _, request, tmo): + msg = tmo.getvalue() + request.setResponseCode(http.OK) + request.setHeader("content-type", "application/x-thrift") + request.write(msg) + request.finish() + + def render_POST(self, request): + request.content.seek(0, 0) + data = request.content.read() + tmi = TTransport.TMemoryBuffer(data) + tmo = TTransport.TMemoryBuffer() + + iprot = self.inputProtocolFactory.getProtocol(tmi) + oprot = self.outputProtocolFactory.getProtocol(tmo) + + d = self.processor.process(iprot, oprot) + d.addCallback(self._cbProcess, request, tmo) + return server.NOT_DONE_YET diff --git a/storefront/thrift/transport/__init__.py b/storefront/thrift/transport/__init__.py new file mode 100644 index 0000000..02c6048 --- /dev/null +++ b/storefront/thrift/transport/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TTransport', 'TSocket', 'THttpClient'] diff --git a/storefront/update_blog_posts.py b/storefront/update_blog_posts.py new file mode 100755 index 0000000..e7a2b83 --- /dev/null +++ b/storefront/update_blog_posts.py @@ -0,0 +1,13 @@ +import csv, feedparser, time +from datetime import date +from storefront.models import BlogPostInfo + +blog_posts = BlogPostInfo.objects.all() +feed = feedparser.parse( 'http://blog.indextank.com/feed/' ) + +for item in feed['items']: + if not any(item['link'] == post.url for post in blog_posts ): + # if there isn't a post with this url, then create it + d = date(item['date_parsed'][0], item['date_parsed'][1], item['date_parsed'][2]) + BlogPostInfo.objects.create(url=item['link'].encode('utf-8'), title=item['title'].encode('utf-8'), author=item['author'].encode('utf-8'), date=d) + diff --git a/storefront/update_blog_posts.sh b/storefront/update_blog_posts.sh new file mode 100755 index 0000000..26aa8d8 --- /dev/null +++ b/storefront/update_blog_posts.sh @@ -0,0 +1,5 @@ +export DJANGO_SETTINGS_MODULE=settings +export PYTHONPATH=../:. + +python update_blog_posts.py + diff --git a/storefront/updates/__init__.py b/storefront/updates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/storefront/updates/updates.py b/storefront/updates/updates.py new file mode 100644 index 0000000..7753292 --- /dev/null +++ b/storefront/updates/updates.py @@ -0,0 +1,160 @@ +from util import column_exists, safe_add_column, makeNullable, get_column_type + +def perform_updates(): + enlarge_username() + add_change_password() + add_analyzer_config() + add_account_default_analyzer() + add_effective_bdb() + update_custom_subscription() + add_public_api() + add_account_provisioner() + add_deploy_dying() + enlarge_ccdigits() + enlarge_function_storage() + add_index_deleted() + +def add_public_api(): + safe_add_column('storefront_index', 'public_api', 'TINYINT(1) NOT NULL DEFAULT 0') + +def add_deploy_dying(): + safe_add_column('storefront_deploy', 'dying', 'TINYINT(1) NOT NULL DEFAULT 0') + +def add_index_status(): + safe_add_column('storefront_index', 'status', 'varchar(50) NOT NULL') + +def add_index_deleted(): + safe_add_column('storefront_index', 'deleted', 'TINYINT(1) NOT NULL DEFAULT 0') + +def enlarge_username(): + username_type = get_column_type('auth_user', 'username') + if username_type != 'varchar(100)': + print 'SCHEMA UPDATE: Enlarging auth_user.username column length to varchar(100)' + + from django.db import connection, transaction + cursor = connection.cursor() + sql = 'ALTER TABLE `auth_user` MODIFY COLUMN `username` varchar(100) NOT NULL' + cursor.execute(sql) + + transaction.commit_unless_managed() + + +def enlarge_ccdigits(): + username_type = get_column_type('storefront_accountpayinginformation', 'credit_card_last_digits') + if username_type != 'varchar(4)': + print 'SCHEMA UPDATE: Enlarging auth_user.username column length to varchar(100)' + + from django.db import connection, transaction + cursor = connection.cursor() + sql = 'ALTER TABLE `storefront_accountpayinginformation` MODIFY COLUMN `credit_card_last_digits` varchar(4)' + cursor.execute(sql) + + transaction.commit_unless_managed() + + +def add_analyzer_config(): + safe_add_column('storefront_index', 'analyzer_config', 'TEXT NULL') + +def update_custom_subscription(): + if not column_exists('storefront_accountpayinginformation', 'monthly_amount'): + print 'SCHEMA UPDATE: Upgrading table: storefront_accountpayinginformation' + + from django.db import connection, transaction + cursor = connection.cursor() + sql = 'ALTER TABLE `storefront_accountpayinginformation` MODIFY `first_name` varchar(50) NULL' + cursor.execute(sql) + sql = 'ALTER TABLE `storefront_accountpayinginformation` MODIFY `last_name` varchar(50) NULL' + cursor.execute(sql) + sql = 'ALTER TABLE `storefront_accountpayinginformation` MODIFY `contact_email` varchar(255) NULL' + cursor.execute(sql) + sql = 'ALTER TABLE `storefront_accountpayinginformation` MODIFY `credit_card_last_digits` varchar(3) NULL' + cursor.execute(sql) + sql = 'ALTER TABLE `storefront_accountpayinginformation` ADD COLUMN `subscription_status` varchar(30) NULL' + cursor.execute(sql) + sql = 'ALTER TABLE `storefront_accountpayinginformation` ADD COLUMN `subscription_type` varchar(30) NULL' + cursor.execute(sql) + sql = 'ALTER TABLE `storefront_accountpayinginformation` ADD COLUMN `monthly_amount` integer NULL' + cursor.execute(sql) + + transaction.commit_unless_managed() + +def add_account_default_analyzer(): + if not column_exists('storefront_account', 'default_analyzer_id'): + print 'SCHEMA UPDATE: Adding missing column: storefront_account.default_analyzer' + + from django.db import connection, transaction + cursor = connection.cursor() + sql = 'ALTER TABLE `storefront_account` ADD COLUMN `default_analyzer_id` integer NULL' + cursor.execute(sql) + + sql = 'ALTER TABLE `storefront_account` ADD CONSTRAINT `default_analyzer_id_refs_id_44eb0c5b` FOREIGN KEY (`default_analyzer_id`) REFERENCES `storefront_analyzer` (`id`)' + cursor.execute(sql) + transaction.commit_unless_managed() + +def add_change_password(): + if not column_exists('storefront_pfuser', 'change_password'): + print 'SCHEMA UPDATE: Adding missing column: storefront_pfuser.change_password' + + from django.db import connection, transaction + cursor = connection.cursor() + sql = 'ALTER TABLE `storefront_pfuser` ADD COLUMN `change_password` tinyint NULL' + cursor.execute(sql) + + print 'DATA UPDATE' + sql = 'UPDATE storefront_pfuser set change_password=0' + cursor.execute(sql) + sql = 'ALTER TABLE `storefront_pfuser` MODIFY COLUMN `change_password` tinyint NOT NULL' + cursor.execute(sql) + transaction.commit_unless_managed() + +def add_effective_bdb(): + if not column_exists('storefront_deploy', 'effective_bdb'): + print 'SCHEMA UPDATE: Adding missing column: storefront_deploy.effective_bdb' + + from django.db import connection, transaction + cursor = connection.cursor() + sql = 'ALTER TABLE `storefront_deploy` ADD COLUMN `effective_bdb` integer NOT NULL DEFAULT 0' + cursor.execute(sql) + + transaction.commit_unless_managed() + +def add_account_provisioner(): + safe_add_column('storefront_account', 'provisioner_id', 'integer NULL') + +def enlarge_function_storage(): + func_def_type = get_column_type('storefront_scorefunction', 'definition') + if func_def_type != 'varchar(4096)': + print 'SCHEMA UPDATE: Enlarging storefront_scorefunction.definition column length to varchar(4096)' + + from django.db import connection, transaction + cursor = connection.cursor() + sql = 'ALTER TABLE `storefront_scorefunction` MODIFY COLUMN `definition` varchar(4096) DEFAULT NULL;' + cursor.execute(sql) + + transaction.commit_unless_managed() + + + +#def add_tag_slug(): +# if not column_exists('core_tag', 'slug'): +# print 'SCHEMA UPDATE: Adding missing column: core_tag.slug' +# +# from django.db import connection, transaction +# cursor = connection.cursor() +# sql = 'ALTER TABLE `core_tag` ADD COLUMN `slug` varchar(50) NULL' +# cursor.execute(sql) +# +# print 'DATA UPDATE: Normalizing core.Tag texts, and generating slugs.' +# sql = 'SELECT `id`, `text` FROM `core_tag`' +# cursor.execute(sql) +# for row in cursor: +# id = row[0] +# text = row[1] +# text = Tag.normalize(text) +# slug = slughifi(text) +# sql = 'UPDATE `core_tag` SET `text`=%s, `slug`=%s WHERE `id`=%s' +# cursor.execute(sql, (text, slug, id)) +# +# sql = 'ALTER TABLE `core_tag` MODIFY COLUMN `slug` varchar(50) NOT NULL' +# cursor.execute(sql) +# transaction.commit_unless_managed() diff --git a/storefront/updates/util.py b/storefront/updates/util.py new file mode 100644 index 0000000..dcc2f57 --- /dev/null +++ b/storefront/updates/util.py @@ -0,0 +1,35 @@ +def column_exists(table, column): + from django.db import connection, transaction + from django.db.backends.mysql import introspection + + cursor = connection.cursor() + desc = connection.introspection.get_table_description(cursor, table) + for x in desc: + if x[0] == column: + return True + return False + +def get_column_type(table, column): + from django.db import connection, transaction + + cursor = connection.cursor() + cursor.execute('desc %s' % table) + for r in cursor.cursor: + if r[0] == column: + return r[1] + return None + +def safe_add_column(table, column, sqlargs): + if not column_exists(table, column): + print 'SCHEMA UPDATE: Adding missing column: %s.%s %s' % (table, column, sqlargs) + from django.db import connection, transaction + cursor = connection.cursor() + sql = 'ALTER TABLE `%s` ADD COLUMN `%s` %s' % (table, column, sqlargs) + cursor.execute(sql) + +def makeNullable(table, column, columnTypeStr): + print 'SCHEMA UPDATE: changing column %s in table %s to nullable' % (column, table) + from django.db import connection, transaction + cursor = connection.cursor() + sql = 'ALTER TABLE `%s` MODIFY COLUMN `%s` %s NULL;' % (table,column,columnTypeStr) + cursor.execute(sql) diff --git a/storefront/urls.py b/storefront/urls.py new file mode 100644 index 0000000..22d5b67 --- /dev/null +++ b/storefront/urls.py @@ -0,0 +1,49 @@ +from django.conf.urls.defaults import * #@UnusedWildImport + +# Uncomment the next two lines to enable the admin: +#from django.contrib import admin +#admin.autodiscover() + +urlpatterns = patterns('', + url(r'^_static/(?P.*)$', 'django.views.static.serve', {'document_root': 'static'}, name='static'), + url(r'^$', 'storefront.views.root', name='root'), + url(r'^home$', 'storefront.views.home', name='home'), + url(r'^poweredby$', 'storefront.views.poweredby', name='poweredby'), + url(r'^packages$', 'storefront.views.packages', name='packages'), + url(r'^documentation/$', 'storefront.views.documentation', name='documentation'), + url(r'^documentation/(?P.*)$', 'storefront.views.documentation', name='documentation'), + url(r'^login$', 'storefront.views.login', name='login'), + url(r'^forgot-password$', 'storefront.views.forgot_password', name='forgot_password'), + url(r'^thanks-notice$', 'storefront.views.thanks_notice', name='thanks_notice'), + url(r'^change-password/$', 'storefront.views.change_password', name='change_password'), + #url(r'^invite_sign_up/(?P.*)$', 'storefront.views.invite_sign_up', name='invite_sign_up'), + #url(r'^beta-request$', 'storefront.views.beta_request', name='beta_request'), + #url(r'^get-started/$', 'storefront.views.get_started', name='get_started'), + #url(r'^upgrade/$', 'storefront.views.upgrade', name='upgrade'), + url(r'^why/$', 'storefront.views.why', name='why'), + url(r'^pricing/$', 'storefront.views.pricing', name='pricing'), + url(r'^enter-payment/$', 'storefront.views.enter_payment', name='enter_payment'), + url(r'^dashboard$', 'storefront.views.dashboard', name='dashboard'), + url(r'^heroku-dashboard$', 'storefront.views.heroku_dashboard', name='heroku-dashboard'), + url(r'^dashboard/insights/(?P[^/]*)$', 'storefront.views.insights', name='insights'), + url(r'^dashboard/manage/(?P[^/]*)$', 'storefront.views.manage_index', name='manage_index'), + url(r'^dashboard/inspect/(?P[^/]*)$', 'storefront.views.manage_inspect', name='manage_inspect'), + url(r'^create-index$', 'storefront.views.create_index', name='create_index'), + url(r'^close-account$', 'storefront.views.close_account', name='close_account'), + url(r'^delete-index$', 'storefront.views.delete_index', name='delete_index'), + url(r'^select-package$', 'storefront.views.select_package', name='select_package'), + url(r'^logout$', 'storefront.views.logout', name='logout'), + url(r'^score-functions$', 'storefront.views.score_functions', name='score_functions'), + url(r'^remove-function$', 'storefront.views.remove_function', name='remove_function'), + url(r'^search/$', 'storefront.views.search', name='search'), + url(r'^quotes$', 'storefront.views.quotes', name='quotes'), + + url(r'^provider/resources/(?P.*)$', 'storefront.views.sso', name='sso'), + url(r'^heroku/resources/(?P.*)$', 'storefront.views.sso_heroku', name='sso_heroku'), + url(r'^demoindex$', 'storefront.views.demo_index', name='demoindex'), + + url(r'^accounting/register-index$', 'storefront.views.api_register_index'), + url(r'^accounting/delete-index$', 'storefront.views.api_delete_index'), + url(r'^accounting/list-indexes$', 'storefront.views.api_list'), + +) diff --git a/storefront/util.py b/storefront/util.py new file mode 100644 index 0000000..f192aa3 --- /dev/null +++ b/storefront/util.py @@ -0,0 +1,169 @@ +from django import shortcuts +from django.utils.translation import check_for_language, activate +from django.contrib.auth.decorators import login_required as dj_login_required +from django.http import HttpResponseRedirect, HttpResponseForbidden +from django.contrib import auth +from django.contrib.auth.models import User +from django.utils.http import urlquote +import socket +import random +import base64 +import re +from django.template import RequestContext, loader +from django.conf import settings +#from ncrypt.cipher import CipherType, EncryptCipher, DecryptCipher +import binascii +import random +import hashlib +from lib import encoder +from django.contrib import messages +from templatetags.google_analytics import SCOPE_VISITOR, SCOPE_SESSION, SCOPE_PAGE + +extra_context = {} +def render(template, request, context_dict={}): + ab_test_suffix = _ab_test_suffix(template, request) + context = _getEnhacedRequestContext(request, context_dict) + _fill_analytical_info(request, context, ab_test_suffix) + if ab_test_suffix: + return shortcuts.render_to_response(str(template) + str(ab_test_suffix), {}, context) + else: + return shortcuts.render_to_response(template, {}, context) + +def _ab_test_suffix(template, request): + try: + v = request.COOKIES['ab_test_version'] + #this is paranoid, just to prevent injection. + if not v == '.A': + v = '.B' + print 'already there' + except KeyError: + if (random.randint(0,1) == 0): + v = '.A' + else: + v = '.B' + request.COOKIES.set('ab_test_version', str(v)) + print 'setting it' + print 'template cookie is "%s"' % (v) + try: + loader.find_template(template + '.A') + loader.find_template(template + '.B') + print 'template alternatives found. Using them.' + return v + except: + print 'no template alternatives. Using base template' + return None + +def _getEnhacedRequestContext(request, context_dict): + #this constructor calls all the configured processors. + context = RequestContext(request, dict=context_dict) + + #the rest of this method should be probably rewritten an custom processors. + global extra_context + context['request'] = request + + #django.contrib.auth.context_processors.auth fills user. We should check + #that it does what we need. + context['user'] = request.user + + #this variables are used by the analytical package + if request.user.is_authenticated(): + context['google_analytics_var1'] = ('visitor', '0', SCOPE_SESSION) + else: + context['google_analytics_var1'] = ('visitor', '1', SCOPE_SESSION) + + context['messages'] = messages.get_messages(request) + context['LOCAL'] = settings.LOCAL + + if request.user.is_authenticated(): + if request.user.get_profile().account.provisioner: + context['provisioner'] = request.user.get_profile().account.provisioner.name + elif request.user.get_profile().account.package.code.startswith('HEROKU_'): + # HACK UNTIL HEROKU IS SET UP AS A PROVISIONER + context['provisioner'] = 'heroku' + + + for k in extra_context: + if hasattr(extra_context[k], '__call__'): + context[k] = extra_context[k]() + else: + context[k] = extra_context[k] + return context + + +def _fill_analytical_info(request, context, ab_test_suffix): + sp = {} + e = [] + e.append(('pageview-' + request.path, {})) + if request.user.is_authenticated(): + if request.user.get_profile().account.provisioner: + sp['account_source'] = request.user.get_profile().account.provisioner.name + sp['logged_in'] = 'true' + sp['with_account'] = 'true' + else: + sp['logged_in'] = 'false' + if context.get('plan'): + sp['plan'] = context.get('plan') + if ab_test_suffix: + sp['variant'] = ab_test_suffix + if request.method == 'GET': + if 'utm_campaign' in request.GET: + sp['utm_campaign'] = str(request.GET['utm_campaign']) + if 'utm_medium' in request.GET: + sp['utm_medium'] = str(request.GET['utm_medium']) + if 'utm_source' in request.GET: + sp['utm_source'] = str(request.GET['utm_source']) + if 'ad' in request.GET: + sp['ad'] = str(request.GET['ad']) + context['mixpanel'] = {} + context['mixpanel']['super_properties'] = sp + context['mixpanel']['events'] = e + + +MOBILE_PATTERN_1 = '/android|avantgo|blackberry|blazer|compal|elaine|fennec|hiptop|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile|o2|opera mini|palm( os)?|plucker|pocket|pre\/|psp|smartphone|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce; (iemobile|ppc)|xiino/i' +MOBILE_PATTERN_2 = '/1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|e\-|e\/|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(di|rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|xda(\-|2|g)|yas\-|your|zeto|zte\-/i' + +def domain(): + if settings.WEBAPP_PORT == 80: + return settings.COMMON_DOMAIN + else: + return "%s:%d" % (settings.COMMON_DOMAIN, settings.WEBAPP_PORT) + +def is_internal_navigation(request): + ref = request.META.get('HTTP_REFERER') + if ref: + pattern = r'http://[^/]*\.' + domain().replace('.', r'\.') + if re.match(pattern, ref): + return True + else: + return False + else: + return False + +def is_mobile(request): + user_agent = request.META['HTTP_USER_AGENT'].lower() + match1 = re.search(MOBILE_PATTERN_1, user_agent) + match2 = re.search(MOBILE_PATTERN_2, user_agent[:4]) + return match1 or match2 + + +def render_to_response(template, context, *args, **kwargs): + '''TODO: move this to a processor''' + original_template = template + if is_mobile(context['request']): + parts = template.split('/') + parts[-1] = 'mobile.' + parts[-1] + template = '/'.join(parts) + return shortcuts.render_to_response((template, original_template), context, *args, **kwargs) + +def login_required(view, *args, **kwargs): + dj_view = dj_login_required(view, *args, **kwargs) + def decorated(request, *args, **kwargs): + return dj_view(request, *args, **kwargs) + return decorated + + +def get_index_code(id): + return encoder.to_key(id) + +def get_index_id_for_code(code): + return encoder.from_key(code); diff --git a/storefront/views.py b/storefront/views.py new file mode 100644 index 0000000..cc39d2e --- /dev/null +++ b/storefront/views.py @@ -0,0 +1,1325 @@ +from django.contrib.auth.models import User +from django.contrib import auth + +from util import render, login_required, get_index_code, get_index_id_for_code +from models import PFUser, Account, Package, Index, ScoreFunction, BetaTestRequest, BetaInvitation, AccountPayingInformation, PaymentSubscription, ContactInfo, BlogPostInfo +import forms +from django.http import HttpResponseRedirect, Http404, HttpResponseNotFound,\ + HttpResponse, HttpResponseForbidden, HttpResponsePermanentRedirect,\ + HttpResponseBadRequest +from django.db import IntegrityError +from datetime import timedelta +import datetime +import time +from forms import IndexForm, ScoreFunctionForm, BetaTestForm +from django.core.urlresolvers import reverse +from django.template import TemplateDoesNotExist, loader +from lib.indextank.client import ApiClient, IndexAlreadyExists, TooManyIndexes, InvalidDefinition, InvalidQuery +from django.db.models import Max +from django.utils import simplejson as json + +from models import generate_forgotpass + +from lib.authorizenet import AuthorizeNet, BillingException +from lib import mail, encoder + +import hashlib +import urllib, urllib2 + +import os +from django.conf import settings + +from flaptor.indextank.rpc import DeployManager as TDeployManager + +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol +from thrift.server import TServer + +from django.core.mail import send_mail +from django.forms.models import ModelForm +from django.contrib import messages +from django.contrib.auth.management.commands.createsuperuser import is_valid_email + +import csv +from django.template.context import Context + +class JsonResponse(HttpResponse): + def __init__(self, json_object, *args, **kwargs): + body = json.dumps(json_object) + if 'callback' in kwargs: + callback = kwargs.pop('callback') + if callback: + body = '%s(%s)' % (callback, body) + super(JsonResponse, self).__init__(body, *args, **kwargs) + +def force_https(func): + if settings.DEBUG: + return func + def wrapped_func(request,*args,**kwargs): + if request.is_secure(): + return func(request,*args,**kwargs) + else: + return HttpResponsePermanentRedirect("https://%s%s" % (request.get_host(), request.get_full_path())) + return wrapped_func + +def root(request): + if request.user.is_authenticated(): + return HttpResponseRedirect(reverse('dashboard')) + return home(request) + +def poweredby(request): + #Eventually we may want to track this links separated from the rest of the page. + return HttpResponseRedirect(reverse('root')) + +def home(request): + if request.method == 'POST': + name = request.POST.get('name') + email = request.POST.get('email') + if name and email: + try: + ci = ContactInfo() + ci.name = name + ci.email = email + ci.source = 'contestinfo@home' + ci.save() + messages.success(request, 'We\'ve added %s. Thanks for subscribing for details on our upcoming contests.' % email) + return HttpResponseRedirect(reverse('home')) + except: + messages.error(request, 'You need to enter both your name and email.') + else: + messages.error(request, 'You need to enter both your name and email') + + blog_posts = [] + + try: + blog_posts_obj = BlogPostInfo.objects.all().order_by('-date')[:3] + for post in blog_posts_obj: + blog_posts.append({ + 'url':post.url, + 'title':post.title, + 'date':post.date.strftime('%B %d'), + 'author':post.author}) + + except: + pass + + return render('home.html', request, context_dict={'navigation_pos': 'home', 'blog_posts': blog_posts}) + +def pricing(request): + return render('pricing.html', request, context_dict={'navigation_pos': 'pricing',}) + +def packages(request): + return render('packages.html', request, context_dict={'navigation_pos': 'home'}) + +def quotes(request): + return render('quotes.html', request, context_dict={'navigation_pos': 'home'}) + +def documentation(request, path='documentation'): + if '.' in path[-5:]: + template = 'documentation/%s' % path + else: + template = 'documentation/%s.html' % path + try: + return render(template, request, context_dict={'navigation_pos': 'documentation'}) + except TemplateDoesNotExist: + return render('coming-soon.html', request, context_dict={'navigation_pos': 'documentation'}) + +def search(request): + if request.method == 'GET': + query = request.GET.get('query').strip() + context = {'query': query} + return render('search_results.html', request, context_dict=context) + return HttpResponseRedirect(request.META.get('HTTP_REFERER')) + + +@force_https +def login(request): + login_form = forms.LoginForm() + login_message = '' + if request.method == 'POST': + login_form = forms.LoginForm(data=request.POST) + if login_form.is_valid(): + try: + email = login_form.cleaned_data['email'] + if email == 'apiurl@indextank.com': + username = email + else: + username = PFUser.objects.get(email=email).user.username + user = auth.authenticate(username=username, password=login_form.cleaned_data['password']) + if user is not None: + if user.is_active: + auth.login(request, user) + return HttpResponseRedirect(request.GET.get('next') or '/dashboard'); + else: + login_message = 'Account disabled' + else: + login_message = 'Wrong email or password' + except PFUser.DoesNotExist: #@UndefinedVariable + login_message = 'Wrong email or password' + + context = { + 'login_form': login_form, + 'login_message': login_message, + 'navigation_pos': 'home', + 'next': request.GET.get('next') or '/dashboard', + } + + return render('login.html', request, context_dict=context) + +@force_https +def forgot_password(request): + forgot_form = forms.ForgotPassForm() + message = '' + if request.method == 'POST': + forgot_form = forms.ForgotPassForm(data=request.POST) + if forgot_form.is_valid(): + try: + pfuser = PFUser.objects.get(email=forgot_form.cleaned_data['email']) + if pfuser is not None: + if pfuser.user.is_active: + new_pass = generate_forgotpass(pfuser.id) + pfuser.user.set_password(new_pass) + pfuser.user.save() + pfuser.change_password = True + pfuser.save() + send_mail('IndexTank password reset', 'Your IndexTank password has been reset to: ' + new_pass, 'IndexTank ', [pfuser.email], fail_silently=False) + messages.success(request, 'Password reset successfully. Check your email inbox.') + return HttpResponseRedirect(reverse('login')) + else: + message = 'Account disabled' + else: + message = 'The email address does not belong to an IndexTank account' + except PFUser.DoesNotExist: #@UndefinedVariable + message = 'The email address does not belong to an IndexTank account' + + context = { + 'forgot_form': forgot_form, + 'message': message, + 'navigation_pos': 'home', + } + + return render('forgot_pass.html', request, context_dict=context) + +def default_sso_account_fetcher(id): + return Account.objects.get(apikey__startswith=id+'-') +def heroku_sso_account_fetcher(id): + return Account.objects.get(id=id) + +def sso_heroku(request, id): + return sso(request, id, fetcher=heroku_sso_account_fetcher) + +def sso(request, id, fetcher=default_sso_account_fetcher): + ''' + SSO for provisioners + ''' + timestamp = int(request.GET.get('timestamp',0)) + token = request.GET.get('token','') + calculated_token = hashlib.sha1("%s:%s:%s"%(id,'D9YmWpRfZv0pJn05',timestamp)).hexdigest() + # check token + if token != calculated_token: + return HttpResponseForbidden("token") + + # token expire on a 5 minute window + if abs(int(time.time()) - timestamp) > 300: + return HttpResponseForbidden("expired") + + # so, just log him in. + account = fetcher(id) + user = auth.authenticate(username=account.user.user.username, password=account.apikey.split('-', 1)[1]) + if user is not None: + if user.is_active: + auth.login(request,user) + + cookies = {} + request.session['provisioner_navbar_html'] = '' + #if account.provisioner and account.provisioner.name == 'heroku': + # HACK TO SUPPORT HEROKU TRANSITION (UNTIL IT's A PROVISIONER) + if fetcher == heroku_sso_account_fetcher: + # fetch heroku css and html nav bar + hrequest = urllib2.Request('http://nav.heroku.com/v1/providers/header') + hrequest.add_header('Accept','application/json') + data = urllib2.urlopen(hrequest).read() + if data: + request.session['provisioner_navbar_html'] = data + cookies['heroku-nav-data'] = request.GET.get('nav-data', '') + if account.provisioner and account.provisioner.name == 'appharbor': + # fetch heroku css and html nav bar + hrequest = urllib2.Request('http://appharbor.com/header') + #jsonrequest.add_header('Accept','application/json') + data = urllib2.urlopen(hrequest).read() + if data: + request.session['provisioner_navbar_html'] = data + cookies['appharbor-nav-data'] = request.GET.get('nav-data', '') + + #request.session['heroku'] = True + response = HttpResponseRedirect('/dashboard') + for k,v in cookies.items(): + max_age = 365*24*60*60 #one year + expires = datetime.datetime.strftime(datetime.datetime.utcnow() + datetime.timedelta(seconds=max_age), "%a, %d-%b-%Y %H:%M:%S GMT") + response.set_cookie(k, v, max_age=max_age, expires=expires, domain=settings.SESSION_COOKIE_DOMAIN, secure=settings.SESSION_COOKIE_SECURE or None) + return response + + return HttpResponseForbidden() + + +def do_sign_up(request, form_data, package=None, invitation=None): + dt = datetime.datetime.now() + email = form_data.get('email') + password = form_data.get('password') + account, pfu = Account.create_account(dt, email, password) + if invitation: + invitation.account = account + invitation.save() + account.apply_package(invitation.forced_package) + else: + account.apply_package(package) + + account.save() + + user = auth.authenticate(username=pfu.user.username, password=password) + auth.login(request, user) + + return account + +@force_https +def upgrade(request): + pass + +@force_https +def get_started(request): + if request.user.is_authenticated(): + package = request.user.get_profile().account.package + plan = package.code + else: + plan = request.GET.get('plan', 'FREE') + package = Package.objects.get(code=plan) + if request.is_ajax() and request.method == 'POST': + email = request.POST['email'] + try: + is_valid_email(email) + except: + return HttpResponse('Invalid email address', status=400) + + try: + account = create_account(request, email, package) + except IntegrityError: + return HttpResponse('Email address already used', status=400) + + return JsonResponse({'private_api_url':account.get_private_apiurl(), 'public_api_url':account.get_public_apiurl(), 'email': email}) + + context = { + 'navigation_pos': 'get_started', + 'package': package, + 'plan': plan + } + return render('get_started.html', request, context_dict=context) + +def why(request): + context = { + 'navigation_pos': 'why_us', + } + return render('why_us.html', request, context_dict=context) + + +def create_account(request, email, package): + dt = datetime.datetime.now() + account, pfu = Account.create_account(dt, email) + account.apply_package(package) + + # Demo index creation + account.create_demo_index() + + if account.package.base_price == 0: + account.status = Account.Statuses.operational + + mail.report_new_account(account) + account.save() + password=account.apikey.split('-', 1)[1] + + user = auth.authenticate(username=pfu.user.username, password=password) + auth.login(request, user) + + send_mail('Welcome to IndexTank!', 'Thanks signing-up for IndexTank!\nYour password for logging in to your dashboard is %s' % (password), 'IndexTank ', [email], fail_silently=False) + return account + +@force_https +def old_get_started(request): + if request.user.is_authenticated(): + account = request.user.get_profile().account + if account.package.base_price == 0 or account.payment_informations.count(): + logout(request) + + plan = request.GET.get('plan') + if plan is None: + return _get_started_step1(request) + else: + return _get_started_step2(request, plan) + +def _get_started_step3(request): + messages.success(request, "Great! You have successfully created an IndexTank account.") + return HttpResponseRedirect(reverse('dashboard')) + + +@login_required +def enter_payment(request): + message = None + account = request.user.get_profile().account + package = account.package + plan = account.package.code + data = request.POST if request.method == 'POST' else None + + if package.base_price == 0: + return HttpResponseRedirect(reverse('dashboard')) + + if account.payment_informations.count() > 0: + messages.info(request, "You have already entered your payment information. If you wish to change it please contact us.") + return HttpResponseRedirect(reverse('dashboard')) + + payment_form = forms.PaymentInformationForm(data=data) + + if data: + if payment_form.is_valid(): + form_data = payment_form.cleaned_data + try: + process_payment_information(account, form_data) + account.status = Account.Statuses.operational + account.save() + + #mail.report_payment_data(account) + + return _get_started_step3(request) + except BillingException, e: + message = e.msg + if message is None: + messages.success(request, "You have successfully entered your payment information.") + return HttpResponseRedirect(reverse('dashboard')) + context = { + 'navigation_pos': 'get_started', + 'payment_form': payment_form, + 'message': message, + 'step': '2', + 'package': package, + } + return render('enter_payment.html', request, context_dict=context) + + +def _get_started_step2(request, plan): + message = None + data = request.POST if request.method == 'POST' else None + account = None + package = Package.objects.get(code=plan) + if request.user.is_authenticated(): + sign_up_form = None + account = request.user.get_profile().account + account.apply_package(package) + account.save() + if package.base_price > 0: + payment_form = forms.PaymentInformationForm(data=data) + else: + return _get_started_step3(request) + else: + sign_up_form = forms.SignUpForm(data=data) + if package.base_price > 0: + payment_form = forms.PaymentInformationForm(data=data) + else: + payment_form = None + + if data: + sign_up_ok = sign_up_form is None or sign_up_form.is_valid() + payment_ok = payment_form is None or payment_form.is_valid() + if sign_up_ok and payment_ok: + if sign_up_form is not None: + form_data = sign_up_form.cleaned_data + try: + account = do_sign_up(request, form_data, package) + sign_up_form = None + except IntegrityError, e: + message = 'Email already exists.' + if message is None and payment_form is not None: + form_data = payment_form.cleaned_data + try: + process_payment_information(account, form_data) + account.status = Account.Statuses.operational + account.save() + + mail.report_new_account(account) + + return _get_started_step3(request) + except BillingException, e: + message = e.msg + if message is None: + return _get_started_step3(request) + + context = { + 'navigation_pos': 'get_started', + 'sign_up_form': sign_up_form, + 'payment_form': payment_form, + 'message': message, + 'step': '2', + 'package': package, + 'next': request.GET.get('next') or '/', + } + return render('get_started.html',request, context_dict=context) + +def _get_started_step1(request): + context = { + 'navigation_pos': 'get_started', + 'step': '1', + 'step_one': True, + 'next': request.GET.get('next') or '/', + } + return render('get_started.html', request, context_dict=context) + +def beta_request(request): + form = None + message = None + if request.method == 'GET': + form = forms.BetaTestForm() + else: + form = forms.BetaTestForm(data=request.POST) + if form.is_valid(): + form_data = form.cleaned_data + try: + email = form_data.get('email') + summary = form_data.get('summary') + site_url = form_data.get('site_url') + + beta_request = BetaTestRequest(site_url=site_url, email=email, summary=summary) + beta_request.save() + + send_mail('IndexTank beta testing account', 'Thanks for requesting a beta testing account for IndexTank! We\'ll get back to you shortly', 'IndexTank ', [email], fail_silently=False) + + #do_sign_up(request, form_data) + return HttpResponseRedirect(reverse('thanks_notice')) + except IntegrityError, e: + message = 'Email already used.' + + context = { + 'request_form': form, + 'message': message, + 'navigation_pos': 'beta_request', + 'next': request.GET.get('next') or '/', + } + + return render('beta_request.html', request, context_dict=context) + +def thanks_notice(request): + return render('thanks_notice.html', request) + +def invite_sign_up(request, password=None): + try: + invitation = BetaInvitation.objects.get(password=password) + if invitation.account: + return render('used_invite.html', request) + except BetaInvitation.DoesNotExist: + return HttpResponseNotFound() + + form = None + message = None + if request.method == 'GET': + if invitation.beta_requester: + form = forms.SignUpForm(initial={'email':invitation.beta_requester.email}) + else: + form = forms.SignUpForm() + else: + form = forms.SignUpForm(data=request.POST) + if form.is_valid(): + form_data = form.cleaned_data + try: + do_sign_up(request, form_data, invitation) + return HttpResponseRedirect(request.GET.get('next') or '/') + except IntegrityError, e: + message = 'Email already exists.' + + context = { + 'invitation': invitation, + 'sign_up_form': form, + 'message': message, + 'navigation_pos': 'get_started', + 'next': request.GET.get('next') or '/', + } + + return render('sign_up.html', request, context_dict=context) + +@force_https +def sign_up(request, package=None): + account_package = None + + if package: + account_package = Package.objects.get(code=package) + + if request.user.is_authenticated(): + account = request.user.get_profile().account + if account.payment_informations.count(): + raise Http404 + else: + account.apply_package(account_package) + return HttpResponseRedirect(reverse('dashboard')) + + + form = None + message = None + if request.method == 'GET': + form = forms.SignUpForm() + else: + form = forms.SignUpForm(data=request.POST) + if form.is_valid(): + form_data = form.cleaned_data + try: + do_sign_up(request, form_data, package=account_package) + return HttpResponseRedirect(request.GET.get('next') or '/') + except IntegrityError, e: + message = 'Email already exists.' + + context = { + 'sign_up_form': form, + 'message': message, + 'navigation_pos': 'get_started', + 'next': request.GET.get('next') or '/', + 'package': package, + } + + return render('sign_up.html', request, context_dict=context) + +@login_required +@force_https +def close_account(request): + user = request.user + message = None + if request.method == 'GET': + form = forms.CloseAccountForm() + else: + form = forms.CloseAccountForm(data=request.POST) + + if form.is_valid(): + form_data = form.cleaned_data + + password = form_data.get('password') + + if user.check_password(password): + user.get_profile().account.close() + return HttpResponseRedirect(reverse('logout')) + else: + message = 'Wrong password' + + context = { + 'form': form, + 'message': message, + 'navigation_pos': 'dashboard', + } + + return render('close_account.html', request, context_dict=context) + + +@login_required +@force_https +def change_password(request): + user = request.user + message = None + if request.method == 'GET': + form = forms.ChangePassForm() + else: + form = forms.ChangePassForm(data=request.POST) + + if form.is_valid(): + form_data = form.cleaned_data + + old_pass = form_data.get('old_password') + new_pass = form_data.get('new_password') + new_pass_again = form_data.get('new_password_again') + + if user.check_password(old_pass): + user.set_password(new_pass) + user.save() + user.get_profile().change_password = False + user.get_profile().save() + messages.success(request, 'Your password was changed successfully.') + return HttpResponseRedirect(reverse('dashboard')) + else: + message = 'Current password is wrong' + + context = { + 'form': form, + 'message': message, + 'navigation_pos': 'dashboard', + } + + return render('change_password.html', request, context_dict=context) + + +@login_required +def dashboard(request): + # Possible statuses: + # - No index + # - Index but no docs + # - Index with docs + + account_status = None + + if request.user.get_profile().change_password: + messages.info(request, 'Your password was reset and you need to change it.') + return HttpResponseRedirect(reverse('change_password')) + + account = request.user.get_profile().account + + #if not account.package: + # return HttpResponseRedirect(reverse('select_package')) + if not account.status == Account.Statuses.operational and not account.payment_informations.count(): + if account.package.base_price > 0: + messages.info(request, 'Before accessing your dashboard you need to enter your payment information') + return HttpResponseRedirect(reverse('enter_payment')) + elif account.status == Account.Statuses.creating: + account.status = Account.Statuses.operational + + mail.report_new_account(account) + account.save() + else: + return HttpResponseRedirect(reverse('logout')) + + indexes = account.indexes.filter(deleted=False) + + has_indexes_left = (len(indexes) < account.package.max_indexes) + + totals = dict(size=0, docs=0, qpd=0) + for index in indexes: + totals['docs'] += index.current_docs_number + totals['size'] += index.current_size + totals['qpd'] += index.queries_per_day + + if len(indexes) == 0: + account_status = 'NOINDEX' + elif totals['docs'] == 0: + account_status = 'INDEXNODOCS' + else: + account_status = 'INDEXWITHDOCS' + + percentages = {} + def add_percentage(k, max, t, p): + p[k] = 100.0 * t[k] / max + + KB = 1024 + MB = KB * KB + max_docs = account.package.index_max_size + max_size = account.package.max_size_mb() + max_qpd = account.package.searches_per_day + + add_percentage('docs', max_docs, totals, percentages) + add_percentage('size', max_size, totals, percentages) + add_percentage('qpd', max_qpd, totals, percentages) + + for index in indexes: + insights = {} + insights_update = {} + #for i in index.insights.all(): + # try: + # insights[i.code] = json.loads(i.data) + # insights_update[i.code] = i.last_update + # except: + # print 'Failed to load insight %s for %s' % (i.code, index.code) + #index.insights_map = insights + #index.insights_update = insights_update + + context = { + 'account': account, + 'indexes': indexes, + 'has_indexes_left': has_indexes_left, + 'account_status': account_status, + 'totals': totals, + 'percentages': percentages, + 'navigation_pos': 'dashboard', + } + + return render('dashboard.html', request, context_dict=context) + +@login_required +def heroku_dashboard(request): + # Possible statuses: + # - No index + # - Index but no docs + # - Index with docs + + account_status = None + + if request.user.get_profile().change_password: + messages.info(request, 'Your password was reset and you need to change it.') + return HttpResponseRedirect(reverse('change_password')) + + account = request.user.get_profile().account + + indexes = account.indexes.all() + + has_indexes_left = (len(indexes) < account.package.max_indexes) + + totals = dict(size=0, docs=0, qpd=0) + for index in indexes: + totals['docs'] += index.current_docs_number + totals['size'] += index.current_size + totals['qpd'] += index.queries_per_day + + if len(indexes) == 0: + account_status = 'NOINDEX' + elif totals['docs'] == 0: + account_status = 'INDEXNODOCS' + else: + account_status = 'INDEXWITHDOCS' + + percentages = {} + def add_percentage(k, max, t, p): + p[k] = 100.0 * t[k] / max + + KB = 1024 + MB = KB * KB + max_docs = account.package.index_max_size + max_size = account.package.max_size_mb() + max_qpd = account.package.searches_per_day + + add_percentage('docs', max_docs, totals, percentages) + add_percentage('size', max_size, totals, percentages) + add_percentage('qpd', max_qpd, totals, percentages) + + context = { + 'account': account, + 'indexes': indexes, + 'has_indexes_left': has_indexes_left, + 'account_status': account_status, + 'totals': totals, + 'percentages': percentages, + 'navigation_pos': 'dashboard', + } + + return render('heroku-dashboard.html', request, context_dict=context) + + +@login_required +@force_https +def enter_payment_information(request): + account = request.user.get_profile().account + + if request.method == 'GET': + form = forms.PaymentInformationForm() + else: + form = forms.PaymentInformationForm(data=request.POST) + if form.is_valid(): + form_data = form.cleaned_data + + try: + if account.package.base_price > 0: + process_payment_information(account, form_data) + + account.status = Account.Statuses.operational + mail.report_new_account(account) + account.save() + + return HttpResponseRedirect(reverse('dashboard')) + except BillingException, e: + messages.error(request, e.msg) + + + context = { + 'form': form, + 'navigation_pos': 'get_started', + 'next': request.GET.get('next') or '/', + 'account': account, + } + + return render('payment_info.html', request, context_dict=context) + +def process_payment_information(account, form_data): + + payment_infos = account.payment_informations.all() + + # Right now there can only be ONE payment info per account + if payment_infos: + pass + else: + payment_info = AccountPayingInformation() + + payment_info.account = account + + payment_info.first_name = form_data['first_name'] + payment_info.last_name = form_data['last_name'] + payment_info.address = form_data['address'] + payment_info.city = form_data['city'] + payment_info.state = form_data['state'] + payment_info.zip_code = form_data['zip_code'] + payment_info.country = form_data['country'] + + payment_info.contact_email = account.user.email + payment_info.monthly_amount = str(account.package.base_price) + payment_info.subscription_status = 'Active' + payment_info.subscription_type = 'Authorize.net' + + cc_number = form_data['credit_card_number'] + exp_month, exp_year = form_data['exp_month'].split('/', 1) + payment_info.credit_card_last_digits = cc_number[-4:] + + auth = AuthorizeNet() + + # add one day to avoid day change issues + today = (datetime.datetime.now() + timedelta(days=1)).strftime('%Y-%m-%d') + payment_info.save() + + ref_id = str(payment_info.id) + freq_length = 1 + freq_unit = 'months' + + try: + subscription_id = auth.subscription_create(ref_id, 'IndexTank - ' + account.package.name, str(freq_length), freq_unit, today, '9999', '1', + "%.2f" % account.package.base_price, '0', cc_number, '20' + exp_year + '-' + exp_month, payment_info.first_name, payment_info.last_name, + "", payment_info.address, payment_info.city, payment_info.state, payment_info.zip_code, payment_info.country) + except BillingException, e: + payment_info.delete() + raise e + except Exception, e: + payment_info.delete() + raise BillingException('An error occurred when verifying the credit card. Please try again later') + + payment_subscription = PaymentSubscription() + + payment_subscription.account = payment_info + payment_subscription.subscription_id = subscription_id + payment_subscription.reference_id = ref_id + payment_subscription.amount = str(account.package.base_price) + + payment_subscription.start_date = today + payment_subscription.frequency_length = freq_length + payment_subscription.frequency_unit = freq_unit + + payment_subscription.save() + + #### CREATE IN AUTHORIZE NET + + + +@login_required +def insights(request, index_code=None): + index = Index.objects.get(code=index_code) + insights = {} + insights_update = {} + for i in index.insights.all(): + try: + insights[i.code] = json.loads(i.data) + insights_update[i.code] = i.last_update + except: + print 'Failed to load insight %s for %s' % (i.code, index.code) + + context = { + 'account': request.user.get_profile().account, + 'navigation_pos': 'dashboard', + 'index': index, + 'insights': insights, + 'insights_update': insights_update, + 'index_code': index_code, + } + + return render('insights.html', request, context_dict=context) + + +@login_required +def manage_index(request, index_code=None): + account = request.user.get_profile().account + + index = Index.objects.get(code=index_code) + + if index: + if index.account == account: + if request.method == 'GET': + index = Index.objects.get(code=index_code) + + largest_func = max([int(f.name) + 1 for f in index.scorefunctions.all()] + [5]) + functions = get_functions(index, upto=largest_func) + + context = { + 'account': request.user.get_profile().account, + 'navigation_pos': 'dashboard', + 'functions': functions, + 'index': index, + 'index_code': index_code, + 'largest_func': largest_func + } + + if 'query' in request.GET: + maxim = int(request.GET.get('max', '25')) + index_client = ApiClient(account.get_private_apiurl()).get_index(index.name) + context['results'] = index_client.search(request.GET['query'], length=max) + context['query'] = request.GET['query'] + context['more'] = maxim + 25 + + + return render('manage_index.html', request, context_dict=context) + else: + if 'definition' in request.POST: + name = request.POST['name'] + definition = request.POST['definition'] + + client = ApiClient(account.get_private_apiurl()).get_index(index.name) + try: + if definition: + client.add_function(int(name), definition) + else: + client.delete_function(int(name)) + except InvalidDefinition, e: + return HttpResponse('Invalid function', status=400) + + return JsonResponse({'largest': 5}) + elif 'public_api' in request.POST: + index.public_api = request.POST['public_api'] == 'true' + index.save() + return JsonResponse({'public_api': index.public_api}) + else: + raise HttpResponseForbidden + + else: + raise Http404 + +functions_number = 6 +@login_required +def manage_inspect(request, index_code=None): + account = request.user.get_profile().account + + index = Index.objects.get(code=index_code) + + if index: + if index.account == account: + context = { + 'account': request.user.get_profile().account, + 'navigation_pos': 'dashboard', + 'index_code': index_code, + 'index': index + } + + return render('manage_inspect.html', request, context_dict=context) + else: + raise HttpResponseForbidden + else: + raise Http404 + + + +@login_required +def score_functions(request): + #TODO: make it part of index/package configuration + account = request.user.get_profile().account + if request.method == 'GET': + index_code = request.GET['index_code'] + index = Index.objects.get(code=index_code) + + functions = get_functions(index) + + form = ScoreFunctionForm() + + context = { + 'form': form, + 'account': request.user.get_profile().account, + 'navigation_pos': 'dashboard', + 'functions': functions, + 'index_code': index_code, + 'functions_available': len(functions) < functions_number, + } + + return render('score_functions.html', request, context_dict=context) + else: + form = ScoreFunctionForm(data=request.POST) + + if form.is_valid(): + index_code = request.POST['index_code'] + index = Index.objects.get(code=index_code) + name = form.cleaned_data['name'] + definition = form.cleaned_data['definition'] + + client = ApiClient(account.get_private_apiurl()).get_index(index.name) + try: + client.add_function(int(name), definition) + except InvalidDefinition, e: + index = Index.objects.get(code=index_code) + functions = get_functions(index) + form = ScoreFunctionForm(initial={'name': name, 'definition': definition}) + messages.error(request, 'Problem processing your formula: %s', str(e)) + + context = { + 'form': form, + 'account': request.user.get_profile().account, + 'navigation_pos': 'dashboard', + 'functions': functions, + 'index_code': index_code, + 'functions_available': len(functions) < functions_number, + } + + return render('score_functions.html', request, context_dict=context) + + return HttpResponseRedirect(reverse('score_functions') + '?index_code=' + index_code) + +@login_required +def remove_function(request): + account = request.user.get_profile().account + if request.method == 'GET': + index_code = request.GET['index_code'] + index = Index.objects.get(code=index_code) + function_name = request.GET['function_name'] + client = ApiClient(account.get_private_apiurl()).get_index(index.name) + client.delete_function(function_name) + + return HttpResponseRedirect(reverse('score_functions') + '?index_code=' + index_code) + +def get_functions(index, upto=5): + functions = index.scorefunctions.order_by('name') + functions_dict = {} + final_functions = [] + + for function in functions: + functions_dict[function.name] = function + + max_key = 0 + if functions_dict.keys(): + max_key = max(functions_dict.keys()) + + for i in xrange(upto+1): + pos = i + if pos in functions_dict: + final_functions.append(functions_dict[pos]) + else: + new_function = ScoreFunction(name=str(pos), definition=None) + final_functions.append(new_function) + + return final_functions + + +def select_package(request): + account = request.user.get_profile().account + if request.method == 'GET': + packages_list = Package.objects.all() + packages = {} + package_availability = {} + for package in packages_list: + packages[package.code] = package + package_availability[package.code] = package.max_indexes >= Index.objects.filter(account=account).count() + + context = { + 'account': account, + 'packages': packages, + 'package_availability': package_availability, + 'navigation_pos': 'dashboard', + } + return render('packages.html', request, context_dict=context) + else: + package = Package.objects.get(id=request.POST['package_id']) + account.apply_package(package) + account.save() + return HttpResponseRedirect(reverse('dashboard')) + + + +@login_required +def demo_index(request): + ''' + Renders the demo frontend for INSTRUMENTS index. + if we want to do it for every index, sometime in the future, + just add an 'index=' parameter to this view. + ''' + account = request.user.get_profile().account + for index in account.indexes.all(): + + if index.is_demo(): + context = { + 'index': index + } + return render('instruments.html', request, context_dict=context) + # else continue + + # no index -> 404 + return render("404.html", request) + + + + +# Search API hack +BASE_URL = 'http://api.indextank.com/api/v0' +def call_api_delete(index): + url = BASE_URL + '/inform_del_index?apikey=' + urllib.quote(index.account.apikey) + '&indexcode=' + urllib.quote(index.code) + data = urllib.urlopen(url).read() + +def call_api_create(index): + url = BASE_URL + '/inform_add_index?apikey=' + urllib.quote(index.account.apikey) + '&indexcode=' + urllib.quote(index.code) + data = urllib.urlopen(url).read() + +# End hack + +def delete_index(request): + if request.method == 'GET': + return HttpResponseRedirect(reverse('dashboard')) + else: + index = Index.objects.get(id=request.POST['index_id']) + + index_client = ApiClient(index.account.get_private_apiurl()).get_index(index.name) + index_client.delete_index() + + return HttpResponseRedirect(reverse('dashboard')) + +def get_max_function(index): + max_function = ScoreFunction.objects.filter(index=index).aggregate(Max('name'))['name__max'] + if max_function == None: + max_function = 0 + return max_function + +STARTING_BASE_PORT = 20000 + +def create_index(request): + account = request.user.get_profile().account + if request.method == 'GET': + index_qty = len(account.indexes.all()) + default_name = '' #'Index_' + str(index_qty + 1) + + form = IndexForm(initial={'name': default_name}) + context = { + 'form': form, + 'account': request.user.get_profile().account, + 'navigation_pos': 'dashboard', + } + return render('new-index.html', request, context_dict=context) + else: + form = IndexForm(data=request.POST) + if form.is_valid(): + try: + client = ApiClient(account.get_private_apiurl()) + client.create_index(form.cleaned_data['name']) + messages.success(request, 'New index created successfully.') + except IndexAlreadyExists: + context = { + 'form': form, + 'account': request.user.get_profile().account, + 'navigation_pos': 'dashboard', + } + messages.error(request, 'You already have an Index with that name.') + return render('new-index.html', request, context_dict=context) + except TooManyIndexes: + context = { + 'form': form, + 'account': request.user.get_profile().account, + 'navigation_pos': 'dashboard', + } + messages.error(request, 'You already have the maximum number of indexes allowed for your account. If you need more, please contact support.') + return render('new-index.html', request, context_dict=context) + except Exception, e: + print e + messages.error(request, 'Unexpected error creating the index. Try again in a few minutes') + return HttpResponseRedirect(reverse('dashboard')) + else: + context = { + 'form': form, + 'account': request.user.get_profile().account, + 'navigation_pos': 'dashboard', + } + return render('new-index.html', request, context_dict=context) + +def logout(request): + is_heroku_logout = request.user.get_profile().account.provisioner and request.user.get_profile().account.provisioner.name == 'heroku' + + auth.logout(request) + # In case this was a + request.session['heroku'] = False + + if is_heroku_logout: + return HttpResponseRedirect('http://api.heroku.com/logout') + + return HttpResponseRedirect('/') # request.GET['next']); + + +## MOCK ## +def api_register_index(request): + apikey = request.GET['api_key'] + index_name = request.GET['index_name'] + + try: + account = Account.objects.get(apikey=apikey) + except Account.DoesNotExist: #@UndefinedVariable + return HttpResponse('{"status":"ERROR", "message":"Invalid account"}') + + if len(account.indexes.all()) >= account.package.max_indexes: + return HttpResponse('{"status":"ERROR", "message":"Account limit reached"}') + else: + index = Index() + index.populate_for_account(account); + index.name = index_name + index.creation_time = datetime.datetime.now() + index.language_code = 'en' + try: + index.save() + except IntegrityError, ie: + print('integrityError in api_register_index.', ie) + return HttpResponse('{"status":"ERROR", "message":"You already have and Index with that name or code."}') + + index.base_port = STARTING_BASE_PORT + 10 * index.id + index.code = get_index_code(index.id) + index.save() + start_index(index) + response = '{"status":"OK", "index_code":"%s"}' % (index.code) + return HttpResponse(response) + +def api_delete_index(request): + apikey = request.GET['apikey'] + index_name = request.GET['indexcode'] + + index = None + + try: + account = Account.objects.get(apikey=apikey) + except Account.DoesNotExist: #@UndefinedVariable + return HttpResponse("1") + + try: + index = Index.objects.get(code=index_name, account=account) + except Index.DoesNotExist: #@UndefinedVariable + return HttpResponse("2") + + stop_index(index) + index.delete() + + return HttpResponse("0") + +def api_list(request): + apikey = request.GET['apikey'] + + try: + account = Account.objects.get(apikey=apikey) + except Account.DoesNotExist: #@UndefinedVariable + return HttpResponse(",") + + list = [x.code for x in account.indexes.all()] + + return HttpResponse(','.join(list)) + +def start_index(index): + dm = getThriftDeployManagerClient() + dm.start_index(index.code, 1000) # TODO get ram from package. + +def stop_index(index): + dm = getThriftDeployManagerClient() + dm.delete_index(index.code) + +''' +THRIFT STUFF +''' +deploymanager_port = 8899 +def getThriftDeployManagerClient(): + protocol, transport = __getThriftProtocolTransport('deploymanager',deploymanager_port) + client = TDeployManager.Client(protocol) + transport.open() + return client + +def __getThriftProtocolTransport(host, port=0): + ''' returns protocol,transport''' + # Make socket + transport = TSocket.TSocket(host, port) + + # Buffering is critical. Raw sockets are very slow + transport = TTransport.TBufferedTransport(transport) + + # Wrap in a protocol + protocol = TBinaryProtocol.TBinaryProtocol(transport) + return protocol, transport + +def is_luhn_valid(cc): + num = map(int, cc) + return not sum(num[::-2] + map(lambda d: sum(divmod(d * 2, 10)), num[-2::-2])) % 10 +