Permalink
Browse files

updated README, removed more unnecessary arguments (AWS properties). …

…Added a config.json for overriding environment variables, centralized some code in utils.py, added a new script for managing which branches are shown in the web interface (tools/branch_builds.py)
  • Loading branch information...
1 parent 6eee79b commit e24ab4a6260c514b1b7752ce006ae3203ee19852 @marshall marshall committed Oct 2, 2010
Showing with 183 additions and 43 deletions.
  1. +26 −2 README
  2. +1 −0 common/.gitignore
  3. +9 −9 common/s3_cleaner.py
  4. +12 −13 common/s3_uploader.py
  5. +63 −0 common/utils.py
  6. +3 −0 config.json
  7. +3 −3 desktop/driver.sh
  8. +4 −8 mobile/driver.sh
  9. +53 −0 tools/branch_builds.py
  10. +9 −8 {common → tools}/web_updater.py
View
@@ -3,8 +3,32 @@ This is a Hudson-based build system for Titanium Mobile and Desktop.
The basic configuration is:
- Tomcat 6 + Latest Hudson
- Git plugin
- - Setup the ANDROID_SDK environment variable for Android build
- - Setup AWS_KEY and AWS_SECRET environment variables for upload
+ - Set the ANDROID_SDK environment variable to the top level path of the Android SDK
+ - Setup config properties (see below)
- Point the mobile config at mobile/driver.sh
- Point the desktop config at desktop/driver.sh
+If you make any changes to the web interface, make sure to upload the changes by using:
+ ./tools/web_updater.py
+
+To manage branch builds shown in the web interface:
+ - Add a branch build:
+ ./tools/branch_builds.py add (mobile|desktop) <branch>
+ - Remove a branch build:
+ ./tools/branch_builds.py remove (mobile|desktop) <branch>
+ - List currently visible branch builds:
+ ./tools/branch_builds.py list (mobile|desktop)
+
+Config properties
+-----------------
+
+This build accepts properties either as environment variables, or from the "config.json" file at the top of the build tree. Currently accepted properties are:
+
+- AWS_KEY (required)
+ The Amazon Web Services key for uploading S3 Content
+
+- AWS_SECRET (required)
+ The Amazon Web Services secret key for uploading S3 Content
+
+- UPLOAD_BUCKET (required)
+ The bucket in which to upload builds, the JSON indexes, and web content
View
@@ -0,0 +1 @@
+*.pyc
View
@@ -1,20 +1,20 @@
#!/usr/bin/python
-import os, sys, boto
+import os, sys, boto, utils
from boto.s3.connection import S3Connection
from boto.s3.key import Key
import simplejson
-if len(sys.argv) != 5:
- print "Usage: %s <AWS Access Key> <AWS Secret Key> <desktop|mobile> <branch>" % sys.argv[0]
+if len(sys.argv) != 3:
+ print "Usage: %s <desktop|mobile> <branch>" % sys.argv[0]
sys.exit(1)
-access_key = sys.argv[1]
-secret_key = sys.argv[2]
-type = sys.argv[3]
-branch = sys.argv[4]
+(type, branch) = sys.argv
+cfg = utils.get_build_config()
+if not cfg.verify_aws():
+ print "Error: Need both AWS_KEY and AWS_SECRET in the environment or config.json"
+ sys.exit(1)
-conn = S3Connection(access_key, secret_key)
-bucket = conn.get_bucket('builds.appcelerator.com')
+bucket = cfg.open_bucket()
keys = []
prefix = type+'/'+branch
View
@@ -1,28 +1,27 @@
#!/usr/bin/python
-import sys, os, socket
+import sys, os, socket, utils
from boto.s3.connection import S3Connection
from boto.s3.key import Key
import simplejson
-if len(sys.argv) != 9:
- print "Usage: %s <AWS Access Key> <AWS Secret Key> <desktop|mobile> <path> <branch> <revision> <build url> <sha1>" % sys.argv[0]
+if len(sys.argv) != 6:
+ print "Usage: %s <desktop|mobile> <path> <branch> <revision> <build url>" % sys.argv[0]
sys.exit(1)
-access_key = sys.argv[1]
-secret_key = sys.argv[2]
-type = sys.argv[3]
-path = sys.argv[4]
-branch = sys.argv[5]
-revision = sys.argv[6]
-build_url = sys.argv[7]
-sha1 = sys.argv[8]
+(type, path, branch, revision, build_url) = sys.argv
+cfg = utils.get_build_config()
+if not cfg.verify_aws():
+ print "Error: Need both AWS_KEY and AWS_SECRET in the environment or config.json"
+ sys.exit(1)
+
+bucket = cfg.open_bucket()
+
+sha1 = utils.shasum(path)
filename = os.path.basename(path)
filesize = os.path.getsize(path)
print 'uploading %s (branch %s / revision %s)...' % (filename, branch, revision)
-conn = S3Connection(access_key, secret_key)
-bucket = conn.get_bucket('builds.appcelerator.com')
key = Key(bucket)
key.key = '%s/%s/%s' % (type, branch, filename)
key.set_metadata('git_revision', revision)
View
@@ -0,0 +1,63 @@
+import os, sys, re
+import simplejson
+from subprocess import Popen, PIPE
+from boto.s3.connection import S3Connection
+from boto.s3.key import Key
+
+class Config(object):
+ def __init__(self, file=None):
+ if file != None:
+ self.json = simplejson.loads(open(file, 'r').read())
+ else:
+ self.json = {}
+
+ def get_aws_key(self):
+ return self.get('AWS_KEY')
+
+ def get_aws_secret(self):
+ return self.get('AWS_SECRET')
+
+ def get_upload_bucket(self):
+ return self.get('UPLOAD_BUCKET')
+
+ def open_bucket(self):
+ conn = S3Connection(self.get_aws_key(), self.get_aws_secret())
+ return conn.get_bucket(self.get_upload_bucket())
+
+ def verify_aws(self):
+ return self.get_aws_key() != None and self.get_aws_secret() != None
+
+ def get(self, property):
+ if property in self.json:
+ return self.json[property]
+ elif property in os.environ:
+ return os.environ[property]
+ else: return None
+
+def get_build_config():
+ config_json = os.path.join(os.path.dirname(
+ os.path.abspath(os.path.dirname(__file__))), 'config.json')
+ return Config(config_json)
+
+def shasum(path):
+ out = Popen(['shasum', path], stdout=PIPE).communicate()[0]
+ return re.split(r' *', out.rstrip())[0]
+
+def get_key_json_array(bucket, key):
+ array = []
+ key_obj = bucket.get_key(key)
+ if key_obj == None:
+ key_obj = Key(bucket)
+ key_obj.key = key
+ else:
+ array = simplejson.loads(key_obj.get_contents_as_string())
+ return array
+
+def set_key_json_array(bucket, key, array):
+ key_obj = bucket.get_key(key)
+ if key_obj == None:
+ key_obj = Key(bucket)
+ key_obj.key = key
+
+ key_obj.set_contents_from_string(simplejson.dumps(array))
+ key_obj.make_public()
View
@@ -0,0 +1,3 @@
+{
+ "UPLOAD_BUCKET": "builds.appcelerator.com"
+}
View
@@ -33,6 +33,6 @@ scons -j $NUM_CPUS debug=1 breakpad=0 drillbit dist || exit
TIMESTAMP_NAME=build/$PLATFORM/dist/sdk-$VERSION-$TIMESTAMP-$PLATFORM.zip
mv build/$PLATFORM/dist/sdk-$VERSION.zip $TIMESTAMP_NAME
-SHA1=`shasum $TIMESTAMP_NAME | sed 's/ .*//' | tr -d '\n' | tr -d '\r'`
-python $TITANIUM_BUILD/common/s3_cleaner.py $AWS_KEY $AWS_SECRET desktop $GIT_BRANCH
-python $TITANIUM_BUILD/common/s3_uploader.py $AWS_KEY $AWS_SECRET desktop $TIMESTAMP_NAME $GIT_BRANCH $GIT_REVISION $BUILD_URL $SHA1
+#SHA1=`shasum $TIMESTAMP_NAME | sed 's/ .*//' | tr -d '\n' | tr -d '\r'`
+python $TITANIUM_BUILD/common/s3_cleaner.py desktop $GIT_BRANCH
+python $TITANIUM_BUILD/common/s3_uploader.py desktop $TIMESTAMP_NAME $GIT_BRANCH $GIT_REVISION $BUILD_URL
View
@@ -15,11 +15,7 @@ mv dist/mobilesdk-$VERSION-osx.zip $BASENAME-osx.zip
mv dist/mobilesdk-$VERSION-win32.zip $BASENAME-win32.zip
mv dist/mobilesdk-$VERSION-linux.zip $BASENAME-linux.zip
-OSX_SHA1=`shasum $BASENAME-osx.zip | sed 's/ .*//' | tr -d '\n' | tr -d '\r'`
-WIN32_SHA1=`shasum $BASENAME-win32.zip | sed 's/ .*//' | tr -d '\n' | tr -d '\r'`
-LINUX_SHA1=`shasum $BASENAME-linux.zip | sed 's/ .*//' | tr -d '\n' | tr -d '\r'`
-
-python $TITANIUM_BUILD/common/s3_cleaner.py $AWS_KEY $AWS_SECRET mobile $GIT_BRANCH
-python $TITANIUM_BUILD/common/s3_uploader.py $AWS_KEY $AWS_SECRET mobile $BASENAME-osx.zip $GIT_BRANCH $GIT_REVISION $BUILD_URL $OSX_SHA1
-python $TITANIUM_BUILD/common/s3_uploader.py $AWS_KEY $AWS_SECRET mobile $BASENAME-linux.zip $GIT_BRANCH $GIT_REVISION $BUILD_URL $LINUX_SHA1
-python $TITANIUM_BUILD/common/s3_uploader.py $AWS_KEY $AWS_SECRET mobile $BASENAME-win32.zip $GIT_BRANCH $GIT_REVISION $BUILD_URL $WIN32_SHA1
+python $TITANIUM_BUILD/common/s3_cleaner.py mobile $GIT_BRANCH
+python $TITANIUM_BUILD/common/s3_uploader.py mobile $BASENAME-osx.zip $GIT_BRANCH $GIT_REVISION $BUILD_URL
+python $TITANIUM_BUILD/common/s3_uploader.py mobile $BASENAME-linux.zip $GIT_BRANCH $GIT_REVISION $BUILD_URL
+python $TITANIUM_BUILD/common/s3_uploader.py mobile $BASENAME-win32.zip $GIT_BRANCH $GIT_REVISION $BUILD_URL
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+import sys, os
+build_dir = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
+common_dir = os.path.join(build_dir, 'common')
+sys.path.append(common_dir)
+
+import utils
+from boto.s3.connection import S3Connection
+from boto.s3.key import Key
+import simplejson
+
+def usage():
+ print "Usage: %s (add|remove|list) (mobile|desktop) <branch>"
+ sys.exit(1)
+
+if len(sys.argv) < 3: usage()
+
+command = sys.argv[1]
+type = sys.argv[2]
+if command in ['add', 'remove']:
+ if len(sys.argv) != 4:
+ usage()
+ branch = sys.argv[3]
+
+cfg = utils.get_build_config()
+if not cfg.verify_aws():
+ print "Error: Need both AWS_KEY and AWS_SECRET in the environment or config.json"
+ sys.exit(1)
+
+bucket = cfg.open_bucket()
+branches_key = '%s/branches.json' % type
+branches = utils.get_key_json_array(bucket, branches_key)
+
+def list_branches():
+ print 'Branches for %s:' % type
+ for branch in branches:
+ print ' %s' % branch
+if command == 'list':
+ list_branches()
+elif command == 'add':
+ if branch in branches:
+ print 'Error: branch %s already in branch list, ignoring' % branch
+ sys.exit(1)
+ branches.append(branch)
+ utils.set_key_json_array(bucket, branches_key, branches)
+ list_branches()
+elif command == 'remove':
+ if not branch in branches:
+ print 'Error: branch %s isn\'t currently in branch list, ignoring' % branch
+ sys.exit(1)
+ branches.remove(branch)
+ utils.set_key_json_array(bucket, branches_key, branches)
+ list_branches()
@@ -1,20 +1,21 @@
#!/usr/bin/env python
-
import sys, os
+build_dir = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
+common_dir = os.path.join(build_dir, 'common')
+sys.path.append(common_dir)
+
+import utils
from boto.s3.connection import S3Connection
from boto.s3.key import Key
import simplejson
-if len(sys.argv) != 3:
- print "Usage: %s <AWS Access Key> <AWS Secret Key>" % sys.argv[0]
+cfg = utils.get_build_config()
+if not cfg.verify_aws():
+ print "Error: Need both AWS_KEY and AWS_SECRET in the environment or config.json"
sys.exit(1)
-access_key = sys.argv[1]
-secret_key = sys.argv[2]
-
print 'publishing changes to builds.appcelerator.com s3 site...'
-conn = S3Connection(access_key, secret_key)
-bucket = conn.get_bucket('builds.appcelerator.com')
+bucket = cfg.open_bucket()
web_dir = os.path.join(os.path.dirname(os.path.abspath(os.path.dirname(__file__))), 'web')
for root, dirs, files in os.walk(web_dir):

0 comments on commit e24ab4a

Please sign in to comment.