From ead131dc81b00278888077359f5c2a4a05a87af1 Mon Sep 17 00:00:00 2001 From: sunchunqiang Date: Thu, 10 Jan 2019 20:45:07 -0800 Subject: [PATCH] upload all test result to job artifact --- .circleci/config.yml | 92 +++++++++++++---------- CircleciScripts/ReleaseConfiguration.json | 2 +- CircleciScripts/cloudfront_invalidate.py | 12 +-- CircleciScripts/copy_resourcefiles.py | 20 ++--- CircleciScripts/functions.py | 21 +++--- CircleciScripts/generate_javadoc.py | 30 ++++---- CircleciScripts/preserve_olddocument.sh | 3 +- 7 files changed, 99 insertions(+), 81 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 77c7cbdd030..3e0640b6f3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,34 @@ # Android CircleCI 2.0 configuration file # -version: 2 +version: 2.1 +commands: + configure_aws: + description: >- + install aws cli and configure android aws release profile + steps: + - run: + name: install aws cli + command: | + sudo pip install awscli + - run: + name: configure aws profile + command: | + aws configure --profile android_sdk_release set region us-east-1 + echo -e "[android_sdk_release]\naws_access_key_id=${AWS_ACCESS_KEY_ID_ANDROID}\naws_secret_access_key=${AWS_SECRET_ACCESS_KEY_ANDROID}\n" >> ~/.aws/credentials + + set_release_version: + description: >- + set release_version environment + steps: + - run: + name: set release_version environment + command: | + release_tag=${CIRCLE_TAG} + echo "export release_tag=$release_tag" >> $BASH_ENV + release_version=$(echo "$release_tag" | sed 's|.*v\([0-9\.]*\).*|\1|') + echo "export release_version=$release_version" >> $BASH_ENV + + jobs: build: working_directory: ~/code @@ -13,6 +41,7 @@ jobs: - run: name: build the whole project command: | + echo "id=${android_distribution_id}" bash gradlew build -x test buildforrelease: @@ -51,7 +80,15 @@ jobs: python3 CircleciScripts/run_unittest.py test_results "$currentfolder" - run: name : check unit test result - command : bash CircleciScripts/check_testresult.sh test_results + command : | + echo "testresult=$testresult" + if [ "$testresult" == "0" ] + then + echo "test succeed!" + else + echo "There are test failures" + exit 1 + fi - store_artifacts: path: test_results @@ -61,15 +98,14 @@ jobs: - image: circleci/golang:1.11 steps: - checkout + - set_release_version - run: name: install github-release command: go get github.com/aktau/github-release - run: name: release the tag command: | - export release_tag=${CIRCLE_TAG} - export version=$(echo "$release_tag" | sed 's|.*v\([0-9\.]*\).*|\1|') - export tagdescription=$(sed -n "/## \[Release $version\]/,/## \[Release [0-9]*\.[0-9]*\.[0-9]\]/p" CHANGELOG.md | sed '1d' | sed '$d') + tagdescription=$(sed -n "/## \[Release $release_version\]/,/## \[Release [0-9]*\.[0-9]*\.[0-9]\]/p" CHANGELOG.md | sed '1d' | sed '$d') echo "$tagdescription" | github-release release -s ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -t $release_tag --name "$tagname" -d - release_javadoc: @@ -79,6 +115,7 @@ jobs: JVM_OPTS: -Xmx3200m steps: - checkout + - set_release_version - run: name: install json parser command: | @@ -97,13 +134,6 @@ jobs: aws s3 cp s3://android-circleci-payload/third-party/third-party.zip third-party.zip unzip third-party.zip cp "$ANDROID_HOME/platforms/android-27/android.jar" third-party/android.jar - - run: - name: set version - command: | - export release_tag=${CIRCLE_TAG} - echo "release_tag=$release_tag" >> $BASH_ENV - export release_version=$(echo "$release_tag" | sed 's|.*v\([0-9\.]*\).*|\1|') - echo "release_version=$release_version" >> $BASH_ENV - run: name: generate documents command: | @@ -139,13 +169,13 @@ jobs: JVM_OPTS: -Xmx3200m steps: - checkout + - set_release_version - run: name: Add documentation tags to gh-pages command: | git config --local user.name "${GITHUB_USER}" git checkout gh-pages - version=$(echo "${CIRCLE_TAG}" | sed 's|.*v\([0-9\.]*\).*|\1|') - git tag -a docs_v$version -m "Add documentation tags to version ${version}" + git tag -a "docs_v$release_version" -m "Add documentation tags to version $release_version" git push --tags -q https://${GITHUB_TOKEN}@github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}.git release_maven: @@ -153,6 +183,8 @@ jobs: xcode: "10.1.0" #xocde version number does not matter here, we just need a macos image steps: - checkout + - configure_aws + - set_release_version - run: name: install android sdk tools command: | @@ -162,10 +194,6 @@ jobs: brew doctor brew tap caskroom/cask brew cask install android-sdk - - run: - name: install aws cli - command: | - sudo pip install awscli - run: name: install android build tools command: | @@ -206,16 +234,10 @@ jobs: echo "$ANDROID_HOME" echo "$M2_HOME" bash CircleciScripts/maven_release.sh - - run: - name: configure aws profile - command: | - aws configure --profile android_sdk_release set region us-east-1 - echo -e "[android_sdk_release]\naws_access_key_id=${AWS_ACCESS_KEY_ID_ANDROID}\naws_secret_access_key=${AWS_SECRET_ACCESS_KEY_ANDROID}\n" >> ~/.aws/credentials - run: name: upload to maven command: | - version=$(echo "${CIRCLE_TAG}" | sed 's|.*v\([0-9\.]*\).*|\1|') - python3 CircleciScripts/s3_maven.py "$(pwd)" $version android_sdk_release "aws-android-sdk-stage/com/amazonaws" + python3 CircleciScripts/s3_maven.py "$(pwd)" $release_version android_sdk_release "aws-android-sdk-stage-csun/com/amazonaws" release_s3: docker: @@ -224,17 +246,13 @@ jobs: JVM_OPTS: -Xmx1024m steps: - checkout - - run: - name: install aws cli - command: | - sudo pip install awscli + - configure_aws + - set_release_version - run: name: copy sdk resource command: | - release_tag=${CIRCLE_TAG} - version=$(echo "$release_tag" | sed 's|.*v\([0-9\.]*\).*|\1|') - sdkversion="aws-android-sdk-$version" - echo "sdkversion=$sdkversion" >> $BASH_ENV + sdkversion="aws-android-sdk-$release_version" + echo "export sdkversion=$sdkversion" >> $BASH_ENV mkdir -p "$sdkversion" python3 CircleciScripts/copy_resourcefiles.py "$(pwd)" "$(pwd)/$sdkversion" - restore_cache: @@ -264,14 +282,12 @@ jobs: - run: name: upload to s3 command: | - aws configure --profile android_sdk_release set region us-east-1 - echo -e "[android_sdk_release]\naws_access_key_id=${AWS_ACCESS_KEY_ID_ANDROID}\naws_secret_access_key=${AWS_SECRET_ACCESS_KEY_ANDROID}\n" >> ~/.aws/credentials - aws s3 cp "$sdkversion.zip" "s3://aws-android-sdk-stage/$sdkversion.zip" --profile android_stage - aws s3 cp "$sdkversion.zip" "s3://aws-android-sdk-stage/latest/aws-android-sdk.zip" --profile android_sdk_release + aws s3 cp "$sdkversion.zip" "s3://aws-android-sdk-stage-csun/$sdkversion.zip" --profile android_sdk_release + aws s3 cp "$sdkversion.zip" "s3://aws-android-sdk-stage-csun/latest/aws-android-sdk.zip" --profile android_sdk_release - run: name: invalidate cloudfront command: | - python3 CircleciScripts/cloudfront_invalidate.py android_sdk_release android_distribution_id aws-android-sdk.zip + python3 CircleciScripts/cloudfront_invalidate.py android_sdk_release "${android_distribution_id}" aws-android-sdk.zip workflows: version: 2 diff --git a/CircleciScripts/ReleaseConfiguration.json b/CircleciScripts/ReleaseConfiguration.json index 19f2c9e91f6..d65cbc02c86 100644 --- a/CircleciScripts/ReleaseConfiguration.json +++ b/CircleciScripts/ReleaseConfiguration.json @@ -62,7 +62,7 @@ "com.amazonaws.mobileconnectors.*", ], - sourthfiles: [ + sourcefiles: [ "com/amazonaws/http/Http*" , "com/amazonaws/http/ApacheHttpClient*" , "com/amazonaws/http/UrlHttpClient*" diff --git a/CircleciScripts/cloudfront_invalidate.py b/CircleciScripts/cloudfront_invalidate.py index a69d16b3774..57aa530268d 100644 --- a/CircleciScripts/cloudfront_invalidate.py +++ b/CircleciScripts/cloudfront_invalidate.py @@ -3,12 +3,12 @@ import datetime from functions import runcommand -awsprofile = sys.args[1] -distribution_id = sys.args[2] -path = sys.args[3] - -invalidation='{"Paths":{"Quantity":1,"Items":["/{0}"]},"CallerReference":"{0}{1}"}'.format(path,datetime.datetime.now()) -returncode = runcommand("aws cloudfront invalidation --distribution-id {0} --invalidation-batch '{1}'".format(distribution_id, invalidation)) +awsprofile = sys.argv[1] +distribution_id = sys.argv[2] +path = sys.argv[3] +invalidation='{{"Paths":{{"Quantity":1,"Items":["/{0}"]}},"CallerReference":"{0}{1}"}}'.format('path', datetime.datetime.now()) +returncode = runcommand(command="aws cloudfront create-invalidation --distribution-id {0} --invalidation-batch '{1}'".format(distribution_id, invalidation), logcommandline = True) +print("returncode=", returncode) exit(returncode) diff --git a/CircleciScripts/copy_resourcefiles.py b/CircleciScripts/copy_resourcefiles.py index d38c1e0fe65..2a67e82030a 100644 --- a/CircleciScripts/copy_resourcefiles.py +++ b/CircleciScripts/copy_resourcefiles.py @@ -3,24 +3,24 @@ from functions import runcommand from shutil import copyfile +root = sys.argv[1] +dest = sys.argv[2] files = { "LICENSE.AMAZON.txt":"LICENSE.AMAZON.txt", "LICENSE.txt":"LICENSE.txt", "NOTICE.txt":"NOTICE.txt", "README.md":"README.md", -"CircleciScripts/source.htmlcat":"src/source.htmlcat", +"CircleciScripts/source.html":"src/source.html", "LICENSE.APACHE.txt":"LICENSE.APACHE.txt", "Logging.html":"Logging.html", -"Proguard.htm":"Proguard.htm", +"Proguard.html":"Proguard.html", "CircleciScripts/samples.html":"samples/samples.html" } -for source,target in files.items: - s = os.path.join(root, source) - t = os.path.join(root,t) - runcommand("mkdir -p '{0}'".format(os.path.dirname(t))) - rn = copyfile(s,t) - if rn != 0 : - print("failed to copy {0} to {1}".format(s,t)) - exit(1) +for source,target in files.items(): + s = os.path.join(root, source) + t = os.path.join(dest, target) + runcommand("mkdir -p '{0}'".format(os.path.dirname(t))) + copyfile(s,t) + diff --git a/CircleciScripts/functions.py b/CircleciScripts/functions.py index b2e65fb358b..36a8818ce13 100644 --- a/CircleciScripts/functions.py +++ b/CircleciScripts/functions.py @@ -21,8 +21,9 @@ def testAction(self): return self.value.testAction -def runcommand(command, timeout=0,pipein=None, pipeout = None): - print("running command: ", command, "......") +def runcommand(command, timeout=0,pipein=None, pipeout = None, logcommandline = True): + if logcommandline: + print("running command: ", command, "......") process = Popen(command, shell=True, stdin=pipein, stdout = pipeout) wait_times = 0 while True: @@ -50,13 +51,15 @@ def runtest(module, testtype, results): testcommand = "bash gradlew {0}:{1} ".format(module, testtype.testAction) print("Running {0} for {1} .......".format(testtype.displayString, module)) exit_code = runcommand(testcommand) - if exit_code != 0 : - print("test failed for {0}".format(module)) - dest = "{0}/{1}".format(results, module) - runcommand("mkdir {0} ".format(dest)) - source = "{0}/build/reports/*".format(module) - if runcommand("cp -rf {0} {1}".format(source,dest)) != 0 : - return 1 + print("test failed for {0}".format(module)) + dest = "{0}/{1}".format(results, module) + runcommand("mkdir {0} ".format(dest)) + source = "{0}/build/reports/*".format(module) + runcommand('echo "export testresult=0" >> $BASH_ENV') + if runcommand("cp -rf {0} {1}".format(source,dest)) != 0 : + return 1 + if exit_code != 0 : + runcommand('echo "export testresult=1" >> $BASH_ENV') return 0 diff --git a/CircleciScripts/generate_javadoc.py b/CircleciScripts/generate_javadoc.py index 4300bed9ba0..d491c93343a 100644 --- a/CircleciScripts/generate_javadoc.py +++ b/CircleciScripts/generate_javadoc.py @@ -5,14 +5,14 @@ import glob from functions import runcommand -def getCommandline(dest, root, modules, packages, sourthfiles, subpackages,excludes, groups, otheroptions, otherargs): +def getCommandline(dest, root, modules, packages, sourcefiles, subpackages,excludes, groups, otheroptions, otherargs): commandline = "javadoc " + \ " -d '{0}'".format(dest) + \ " -sourcepath '{0}'".format(':'.join(map(lambda module: (root + "/" + module + "/src/main/java/"), modules))) + \ " " + ' '.join(packages) + \ - " " + ' '.join(map(lambda sourthfile:("'{0}'".format(sourthfile)), sourthfiles)) + \ + " " + ' '.join(map(lambda sourcefile:("'{0}'".format(sourcefile)), sourcefiles)) + \ " -subpackages '{0}' ".format(':'.join(subpackages)) + \ " -exclude '{0}' ".format(':'.join(excludes)) + \ " ".join(map(lambda group:(" -group '{0}' '{1}' ".format(group['title'], ':'.join(group['packages']))), groups)) + \ @@ -72,19 +72,19 @@ def getPackagesWithPattern(root, pattern): def getAllPackagesWithPattern(root, modules, pattern): packages = set() - sourthpaths = "" + sourcepaths = "" for module in modules: - sourthpath = os.path.join(root, module , "src/main/java/") - p = getPackagesWithPattern(sourthpath, pattern); + sourcepath = os.path.join(root, module , "src/main/java/") + p = getPackagesWithPattern(sourcepath, pattern); packages.update(p); return packages -def getSourthFilesWithPattern(root, module, patterns): +def getsourceFilesWithPattern(root, module, patterns): files = [] - sourthpaths = "" + sourcepaths = "" for module in modules: for pattern in patterns: - sourthpath =os.path.join(root , module , "src/main/java/", pattern) - files.extend(list(filter(lambda x:x.endswith('.html') or x.endswith('.java'), glob.glob(sourthpath)))) + sourcepath =os.path.join(root , module , "src/main/java/", pattern) + files.extend(list(filter(lambda x:x.endswith('.html') or x.endswith('.java'), glob.glob(sourcepath)))) return files @@ -120,9 +120,9 @@ def getJARs(root, libs): def copylib(root, modules, target): for module in modules: - sourthpath = os.path.join(root, module, "build/libs") - if os.path.isdir(sourthpath): - runcommand('cp {0}/*.jar "{1}"'.format(sourthpath, target)) + sourcepath = os.path.join(root, module, "build/libs") + if os.path.isdir(sourcepath): + runcommand('cp {0}/*.jar "{1}"'.format(sourcepath, target)) @@ -173,8 +173,8 @@ def copylib(root, modules, target): excludepackageset = getAllPackagesWithPattern(root, modules, exclude); excludes.update(excludepackageset) -sourthfiles = getSourthFilesWithPattern(root, modules, docConfigure["sourthfiles"]) -print("sourthfiles: ", sourthfiles) +sourcefiles = getsourceFilesWithPattern(root, modules, docConfigure["sourcefiles"]) +print("sourcefiles: ", sourcefiles) groups = docConfigure["groups"] for group in groups: group['packages'] = resolveList(group['packages']) @@ -187,7 +187,7 @@ def copylib(root, modules, target): jarlist.append(os.environ["CLASSPATH"]) os.environ["CLASSPATH"]=':'.join(jarlist) os.environ["sdkVersion"]=sdkVersion -commandline = getCommandline(dest, root, modules, packages,sourthfiles, subpackages,excludes, groups, otheroptions, otherargs) +commandline = getCommandline(dest, root, modules, packages,sourcefiles, subpackages,excludes, groups, otheroptions, otherargs) returncode = runcommand(commandline) print("return code:" , returncode) exit(returncode) diff --git a/CircleciScripts/preserve_olddocument.sh b/CircleciScripts/preserve_olddocument.sh index f8f3fdc1e78..49a61a25863 100644 --- a/CircleciScripts/preserve_olddocument.sh +++ b/CircleciScripts/preserve_olddocument.sh @@ -1,5 +1,5 @@ echo "release_version=$release_version" -previousversion=$(grep -o "AWS SDK for Android - [0-9]\+.[0-9]\+.[0-9]\+" docs/reference/allclasses-frame.html | grep -o "[0-9]\+.[0-9]\+.[0-9]") +previousversion=$(grep -o "AWS SDK for Android - [0-9]\+.[0-9]\+.[0-9]\+" docs/reference/allclasses-frame.html | grep -o "[0-9]\+.[0-9]\+.[0-9]\+") echo "previousversion=$previousversion" previousminorversion=$(echo "$previousversion" | sed -e 's|[0-9]*\.\([0-9]*\)\.[0-9]*$|\1|') echo "previousminorversion=$previousminorversion" @@ -35,4 +35,3 @@ then else echo "don't preserve old document" fi -ls docs \ No newline at end of file