upload all test result to job artifact

.circleci/config.yml

@@ -1,6 +1,34 @@
 # Android CircleCI 2.0 configuration file
 #
-version: 2
+version: 2.1
+commands:
+  configure_aws:
+    description: >-
+      install aws cli and configure android aws release profile 
+    steps:
+      - run:
+          name: install aws cli
+          command: |
+            sudo pip install awscli
+      - run:
+          name: configure aws profile
+          command: |
+            aws configure --profile android_sdk_release set region us-east-1
+            echo -e "[android_sdk_release]\naws_access_key_id=${AWS_ACCESS_KEY_ID_ANDROID}\naws_secret_access_key=${AWS_SECRET_ACCESS_KEY_ANDROID}\n" >> ~/.aws/credentials
+
+  set_release_version:
+    description: >-
+      set release_version environment 
+    steps:
+      - run:
+          name: set release_version environment 
+          command: |
+            release_tag=${CIRCLE_TAG}
+            echo "export release_tag=$release_tag" >> $BASH_ENV
+            release_version=$(echo "$release_tag" | sed 's|.*v\([0-9\.]*\).*|\1|') 
+            echo "export release_version=$release_version" >> $BASH_ENV
+
+
 jobs:
   build:
     working_directory: ~/code
@@ -13,6 +41,7 @@ jobs:
       - run:
           name: build the whole project
           command: |
+            echo "id=${android_distribution_id}" 
             bash gradlew build -x test
 
   buildforrelease:
@@ -51,7 +80,15 @@ jobs:
             python3 CircleciScripts/run_unittest.py test_results "$currentfolder"
       - run:
           name : check unit test result 
-          command : bash CircleciScripts/check_testresult.sh test_results         
+          command : |
+            echo "testresult=$testresult"
+            if [ "$testresult" == "0" ] 
+            then
+                echo "test succeed!"
+            else
+                echo "There are test failures"
+                exit 1
+            fi       
       - store_artifacts:
           path: test_results
 
@@ -61,15 +98,14 @@ jobs:
       - image:  circleci/golang:1.11
     steps:
       - checkout
+      - set_release_version
       - run:
           name: install github-release
           command: go get github.com/aktau/github-release          
       - run:
           name: release the tag
           command: |
-            export release_tag=${CIRCLE_TAG}
-            export version=$(echo "$release_tag" | sed 's|.*v\([0-9\.]*\).*|\1|')
-            export tagdescription=$(sed -n "/## \[Release $version\]/,/## \[Release [0-9]*\.[0-9]*\.[0-9]\]/p"  CHANGELOG.md | sed '1d' | sed '$d')
+            tagdescription=$(sed -n "/## \[Release $release_version\]/,/## \[Release [0-9]*\.[0-9]*\.[0-9]\]/p"  CHANGELOG.md | sed '1d' | sed '$d')
             echo "$tagdescription" | github-release release  -s ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME}  -t $release_tag   --name  "$tagname" -d - 
 
   release_javadoc:
@@ -79,6 +115,7 @@ jobs:
       JVM_OPTS: -Xmx3200m     
     steps:
       - checkout   
+      - set_release_version
       - run: 
           name: install json parser
           command: |
@@ -97,13 +134,6 @@ jobs:
             aws s3 cp  s3://android-circleci-payload/third-party/third-party.zip third-party.zip
             unzip third-party.zip
             cp "$ANDROID_HOME/platforms/android-27/android.jar"  third-party/android.jar
-      - run:
-          name: set version
-          command: |
-            export release_tag=${CIRCLE_TAG}
-            echo "release_tag=$release_tag" >> $BASH_ENV
-            export release_version=$(echo "$release_tag" | sed 's|.*v\([0-9\.]*\).*|\1|') 
-            echo "release_version=$release_version" >> $BASH_ENV
       - run:
           name: generate documents
           command: |
@@ -139,20 +169,22 @@ jobs:
       JVM_OPTS: -Xmx3200m     
     steps:
       - checkout   
+      - set_release_version
       - run:
           name: Add documentation tags to gh-pages
           command: |
             git config --local user.name "${GITHUB_USER}"
             git checkout  gh-pages
-            version=$(echo "${CIRCLE_TAG}" | sed 's|.*v\([0-9\.]*\).*|\1|') 
-            git tag -a docs_v$version  -m "Add documentation tags to version ${version}"            
+            git tag -a "docs_v$release_version"  -m "Add documentation tags to version $release_version"            
             git push --tags -q https://${GITHUB_TOKEN}@github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}.git     
 
   release_maven:
     macos: 
       xcode: "10.1.0"  #xocde version number does not matter here, we just need a macos image
     steps:
       - checkout    
+      - configure_aws
+      - set_release_version
       - run:
           name: install android sdk tools
           command: |
@@ -162,10 +194,6 @@ jobs:
             brew doctor  
             brew tap caskroom/cask
             brew cask install android-sdk
-      - run:
-          name: install aws cli
-          command: |
-            sudo pip install awscli
       - run:
           name: install android build tools
           command: |
@@ -206,16 +234,10 @@ jobs:
             echo "$ANDROID_HOME"
             echo "$M2_HOME"
             bash CircleciScripts/maven_release.sh
-      - run:
-          name: configure aws profile
-          command: |
-            aws configure --profile android_sdk_release set region us-east-1
-            echo -e "[android_sdk_release]\naws_access_key_id=${AWS_ACCESS_KEY_ID_ANDROID}\naws_secret_access_key=${AWS_SECRET_ACCESS_KEY_ANDROID}\n" >> ~/.aws/credentials
       - run:
           name: upload to maven
           command: |
-            version=$(echo "${CIRCLE_TAG}" | sed 's|.*v\([0-9\.]*\).*|\1|') 
-            python3 CircleciScripts/s3_maven.py "$(pwd)" $version android_sdk_release  "aws-android-sdk-stage/com/amazonaws" 
+            python3 CircleciScripts/s3_maven.py "$(pwd)" $release_version android_sdk_release  "aws-android-sdk-stage-csun/com/amazonaws" 
 
   release_s3:
     docker:
@@ -224,17 +246,13 @@ jobs:
       JVM_OPTS: -Xmx1024m     
     steps:
       - checkout   
-      - run:
-          name: install aws cli
-          command: |
-            sudo pip install awscli
+      - configure_aws
+      - set_release_version
       - run:
           name: copy sdk resource
           command: |
-            release_tag=${CIRCLE_TAG}
-            version=$(echo "$release_tag" | sed 's|.*v\([0-9\.]*\).*|\1|') 
-            sdkversion="aws-android-sdk-$version"
-            echo "sdkversion=$sdkversion" >> $BASH_ENV
+            sdkversion="aws-android-sdk-$release_version"
+            echo "export sdkversion=$sdkversion" >> $BASH_ENV
             mkdir -p "$sdkversion"
             python3 CircleciScripts/copy_resourcefiles.py "$(pwd)"  "$(pwd)/$sdkversion"
       - restore_cache:
@@ -264,14 +282,12 @@ jobs:
       - run:
           name: upload to s3
           command: |
-            aws configure --profile android_sdk_release set region us-east-1
-            echo -e "[android_sdk_release]\naws_access_key_id=${AWS_ACCESS_KEY_ID_ANDROID}\naws_secret_access_key=${AWS_SECRET_ACCESS_KEY_ANDROID}\n" >> ~/.aws/credentials
-            aws s3 cp "$sdkversion.zip" "s3://aws-android-sdk-stage/$sdkversion.zip"   --profile android_stage   
-            aws s3 cp "$sdkversion.zip" "s3://aws-android-sdk-stage/latest/aws-android-sdk.zip"   --profile android_sdk_release   
+            aws s3 cp "$sdkversion.zip" "s3://aws-android-sdk-stage-csun/$sdkversion.zip"   --profile android_sdk_release   
+            aws s3 cp "$sdkversion.zip" "s3://aws-android-sdk-stage-csun/latest/aws-android-sdk.zip"   --profile android_sdk_release   
       - run:
           name: invalidate cloudfront
           command: |
-            python3 CircleciScripts/cloudfront_invalidate.py android_sdk_release  android_distribution_id aws-android-sdk.zip
+            python3 CircleciScripts/cloudfront_invalidate.py android_sdk_release  "${android_distribution_id}" aws-android-sdk.zip
 
 workflows:
   version: 2

CircleciScripts/ReleaseConfiguration.json

@@ -62,7 +62,7 @@
 			"com.amazonaws.mobileconnectors.*",		
 		],
 
-		sourthfiles: [
+		sourcefiles: [
 			"com/amazonaws/http/Http*" ,
 			"com/amazonaws/http/ApacheHttpClient*" ,
 			"com/amazonaws/http/UrlHttpClient*"  

CircleciScripts/cloudfront_invalidate.py

@@ -3,12 +3,12 @@
 import datetime
 from functions import runcommand
 
-awsprofile = sys.args[1]
-distribution_id = sys.args[2]
-path = sys.args[3]
-
-invalidation='{"Paths":{"Quantity":1,"Items":["/{0}"]},"CallerReference":"{0}{1}"}'.format(path,datetime.datetime.now())
-returncode = runcommand("aws cloudfront invalidation --distribution-id {0} --invalidation-batch '{1}'".format(distribution_id, invalidation))
+awsprofile = sys.argv[1]
+distribution_id = sys.argv[2]
+path = sys.argv[3]
+invalidation='{{"Paths":{{"Quantity":1,"Items":["/{0}"]}},"CallerReference":"{0}{1}"}}'.format('path',  datetime.datetime.now())
+returncode = runcommand(command="aws cloudfront create-invalidation --distribution-id {0} --invalidation-batch '{1}'".format(distribution_id, invalidation), logcommandline = True)
+print("returncode=", returncode)
 exit(returncode)
 
 

CircleciScripts/copy_resourcefiles.py

@@ -3,24 +3,24 @@
 from functions import runcommand
 from shutil import copyfile
 
+root = sys.argv[1]
+dest = sys.argv[2]
 files = {
 "LICENSE.AMAZON.txt":"LICENSE.AMAZON.txt",
 "LICENSE.txt":"LICENSE.txt",
 "NOTICE.txt":"NOTICE.txt",
 "README.md":"README.md",
-"CircleciScripts/source.htmlcat":"src/source.htmlcat",
+"CircleciScripts/source.html":"src/source.html",
 "LICENSE.APACHE.txt":"LICENSE.APACHE.txt",
 "Logging.html":"Logging.html",
-"Proguard.htm":"Proguard.htm",
+"Proguard.html":"Proguard.html",
 "CircleciScripts/samples.html":"samples/samples.html"
 
 }
 
-for source,target in files.items: 
-	s = os.path.join(root, source)
-	t = os.path.join(root,t)
-	runcommand("mkdir -p '{0}'".format(os.path.dirname(t)))
-	rn = copyfile(s,t)
-	if rn != 0 :
-		print("failed to copy {0} to {1}".format(s,t))
-		exit(1)
+for source,target in files.items(): 
+    s = os.path.join(root, source)
+    t = os.path.join(dest, target)
+    runcommand("mkdir -p '{0}'".format(os.path.dirname(t)))
+    copyfile(s,t)
+

CircleciScripts/functions.py

@@ -21,8 +21,9 @@ def testAction(self):
         return self.value.testAction
 
 
-def runcommand(command, timeout=0,pipein=None, pipeout =  None):
-    print("running command: ", command, "......")
+def runcommand(command, timeout=0,pipein=None, pipeout =  None, logcommandline = True):
+    if logcommandline:
+        print("running command: ", command, "......")
     process = Popen(command, shell=True, stdin=pipein, stdout = pipeout)
     wait_times = 0 
     while True:
@@ -50,13 +51,15 @@ def runtest(module, testtype, results):
     testcommand = "bash gradlew {0}:{1} ".format(module, testtype.testAction)
     print("Running {0} for {1} .......".format(testtype.displayString, module))   
     exit_code = runcommand(testcommand)   
-    if exit_code != 0 :
-        print("test failed for {0}".format(module))
-        dest = "{0}/{1}".format(results, module)
-        runcommand("mkdir {0} ".format(dest))
-        source = "{0}/build/reports/*".format(module)              
-        if runcommand("cp -rf {0} {1}".format(source,dest)) != 0 :
-            return 1
+    print("test failed for {0}".format(module))
+    dest = "{0}/{1}".format(results, module)
+    runcommand("mkdir {0} ".format(dest))
+    source = "{0}/build/reports/*".format(module)             
+    runcommand('echo "export testresult=0" >> $BASH_ENV')
+    if runcommand("cp -rf {0} {1}".format(source,dest)) != 0 :
+        return 1
+    if exit_code != 0 :    
+        runcommand('echo "export testresult=1" >> $BASH_ENV')  
 
     return 0
 

CircleciScripts/generate_javadoc.py

@@ -5,14 +5,14 @@
 import glob
 from functions import runcommand
 
-def getCommandline(dest, root, modules, packages, sourthfiles, subpackages,excludes, groups, otheroptions, otherargs):
+def getCommandline(dest, root, modules, packages, sourcefiles, subpackages,excludes, groups, otheroptions, otherargs):
 
 
     commandline = "javadoc " + \
                 " -d '{0}'".format(dest) + \
                 " -sourcepath  '{0}'".format(':'.join(map(lambda module: (root + "/" + module + "/src/main/java/"), modules)))  + \
                 " " + ' '.join(packages)  +  \
-                " " + ' '.join(map(lambda sourthfile:("'{0}'".format(sourthfile)), sourthfiles))  +  \
+                " " + ' '.join(map(lambda sourcefile:("'{0}'".format(sourcefile)), sourcefiles))  +  \
                 " -subpackages '{0}' ".format(':'.join(subpackages)) + \
                 " -exclude '{0}' ".format(':'.join(excludes)) + \
                 " ".join(map(lambda group:(" -group '{0}' '{1}' ".format(group['title'], ':'.join(group['packages']))), groups)) +  \
@@ -72,19 +72,19 @@ def getPackagesWithPattern(root, pattern):
 
 def getAllPackagesWithPattern(root, modules, pattern):
     packages = set()
-    sourthpaths = ""
+    sourcepaths = ""
     for module in modules:
-        sourthpath = os.path.join(root, module ,  "src/main/java/")
-        p = getPackagesWithPattern(sourthpath, pattern);
+        sourcepath = os.path.join(root, module ,  "src/main/java/")
+        p = getPackagesWithPattern(sourcepath, pattern);
         packages.update(p);
     return packages
-def getSourthFilesWithPattern(root, module, patterns):
+def getsourceFilesWithPattern(root, module, patterns):
     files = []
-    sourthpaths = ""
+    sourcepaths = ""
     for module in modules:
         for pattern in patterns:
-            sourthpath =os.path.join(root , module , "src/main/java/", pattern)
-            files.extend(list(filter(lambda x:x.endswith('.html') or x.endswith('.java'), glob.glob(sourthpath))))
+            sourcepath =os.path.join(root , module , "src/main/java/", pattern)
+            files.extend(list(filter(lambda x:x.endswith('.html') or x.endswith('.java'), glob.glob(sourcepath))))
 
     return files
 
@@ -120,9 +120,9 @@ def getJARs(root, libs):
 
 def copylib(root, modules, target):
     for module in modules:
-        sourthpath = os.path.join(root, module, "build/libs")
-        if os.path.isdir(sourthpath):
-            runcommand('cp {0}/*.jar "{1}"'.format(sourthpath, target))
+        sourcepath = os.path.join(root, module, "build/libs")
+        if os.path.isdir(sourcepath):
+            runcommand('cp {0}/*.jar "{1}"'.format(sourcepath, target))
 
 
 
@@ -173,8 +173,8 @@ def copylib(root, modules, target):
         excludepackageset = getAllPackagesWithPattern(root, modules, exclude);
         excludes.update(excludepackageset)
 
-sourthfiles = getSourthFilesWithPattern(root, modules, docConfigure["sourthfiles"])
-print("sourthfiles: ", sourthfiles)
+sourcefiles = getsourceFilesWithPattern(root, modules, docConfigure["sourcefiles"])
+print("sourcefiles: ", sourcefiles)
 groups = docConfigure["groups"]
 for group in groups:
     group['packages'] = resolveList(group['packages'])
@@ -187,7 +187,7 @@ def copylib(root, modules, target):
     jarlist.append(os.environ["CLASSPATH"])
 os.environ["CLASSPATH"]=':'.join(jarlist)
 os.environ["sdkVersion"]=sdkVersion
-commandline = getCommandline(dest, root, modules, packages,sourthfiles, subpackages,excludes, groups, otheroptions, otherargs)
+commandline = getCommandline(dest, root, modules, packages,sourcefiles, subpackages,excludes, groups, otheroptions, otherargs)
 returncode = runcommand(commandline)
 print("return code:" , returncode)
 exit(returncode)

CircleciScripts/preserve_olddocument.sh

@@ -1,5 +1,5 @@
 echo "release_version=$release_version"
-previousversion=$(grep -o "AWS SDK for Android - [0-9]\+.[0-9]\+.[0-9]\+" docs/reference/allclasses-frame.html | grep -o "[0-9]\+.[0-9]\+.[0-9]")
+previousversion=$(grep -o "AWS SDK for Android - [0-9]\+.[0-9]\+.[0-9]\+" docs/reference/allclasses-frame.html | grep -o "[0-9]\+.[0-9]\+.[0-9]\+")
 echo "previousversion=$previousversion"
 previousminorversion=$(echo  "$previousversion" | sed  -e 's|[0-9]*\.\([0-9]*\)\.[0-9]*$|\1|')
 echo "previousminorversion=$previousminorversion"
@@ -35,4 +35,3 @@ then
 else 
     echo "don't preserve old document"
 fi
-ls docs