Create release source artifact containing all submodules

build.py

@@ -100,13 +100,46 @@ def choose_omero_version():
         if not omero_build:
             omero_version = re.sub("([-]DEV)?-\d+-[a-f0-9]+(-dirty)?",\
                     "-DEV", omero_version)
-        return [ "-Domero.version=%s%s" % (omero_version, omero_build) ]
     except:
         print "Error getting version for BUILD_NUMBER=%s" % omero_build
         if err:
             print err
         sys.exit(1)
 
+    command = [ find_java(), "omero",BUILD_PY,"-q","plainversion" ]
+    err = ""
+    try:
+        p = popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        omero_plain_version, err = p.communicate()
+        omero_plain_version = omero_plain_version.split()[1]
+        if not omero_build:
+            omero_plain_version = re.sub("([-]DEV)?-\d+-[a-f0-9]+(-dirty)?",\
+                    "-DEV", omero_plain_version)
+    except:
+        print "Error getting plain version for BUILD_NUMBER=%s" % omero_build
+        if err:
+            print err
+        sys.exit(1)
+
+    command = [ find_java(), "omero",BUILD_PY,"-q","shortversion" ]
+    err = ""
+    try:
+        p = popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        omero_short_version, err = p.communicate()
+        omero_short_version = omero_short_version.split()[1]
+        if not omero_build:
+            omero_short_version = re.sub("([-]DEV)?-\d+-[a-f0-9]+(-dirty)?",\
+                    "-DEV", omero_short_version)
+    except:
+        print "Error getting short version for BUILD_NUMBER=%s" % omero_build
+        if err:
+            print err
+        sys.exit(1)
+
+    return [ "-Domero.version=%s%s" % (omero_version, omero_build),
+             "-Domero.plainversion=%s" % (omero_plain_version),
+             "-Domero.shortversion=%s" % (omero_short_version) ]
+
 
 def handle_tools(args):
     _ = os.path.sep.join

build.xml

@@ -55,12 +55,17 @@ To get started using Eclipse, execute "./build.py build-dev" and import the top-
     <property name="gitmatch" value="v.[0-9]*.[0-9]*.[0-9]*"/>
     <property name="import.dir" value="${basedir}/components/antlib/resources"/>
     <import file="${import.dir}/global.xml"/>
+    <import file="${import.dir}/gitversion.xml" optional="true"/>
 
     <target name="init" depends="check-bioformats,check-ivy,check-scons,check-ice">
         <mkdir dir="${blitz.comp}/generated"/>
         <mkdir dir="${blitz.comp}/target/generated/src"/>
     </target>
 
+    <target name="check-git">
+      <available file=".git" type="dir" property="git.exists"/>
+    </target>
+
     <target name="check-bioformats" unless="bioformats.done">
         <!-- Prevent future invocations -->
         <property name="bioformats.check.done" value="done"/>
@@ -422,8 +427,8 @@ To get started using Eclipse, execute "./build.py build-dev" and import the top-
         </scons_py>
     </target>
 
-    <target name="release-all" description="Produce all release artifacts"
-        depends="release-clients,release-webstart,release-zip,release-src,release-docs">
+    <target name="release-all" description="Produce all binary release artifacts"
+        depends="release-clients,release-webstart,release-zip,release-docs">
         <!-- Clients must come first to produce the zips needed by webstart,
         followed by webstart in order to add jars to dist/, followed by zip -->
     </target>
@@ -438,17 +443,26 @@ To get started using Eclipse, execute "./build.py build-dev" and import the top-
         </zip>
     </target>
 
-    <target name="release-src" description="Zip the git source into OMERO.source-${omero.version}.zip">
-        <git command="archive" output="release-src-command" failonerror="true">
-            <args>
-                <arg value="--format=zip"/>
-                <arg value="HEAD"/>
-                <arg value="-o"/>
-                <arg value="${target.dir}/OMERO.source-${omero.version}.zip"/>
-            </args>
-        </git>
+    <target name="release-src-embed" description="Package the release source tree into openmicroscopy-${omero.plainversion}.zip (not functional)"
+            depends="check-git" unless="git.exists">
+         <fail message="Releasing is only possible from a git repository"/>
+    </target>
+
+    <target name="release-src-git" description="Package the git source tree into openmicroscopy-${omero.plainversion}.zip"
+            depends="check-git" if="git.exists">
+        <mkdir dir="${target.dir}"/>
+        <exec executable="python" failonerror="true">
+            <arg value="${omero.home}/components/antlib/scripts/source-archive"/>
+            <arg value="openmicroscopy"/>
+            <arg value="${omero.shortversion}"/>
+            <arg value="${omero.plainversion}"/>
+            <arg value="components/antlib/resources/gitversion.xml"/>
+            <arg value="${target.dir}"/>
+        </exec>
     </target>
 
+    <target name="release-src" depends="release-src-embed,release-src-git"/>
+
     <target name="release-clients" description="Zip the Python, Java, and Matlab zips">
         <zip destfile="${omero.home}/target/OMERO.py-${omero.version}.zip">
             <zipfileset dir="${dist.dir}" prefix="OMERO.py-${omero.version}"
@@ -553,14 +567,12 @@ To get started using Eclipse, execute "./build.py build-dev" and import the top-
             <zipfileset dir="${dist.dir}/docs" prefix="OMERO.docs-${omero.version}"/>
         </zip>
 
-        <gitdescribe select="\2"> </gitdescribe>
-
         <exec executable="make" failonerror="true" dir="${sphinx.dir}">
-            <env key="SOURCE_BRANCH" value="v.${version.describe}"/>
-            <env key="OMERO_RELEASE" value="${version.describe}"/>
+            <env key="SOURCE_BRANCH" value="v.${omero.shortversion}"/>
+            <env key="OMERO_RELEASE" value="${omero.shortversion}"/>
             <arg line="clean html latexpdf"/>
         </exec>
-        <copy file="${sphinx.dir}/_build/latex/OMERO-${version.describe}.pdf" tofile="${omero.home}/target/OMERO-${version.describe}.pdf" overwrite="true"/>
+        <copy file="${sphinx.dir}/_build/latex/OMERO-${omero.shortversion}.pdf" tofile="${omero.home}/target/OMERO-${omero.shortversion}.pdf" overwrite="true"/>
         <zip destfile="${omero.home}/target/OMERO.sphinx-${omero.version}.zip">
             <zipfileset dir="${sphinx.dir}/_build/html/" prefix="OMERO.sphinx-${omero.version}"/>
         </zip>
@@ -1039,12 +1051,42 @@ omero.version=${omero.version}
         </sequential>
     </macrodef>
 
-    <target name="version">
+    <target name="version-embed" if="omero.plainversion">
+        <echo>${omero.plainversion}-ice${versions.ice_lib}</echo>
+    </target>
+
+    <target name="version-git" unless="omero.plainversion">
         <gitdescribe select="\2\3\5\6">
         </gitdescribe>
         <echo>${version.describe}-ice${versions.ice_lib}</echo>
     </target>
 
+    <target name="version" depends="version-embed,version-git"/>
+
+    <target name="plainversion-embed" if="omero.plainversion">
+        <echo>${omero.plainversion}</echo>
+    </target>
+
+    <target name="plainversion-git" unless="omero.plainversion">
+        <gitdescribe select="\2\3\5\6">
+        </gitdescribe>
+        <echo>${version.describe}</echo>
+    </target>
+
+    <target name="plainversion" depends="plainversion-embed,plainversion-git"/>
+
+    <target name="shortversion-embed" if="omero.shortversion">
+        <echo>${omero.shortversion}</echo>
+    </target>
+
+    <target name="shortversion-git" unless="omero.shortversion">
+        <gitdescribe select="\2">
+        </gitdescribe>
+        <echo>${version.describe}</echo>
+    </target>
+
+    <target name="shortversion" depends="shortversion-embed,shortversion-git"/>
+
     <target name="print-mem">
         <echo>
             *.maxmem.* settings

components/antlib/scripts/source-archive

@@ -0,0 +1,97 @@
+#!/usr/bin/python
+
+from __future__ import print_function
+
+import os
+from subprocess import call, Popen, PIPE
+import sys
+import zipfile
+
+# Due to "git archive" not supporting archiving of submodules in
+# addition to the base tree, this requires additional support in order
+# to create a complete and functional source archive.
+#
+# This script archives the base tree, and then uses "git submodule
+# foreach" to archive each submodule separately, setting the correct
+# path prefix for each archive, so that they may all be unpacked in
+# the same root to result in a complete and functional source tree.
+# It then repacks each of these zip files into a single zip which is
+# the source release, taking care to preserve timestamps and exectute
+# permissions, etc.  This is done via ZipInfo objects, and the
+# repacking is done entirely in memory so that this should work on any
+# platform irrespective of filesystem support for the archive
+# attributes.  It excludes .gitignore files at this point to avoid
+# polluting the release with version control files.
+
+if __name__ == "__main__":
+    if len(sys.argv) != 6:
+        raise Exception('Usage: %s releasename shortversion fullversion versionfile targetdir')
+
+    release = sys.argv[1]
+    shortversion = sys.argv[2]
+    version = sys.argv[3]
+    versionfile = sys.argv[4]
+    target = os.path.abspath(sys.argv[5])
+    release = "%s-%s" % (release,version)
+
+    if not os.path.isdir('.git'):
+        raise Exception('Releasing is only possible from a git repository')
+
+    print("Releasing %s" % (release))
+    sys.stdout.flush()
+
+    # Create base archive
+    base_archive_status = call(['git', 'archive', '--format', 'zip',
+                                '--prefix', "%s/" % (release),
+                                '--output', "%s/%s-base.zip" % (target, release),
+                                'HEAD'])
+    if base_archive_status != 0:
+        raise Exception('Failed to create git base archive')
+
+    zips = list(["%s/%s-base.zip" % (target, release)])
+
+    # Create submodule archives
+    submodule_archive = Popen(['git', 'submodule', 'foreach', '--quiet', '--recursive',
+                               "npath=\"$(echo \"$path\" | tr / _)\"; \
+                                zip=\"%s/%s-submod-${npath}.zip\"; \
+                                git archive --format zip --prefix \"%s/${path}/\" --output \"${zip}\" HEAD || exit 1; \
+                                echo \"${zip}\"" % (target, release, release)],
+                              stdout=PIPE)
+    submodule_zips = submodule_archive.communicate()[0]
+    if submodule_archive.returncode != 0:
+        raise Exception('Failed to create git submodule archives')
+
+    zips.extend(submodule_zips.splitlines())
+
+    # Create destination zip file
+    print("  - creating %s/%s.zip" % (target, release))
+    sys.stdout.flush()
+    basezip = zipfile.ZipFile("%s/%s.zip" % (target, release), 'w')
+
+    # Repack each of the separate zips into the destination zip
+    for name in zips:
+        subzip = zipfile.ZipFile(name, 'r')
+        print("  - repacking %s" % (name))
+        sys.stdout.flush()
+        # Iterate over the ZipInfo objects from the archive
+        for info in subzip.infolist():
+            # Skip unwanted git and travis files
+            if os.path.basename(info.filename) == '.gitignore' or os.path.basename(info.filename) == '.gitmodule' or os.path.basename(info.filename) == '.travis.yml':
+                continue
+            # Repack a single zip object; preserve the metadata
+            # directly via the ZipInfo object and rewrite the content
+            # (which unfortunately requires decompression and
+            # recompression rather than a direct copy)
+            basezip.writestr(info, subzip.open(info.filename).read())
+
+        # Remove repacked zip
+        os.remove(name)
+
+    # Embed release number
+    basezip.writestr("%s/%s" % (release, versionfile),
+"""<?xml version="1.0" encoding="utf-8"?>
+<project name="gitversion" basedir=".">
+        <property name="omero.shortversion" value="%s"/>
+        <property name="omero.plainversion" value="%s"/>
+</project>
+""" % (shortversion, version))

docs/hudson/OMERO.sh

@@ -15,6 +15,10 @@ echo Building $OMERO_BRANCH
 ./build.py clean
 ./build.py build-default test-compile
 ./build.py release-all
+if [ -d .git ]
+then
+  ./build.py release-src
+fi
 
 # Log information
 echo BUILD_NUMBER=$BUILD_NUMBER > target/$OMERO_BRANCH.log