inject sha256 labels for all target packages

docker_templates/packages.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import gzip
 import string
 import re
 import urllib.request
@@ -22,24 +23,26 @@
 #  r'\d(?!Version\:\s)(.+)(?=(~\w+\n))' but works without a trailing ~
 version_pattern = r'(?<=Version: )\d+\.\d+\.\d+\-\d+'
 
+sha256_pattern = r'(?<=SHA256: )[0-9a-f]{64}'
+
 packagePatternTemplateLookup = {
     'gazebo_packages':  string.Template(r'(\bPackage: gazebo$gazebo_version\n)(.*?(?:\r*\n{2}))'),
     'ros_packages':     string.Template(r'(\bPackage: ros-$rosdistro_name-$package\n)(.*?(?:\r*\n{2}))'),
     'ros2_packages':    string.Template(r'(\bPackage: ros-$ros2distro_name-$package\n)(.*?(?:\r*\n{2}))'),
 }
 
 indexUrlTemplateLookup = {
-    'gazebo_packages':  string.Template('http://packages.osrfoundation.org/gazebo/$os_name-$release/dists/$os_code_name/main/binary-$arch/Packages'),
-    'ros_packages':     string.Template('http://packages.ros.org/ros/ubuntu/dists/$os_code_name/main/binary-$arch/Packages'),
-    'ros2_packages':    string.Template('http://packages.ros.org/ros2/ubuntu/dists/$os_code_name/main/binary-$arch/Packages'),
-    'ros_packages_snapshots':    string.Template('http://snapshots.ros.org/$rosdistro_name/final/ubuntu/dists/$os_code_name/main/binary-$arch/Packages'),
-    'ros2_packages_snapshots':    string.Template('http://snapshots.ros.org/$ros2distro_name/final/ubuntu/dists/$os_code_name/main/binary-$arch/Packages'),
+    'gazebo_packages':  string.Template('http://packages.osrfoundation.org/gazebo/$os_name-$release/dists/$os_code_name/main/binary-$arch/Packages.gz'),
+    'ros_packages':     string.Template('http://packages.ros.org/ros/ubuntu/dists/$os_code_name/main/binary-$arch/Packages.gz'),
+    'ros2_packages':    string.Template('http://packages.ros.org/ros2/ubuntu/dists/$os_code_name/main/binary-$arch/Packages.gz'),
+    'ros_packages_snapshots':    string.Template('http://snapshots.ros.org/$rosdistro_name/final/ubuntu/dists/$os_code_name/main/binary-$arch/Packages.gz'),
+    'ros2_packages_snapshots':    string.Template('http://snapshots.ros.org/$ros2distro_name/final/ubuntu/dists/$os_code_name/main/binary-$arch/Packages.gz'),
 }
 
-packageNameVersionTemplateLookup = {
-    'gazebo_packages':  string.Template('$package=$package_version*'),
-    'ros_packages':     string.Template('ros-$rosdistro_name-$package=$package_version*'),
-    'ros2_packages':    string.Template('ros-$ros2distro_name-$package=$package_version*'),
+packageVersionTemplateLookup = {
+    'gazebo_packages':  string.Template('=$package_version*'),
+    'ros_packages':     string.Template('=$package_version*'),
+    'ros2_packages':    string.Template('=$package_version*'),
 }
 
 packageNameTemplateLookup = {
@@ -54,7 +57,7 @@ def getPackageIndex(data, package_index_url):
     # Download package index
     req = urllib.request.Request(package_index_url)
     with urllib.request.urlopen(req) as response:
-        package_index = response.read().decode('utf-8')
+        package_index = gzip.decompress(response.read()).decode('utf-8')
 
     return package_index
 
@@ -66,12 +69,27 @@ def getPackagePattern(data, package_pattern_template, package):
 
     return package_pattern
 
-def getPackageVersion(data, package_pattern, package, package_index):
-    """Use package index to get package version"""
+def getPackageInfo(package_pattern, package_index):
+    """Use package index to get package info"""
 
-    # Parse for version_number
+    # Parse for package info
     matchs = re.search(package_pattern, package_index)
     package_info = matchs.group(0)
+
+    return package_info
+
+def getPackageSHA256(package_info):
+    """Use package info to get package sha256"""
+
+    # Parse for SHA56
+    package_sha256 = re.search(sha256_pattern, package_info).group(0) # extract sha256
+
+    return package_sha256
+
+def getPackageVersion(package_info):
+    """Use package info to get package version"""
+
+    # Parse for version_number
     package_version = re.search(version_pattern, package_info).group(0) # extract version_number
 
     return package_version
@@ -81,33 +99,26 @@ def getPackageVersions(data, package_index, packages, package_type):
 
     package_versions = []
 
-    if data['version'] != False:
-        for package in packages:
-
-            # Determine package_pattern
-            package_pattern_template = packagePatternTemplateLookup[package_type]
-            package_pattern = getPackagePattern(data, package_pattern_template, package)
-
-            # Determine package_version
-            package_version = getPackageVersion(data, package_pattern, package, package_index)
-
-            # Determine package_pattern
-            package_name_template = packageNameVersionTemplateLookup[package_type]
-            package_name = package_name_template.substitute(data, package=package, package_version=package_version)
-
-            package_versions.append(package_name)
+    # Determine package_pattern
+    package_pattern_template = packagePatternTemplateLookup[package_type]
+    package_name_template = packageNameTemplateLookup[package_type]
+    package_version_template = packageVersionTemplateLookup[package_type]
 
-        return package_versions
-    else:
-        for package in packages:
+    for package in packages:
+        package_pattern = getPackagePattern(data, package_pattern_template, package)
+        package_name = package_name_template.substitute(data, package=package)
+        package_info = getPackageInfo(package_pattern, package_index)
+        package_sha256 = getPackageSHA256(package_info)
 
-            # Determine package_pattern
-            package_name_template = packageNameTemplateLookup[package_type]
-            package_name = package_name_template.substitute(data, package=package)
+        if data['version'] != False:
+            version = getPackageVersion(package_info)
+            package_version = package_version_template.substitute(data, package_version=version)
+        else:        
+            package_version=''
 
-            package_versions.append(package_name)
+        package_versions.append(dict(name=package_name, version=package_version, sha256=package_sha256))
 
-        return package_versions
+    return package_versions
 
 def expandPackages(data):
     for package_type in indexUrlTemplateLookup:

docker_templates/templates/docker_images/create_drcsim_image.Dockerfile.em

@@ -41,11 +41,11 @@ RUN . /etc/os-release \
 @[if 'gazebo_packages' in locals()]@
 @[  if gazebo_packages]@
 
-# install ros packages
-RUN apt-get update && apt-get install -y \
-    @(' \\\n    '.join(gazebo_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
-
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='gazebo',
+    packages=gazebo_packages,
+))@
 @[  end if]@
 @[end if]@
 @[if 'entrypoint_name' in locals()]@

docker_templates/templates/docker_images/create_gzclient_image.Dockerfile.em

@@ -19,7 +19,8 @@
     upstream_packages=upstream_packages if 'upstream_packages' in locals() else [],
 ))@
 @
-# install gazebo packages
-RUN apt-get update && apt-get install -q -y --no-install-recommends \
-    @(' \\\n    '.join(gazebo_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='gazebo',
+    packages=gazebo_packages,
+))@

docker_templates/templates/docker_images/create_gzserverX_image.Dockerfile.em

@@ -60,10 +60,11 @@ ENV LC_ALL C.UTF-8
 RUN rosdep init \
     && rosdep update
 
-# install ros packages
-RUN apt-get update && apt-get install -y \
-    @(' \\\n    '.join(ros_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='ros',
+    packages=ros_packages,
+))@
 @[  end if]@
 @[end if]@
 
@@ -77,10 +78,11 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys D2486D2DD8
 RUN . /etc/os-release \
     && echo "deb http://packages.osrfoundation.org/gazebo/$ID-stable `lsb_release -cs` main" > /etc/apt/sources.list.d/gazebo-latest.list
 
-# install gazebo packages
-RUN apt-get update && apt-get install -q -y \
-    @(' \\\n    '.join(gazebo_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='gazebo',
+    packages=gazebo_packages,
+))@
 @[end if]@
 @[end if]@
 

docker_templates/templates/docker_images/create_gzserver_image.Dockerfile.em

@@ -38,11 +38,11 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys D2486D2DD8
 RUN . /etc/os-release \
     && echo "deb http://packages.osrfoundation.org/gazebo/$ID-stable `lsb_release -sc` main" > /etc/apt/sources.list.d/gazebo-latest.list
 
-# install gazebo packages
-RUN apt-get update && apt-get install -q -y --no-install-recommends \
-    @(' \\\n    '.join(gazebo_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
-
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='gazebo',
+    packages=gazebo_packages,
+))@
 # setup environment
 EXPOSE 11345
 

docker_templates/templates/docker_images/create_gzweb_image.Dockerfile.em

@@ -24,11 +24,11 @@ template_dependencies = [
     upstream_packages=upstream_packages if 'upstream_packages' in locals() else [],
 ))@
 @
-# install gazebo packages
-RUN apt-get update && apt-get install -q -y --no-install-recommends \
-    @(' \\\n    '.join(gazebo_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
-
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='gazebo',
+    packages=gazebo_packages,
+))@
 # clone gzweb
 ENV GZWEB_WS /root/gzweb
 RUN hg clone https://bitbucket.org/osrf/gzweb $GZWEB_WS

docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em

@@ -52,11 +52,11 @@ ENV LC_ALL C.UTF-8
 
 ENV ROS_DISTRO @rosdistro_name
 
-# install ros packages
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    @(' \\\n    '.join(ros_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
-
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='ros',
+    packages=ros_packages,
+))@
 @[if 'entrypoint_name' in locals()]@
 @[  if entrypoint_name]@
 @{

docker_templates/templates/docker_images/create_ros_image.Dockerfile.em

@@ -34,10 +34,10 @@ RUN rosdep init && \
 @
 @[if 'ros_packages' in locals()]@
 @[  if ros_packages]@
-# install ros packages
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    @(' \\\n    '.join(ros_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
-
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='ros',
+    packages=ros_packages,
+))@
 @[  end if]@
 @[end if]@

docker_templates/templates/docker_images_legacy/create_ros_core_image.Dockerfile.em

@@ -48,7 +48,7 @@ ENV LANG en_US.UTF-8
 # install ros packages
 ENV ROS_DISTRO @rosdistro_name
 RUN apt-get update && apt-get install -y \
-    @(' \\\n    '.join(ros_packages))@  \
+    @(' \\\n    '.join('{name}{version}'.format(**p) for p in ros_packages))@  \
     && rm -rf /var/lib/apt/lists/* \
     && mkdir /var/lib/apt/lists/partial
 

docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em

@@ -66,11 +66,11 @@ RUN pip3 install -U \
 @[  end if]@
 @[end if]@
 @
-# install ros2 packages
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    @(' \\\n    '.join(ros2_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
-
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='ros2',
+    packages=ros2_packages,
+))@
 @[if 'entrypoint_name' in locals()]@
 @[  if entrypoint_name]@
 @{

docker_templates/templates/docker_images_ros2/create_ros_image.Dockerfile.em

@@ -52,13 +52,11 @@ RUN pip3 install -U \
 @[  end if]@
 @[end if]@
 @[if 'ros2_packages' in locals()]@
-@[  if ros2_packages]@
-# install ros2 packages
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    @(' \\\n    '.join(ros2_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
-
-@[  end if]@
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='ros2',
+    packages=ros2_packages,
+))@
 @[end if]@
 @[if 'entrypoint_name' in locals()]@
 @[  if entrypoint_name]@

docker_templates/templates/docker_images_ros2/ros1_bridge/create_ros_ros1_bridge_image.Dockerfile.em

@@ -45,23 +45,20 @@ RUN pip3 install -U \
 
 ENV ROS1_DISTRO @rosdistro_name
 ENV ROS2_DISTRO @ros2distro_name
-@[if 'ros_packages' in locals()]@
-@[  if ros_packages]@
-# install ros packages
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    @(' \\\n    '.join(ros_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
 
-@[  end if]@
+@[if 'ros_packages' in locals()]@
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='ros',
+    packages=ros_packages,
+))@
 @[end if]@
 @[if 'ros2_packages' in locals()]@
-@[  if ros2_packages]@
-# install ros2 packages
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    @(' \\\n    '.join(ros2_packages))@  \
-    && rm -rf /var/lib/apt/lists/*
-
-@[  end if]@
+@(TEMPLATE(
+    'snippet/label_and_install_package_list.Dockerfile.em',
+    group='ros2',
+    packages=ros2_packages,
+))@
 @[end if]@
 @[if 'downstream_packages' in locals()]@
 @[  if downstream_packages]@

docker_templates/templates/snippet/label_and_install_package_list.Dockerfile.em

@@ -0,0 +1,10 @@
+@[if packages]@
+# label @group packages
+LABEL @(' \\\n      '.join('sha256.{name}={sha256}'.format(**p) for p in packages))@
+
+
+# install @group packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    @(' \\\n    '.join('{name}{version}'.format(**p) for p in packages))@  \
+    && rm -rf /var/lib/apt/lists/*
+@[end if]