Crawl Oracle Linux UEK kernels

.circleci/config.yml

@@ -154,9 +154,9 @@ workflows:
     - combine:
         requires:
         - repackage
-    - collector:
-        requires:
-        - combine
+#    - collector:
+#        requires:
+#        - combine
     - crawl:
         filters:
           branches:

kernel-crawler/Makefile

@@ -42,8 +42,12 @@ crawl-ubuntu-hwe: build-crawl-container
 crawl-ubuntu-gke: build-crawl-container
 	docker run --rm -i kernel-crawler crawl Ubuntu-GKE > $(CRAWLED_PACKAGE_DIR)/ubuntu-gke.txt
 
+.PHONY: crawl-oracle-uek
+crawl-oracle-uek: build-crawl-container
+	docker run --rm -i kernel-crawler crawl Oracle-UEK5 > $(CRAWLED_PACKAGE_DIR)/oracle-uek.txt
+
 .PHONY: crawl
-crawl: build-crawl-container crawl-centos crawl-kops crawl-amazon crawl-debian crawl-ubuntu-hwe crawl-ubuntu-gke
+crawl: build-crawl-container crawl-centos crawl-kops crawl-amazon crawl-debian crawl-ubuntu-hwe crawl-ubuntu-gke crawl-oracle-uek
 	docker run --rm -i kernel-crawler crawl CoreOS > $(CRAWLED_PACKAGE_DIR)/coreos.txt
 	docker run --rm -i kernel-crawler crawl Container-OptimizedOS > $(CRAWLED_PACKAGE_DIR)/cos.txt
 	docker run --rm -i kernel-crawler crawl Ubuntu > $(CRAWLED_PACKAGE_DIR)/ubuntu-standard.txt

kernel-crawler/kernel-crawler.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 #### LICENSING:
 #### This file is derived from sysdig, in scripts/kernel-crawler.py.
@@ -373,7 +373,15 @@
             "page_pattern" : "/html/body//a[regex:test(@href, '^linux-headers-[4-9].*-gke.*_amd64.deb$')]/@href",
             "exclude_patterns": ubuntu_excludes
         },
-    ]
+    ],
+    "Oracle-UEK5": [
+    	{
+    		"root": "http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/",
+    		"discovery_pattern": "",
+    		"page_pattern": "/html/body//a[regex:test(@href, '^getPackage/kernel-uek-devel-[0-9].*\.x86_64\.rpm$')]/@href",
+    		"subdirs": [""],
+    	},
+    ],
 }
 
 

kernel-package-lists/oracle-uek.txt

@@ -0,0 +1,35 @@
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.5.0.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.10.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.5.1.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.11.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1911.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.1.2.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.4.2.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.5.2.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.6.2.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.12.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1912.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.3.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.4.3.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.6.3.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.13.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1923.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1933.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.4.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.6.4.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.14.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1914.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.5.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.15.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1915.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.6.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1844.0.6.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1916.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.7.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1917.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1818.0.8.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.4.8.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1902.0.9.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1929.el7uek.x86_64.rpm
+http://yum.oracle.com/repo/OracleLinux/OL7/developer_UEKR5/x86_64/getPackage/kernel-uek-devel-4.14.35-1910a.el7uek.x86_64.rpm

kernel-package-lists/reformat.yml

@@ -87,3 +87,9 @@
   type: linuxkit
   file: linuxkit.txt
   reformat: single
+
+- name: oracle
+  description: Oracle UEK Kernels
+  type: oracle
+  file: oracle-uek.txt
+  reformat: single

packers/entrypoint

@@ -67,6 +67,11 @@ main() {
             repackage_linuxkit "$checksum" "$output_dir" "${packages[@]}"
         ;;
 
+        oracle)
+        	log 'Repackaging oracle'
+        	repackage_oracle "$checksum" "$output_dir" "${packages[@]}"
+        ;;
+
         *)
             log 'unknown distro'
             return 1
@@ -460,6 +465,50 @@ repackage_linuxkit() {
     )
 }
 
+# Repackages a linuxkit kernel into a bundle tarball.
+repackage_oracle() {
+    if [[ $# -ne 3 ]]; then
+        log "invalid number of arguments"
+        return 1
+    fi
+
+    local checksum="$1"
+    local output_dir="$2"
+    local input_package_1="$3"
+
+    local linux_src="$(mktemp -d)"
+    (
+    	cd "${linux_src}"
+    	rpm2cpio "$input_package_1" | cpio -idm
+    	kernel_src_dirs=(usr/src/kernels/*)
+    	if [[ "${#kernel_src_dirs[@]}}" -ne 1 ]]; then
+    		log "More than one kernel source directory: ${kernel_src_dirs[*]}"
+    		exit 1
+    	fi
+    	kernel_src_dir="${kernel_src_dirs[0]}"
+    	kernel_uname="$(basename "$kernel_src_dir")"
+
+    	cd "$kernel_src_dir"
+
+        log "Kernel version is ${kernel_uname}"
+
+        # Generate bundle meta files
+        meta_dir="$(bundle_meta "$checksum" 'oracle' "$kernel_uname" '.')"
+
+        # Delete all *.c files, excluding scripts directory
+        find . ! \( -type d \) -not -path "./scripts/*" -name "*.c" -delete
+
+        local bundle_path="${output_dir}/bundle-${kernel_uname}.tgz"
+
+        # Compress only part of the file hierarchy into a tarball.
+        tar --create --dereference --hard-dereference --file - \
+        	--exclude ./scripts/dtc \
+            --directory "$meta_dir" . \
+            --directory . . \
+        | pigz -9 -c > "${bundle_path}"
+    )
+}
+
 # Populates a temporary directory with files containing various pieces of
 # bundle meta-information. All files are prefixed with 'BUNDLE_'.
 bundle_meta() {

scripts/robo-crawl-commit

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 set -eu
 
 main() {
@@ -18,7 +18,11 @@ main() {
     # Create a commit with any modified files, and push it to GitHub.
     git add "$CRAWLED_PACKAGE_DIR"
     git -c "user.name=roxbot" -c "user.email=roxbot@stackrox.com" commit --message "🤖 Updated crawled packages"
-    git push origin
+    upstream_arg=()
+    if [[ "$CIRCLE_BRANCH" =~ .*dev-crawl.* ]]; then
+    	upstream_arg=("$CIRCLE_BRANCH")
+    fi
+    git push origin "${upstream_arg[@]}"
     echo "Crawled packages have been pushed."
 }