Refactor python code: commonlize scripts utils and bootstrap (#437)

* Refactor python code: commonlize scripts utils and bootstrap

And in order to do that, I intended to use the most up-to-date / robust
flavor of a functionality found.

Took this chance also to solve some flake8 warnings that I could.

* Remove bootstrap's dependency on gsutil

That will assist supporting distros that doesn't have gsutil installed.

daisy_workflows/image_build/debian/build.py

@@ -50,39 +50,42 @@ def main():
       'bootstrap_vz_manifest', raise_on_not_found=True)
   bvz_version = utils.GetMetadataParam(
       'bootstrap_vz_version', raise_on_not_found=True)
-  repo = utils.GetMetadataParam('google_cloud_repo', raise_on_not_found=True).strip()
-  image_dest = utils.GetMetadataParam('image_dest', raise_on_not_found=True)
-  outs_path = utils.GetMetadataParam('daisy-outs-path', raise_on_not_found=True)
+  repo = utils.GetMetadataParam('google_cloud_repo',
+      raise_on_not_found=True).strip()
+  image_dest = utils.GetMetadataParam('image_dest',
+      raise_on_not_found=True)
+  outs_path = utils.GetMetadataParam('daisy-outs-path',
+      raise_on_not_found=True)
   if repo not in REPOS:
     raise ValueError(
         'Metadata "google_cloud_repo" must be one of %s.' % REPOS)
 
-  utils.Status('Bootstrap_vz manifest: %s' % bvz_manifest)
-  utils.Status('Bootstrap_vz version: %s' % bvz_version)
-  utils.Status('Google Cloud repo: %s' % repo)
+  utils.LogStatus('Bootstrap_vz manifest: %s' % bvz_manifest)
+  utils.LogStatus('Bootstrap_vz version: %s' % bvz_version)
+  utils.LogStatus('Google Cloud repo: %s' % repo)
 
   # Download and setup bootstrap_vz.
   bvz_url = 'https://github.com/andsens/bootstrap-vz/archive/%s.zip'
   bvz_url %= bvz_version
   bvz_zip_dir = 'bvz_zip'
-  utils.Status('Downloading bootstrap-vz at commit %s' % bvz_version)
+  utils.LogStatus('Downloading bootstrap-vz at commit %s' % bvz_version)
   urllib.urlretrieve(bvz_url, 'bvz.zip')
   with zipfile.ZipFile('bvz.zip', 'r') as z:
     z.extractall(bvz_zip_dir)
-  utils.Status('Downloaded and extracted %s to bvz.zip.' % bvz_url)
+  utils.LogStatus('Downloaded and extracted %s to bvz.zip.' % bvz_url)
   bvz_zip_contents = [d for d in os.listdir(bvz_zip_dir)]
   bvz_zip_subdir = os.path.join(bvz_zip_dir, bvz_zip_contents[0])
   utils.Execute(['mv', bvz_zip_subdir, BVZ_DIR])
-  utils.Status('Moved bootstrap_vz from %s to %s.' % (bvz_zip_subdir, BVZ_DIR))
+  utils.LogStatus('Moved bootstrap_vz from %s to %s.' % (bvz_zip_subdir, BVZ_DIR))
   bvz_bin = os.path.join(BVZ_DIR, 'bootstrap-vz')
   utils.MakeExecutable(bvz_bin)
-  utils.Status('Made %s executable.' % bvz_bin)
+  utils.LogStatus('Made %s executable.' % bvz_bin)
   bvz_manifest_file = os.path.join(BVZ_DIR, 'manifests', bvz_manifest)
 
   # Inject Google Cloud test repo plugin if using staging or unstable repos.
   # This is used to test new package releases in images.
   if repo != 'stable':
-    utils.Status('Adding Google Cloud test repos plugin for bootstrapvz.')
+    utils.LogStatus('Adding Google Cloud test repos plugin for bootstrapvz.')
     repo_plugin_dir = '/build_files/google_cloud_test_repos'
     bvz_plugins = os.path.join(BVZ_DIR, 'bootstrapvz', 'plugins')
     shutil.move(repo_plugin_dir, bvz_plugins)
@@ -96,33 +99,34 @@ def main():
 
   # Run bootstrap_vz build.
   cmd = [bvz_bin, '--debug', bvz_manifest_file]
-  utils.Status('Starting build in %s with params: %s' % (BVZ_DIR, str(cmd)))
+  utils.LogStatus('Starting build in %s with params: %s' % (BVZ_DIR, str(cmd)))
   utils.Execute(cmd, cwd=BVZ_DIR)
 
   # Upload tar.
   image_tar_gz = '/target/disk.tar.gz'
   if os.path.exists(image_tar_gz):
-    utils.Status('Saving %s to %s' % (image_tar_gz, image_dest))
+    utils.LogStatus('Saving %s to %s' % (image_tar_gz, image_dest))
     utils.Gsutil(['cp', image_tar_gz, image_dest])
 
   # Create and upload the synopsis of the image.
-  utils.Status('Creating image synopsis.')
+  utils.LogStatus('Creating image synopsis.')
   synopsis = {}
   packages = collections.OrderedDict()
-  _, output, _ = utils.Execute(['dpkg-query', '-W'], capture_output=True)
+  _, output = utils.Execute(['dpkg-query', '-W'], capture_output=True)
   for line in output.split('\n')[:-1]:  # Last line is an empty line.
     parts = line.split()
     packages[parts[0]] = parts[1]
   synopsis['installed_packages'] = packages
   with open('/tmp/synopsis.json', 'w') as f:
     f.write(json.dumps(synopsis))
-  utils.Status('Uploading image synopsis.')
+  utils.LogStatus('Uploading image synopsis.')
   synopsis_dest = os.path.join(outs_path, 'synopsis.json')
   utils.Gsutil(['cp', '/tmp/synopsis.json', synopsis_dest])
 
+
 if __name__ == '__main__':
   try:
     main()
-    utils.Success('Debian build was successful!')
+    utils.LogSuccess('Debian build was successful!')
   except:
-    utils.Fail('Debian build failed!')
+    utils.LogFail('Debian build failed!')

daisy_workflows/image_build/debian/debian.wf.json

@@ -13,9 +13,9 @@
   },
   "Sources": {
     "build_files/build.py": "./build.py",
-    "build_files/utils.py": "../linux_common/utils.py",
+    "build_files/utils.py": "../../linux_common/utils.py",
     "build_files/google_cloud_test_repos": "./google_cloud_test_repos/",
-    "startup_script": "../linux_common/bootstrap.py"
+    "startup_script": "../../linux_common/bootstrap.py"
   },
   "Steps": {
     "setup": {
@@ -37,8 +37,9 @@
           "Metadata": {
             "bootstrap_vz_manifest": "${bootstrap_vz_manifest}",
             "bootstrap_vz_version": "${bootstrap_vz_version}",
-            "build_files_gcs_dir": "${SOURCESPATH}/build_files",
-            "build_script": "build.py",
+            "files_gcs_dir": "${SOURCESPATH}/build_files",
+            "script": "build.py",
+            "prefix": "Build",
             "google_cloud_repo": "${google_cloud_repo}",
             "image_dest": "${image_dest}"
           },

daisy_workflows/image_build/enterprise_linux/build_installer.py

@@ -43,9 +43,9 @@ def main():
   sap_apps = utils.GetMetadataParam('rhel_sap_apps', raise_on_not_found=False)
   sap_apps = sap_apps == 'true'
 
-  utils.Status('EL Release: %s' % release)
-  utils.Status('Google Cloud repo: %s' % repo)
-  utils.Status('Build working directory: %s' % os.getcwd())
+  utils.LogStatus('EL Release: %s' % release)
+  utils.LogStatus('Google Cloud repo: %s' % repo)
+  utils.LogStatus('Build working directory: %s' % os.getcwd())
 
   iso_file = 'installer.iso'
 
@@ -59,7 +59,7 @@ def main():
 
   # Write the installer disk. Write extlinux MBR, create partition,
   # copy installer ISO and ISO boot files over.
-  utils.Status('Writing installer disk.')
+  utils.LogStatus('Writing installer disk.')
   utils.Execute(['parted', '/dev/sdb', 'mklabel', 'msdos'])
   utils.Execute(['sync'])
   utils.Execute(['parted', '/dev/sdb', 'mkpart', 'primary', '1MB', '100%'])
@@ -105,7 +105,7 @@ def main():
 
     # Print out a the modifications.
     diff = difflib.Differ().compare(oldcfg.splitlines(1), cfg.splitlines(1))
-    utils.Status('Modified extlinux.conf:\n%s' % '\n'.join(diff))
+    utils.LogStatus('Modified extlinux.conf:\n%s' % '\n'.join(diff))
 
     f.seek(0)
     f.write(cfg)
@@ -118,6 +118,6 @@ def main():
 if __name__ == '__main__':
   try:
     main()
-    utils.Success('EL Installer build successful!')
+    utils.LogSuccess('EL Installer build successful!')
   except:
-    utils.Fail('EL Installer build failed!')
+    utils.LogFail('EL Installer build failed!')

daisy_workflows/image_build/enterprise_linux/enterprise_linux.wf.json

@@ -38,11 +38,11 @@
   "Sources": {
     "build_files/build_installer.py": "./build_installer.py",
     "build_files/installer.iso": "${installer_iso}",
-    "build_files/utils.py": "../linux_common/utils.py",
+    "build_files/utils.py": "../../linux_common/utils.py",
     "build_files/kickstart": "./kickstart/",
     "build_files/ks_helpers.py": "./ks_helpers.py",
     "build_files/save_logs.py": "./save_logs.py",
-    "installerprep_startup_script": "../linux_common/bootstrap.py"
+    "installerprep_startup_script": "../../linux_common/bootstrap.py"
   },
   "Steps": {
     "setup-disks": {
@@ -72,8 +72,9 @@
           "Disks": [{"Source": "disk-installerprep"}, {"Source": "disk-installer"}],
           "MachineType": "n1-standard-4",
           "Metadata": {
-            "build_files_gcs_dir": "${SOURCESPATH}/build_files",
-            "build_script": "build_installer.py",
+            "files_gcs_dir": "${SOURCESPATH}/build_files",
+            "script": "build_installer.py",
+            "prefix": "Build",
             "el_release": "${el_release}",
             "el_savelogs": "${el_savelogs}",
             "google_cloud_repo": "${google_cloud_repo}",
@@ -129,8 +130,8 @@
           "Disks": [{"Source": "disk-installerprep"}, {"Source": "disk-installer"}],
           "MachineType": "n1-standard-1",
           "Metadata": {
-            "build_files_gcs_dir": "${SOURCESPATH}/build_files",
-            "build_script": "save_logs.py"
+            "files_gcs_dir": "${SOURCESPATH}/build_files",
+            "script": "save_logs.py"
           },
           "Scopes": ["https://www.googleapis.com/auth/devstorage.read_write"],
           "StartupScript": "installerprep_startup_script"

daisy_workflows/image_build/enterprise_linux/ks_helpers.py

@@ -184,57 +184,57 @@ def BuildKsConfig(release, google_cloud_repo, byol, sap_hana, sap_apps):
   ks_packages = FetchConfigPart('common-packages.cfg')
   # For BYOL RHEL, don't remove subscription-manager.
   if byol:
-    utils.Status('Building RHEL BYOL image.')
+    utils.LogStatus('Building RHEL BYOL image.')
     rhel_byol_post = FetchConfigPart('rhel-byol-post.cfg')
 
   if release == 'rhel6':
-    utils.Status('Building RHEL 6 image.')
+    utils.LogStatus('Building RHEL 6 image.')
     ks_options = FetchConfigPart('el6-options.cfg')
     custom_post = FetchConfigPart('el6-post.cfg')
     if byol:
       custom_post = '\n'.join([custom_post, rhel_byol_post])
     cleanup = FetchConfigPart('el6-cleanup.cfg')
     repo_version = 'el6'
   elif release == "centos6":
-    utils.Status('Building CentOS 6 image.')
+    utils.LogStatus('Building CentOS 6 image.')
     ks_options = FetchConfigPart('el6-options.cfg')
     custom_post = FetchConfigPart('co6-post.cfg')
     cleanup = FetchConfigPart('el6-cleanup.cfg')
     repo_version = 'el6'
   elif release == "rhel7":
-    utils.Status('Building RHEL 7 image.')
+    utils.LogStatus('Building RHEL 7 image.')
     ks_options = FetchConfigPart('el7-options.cfg')
     custom_post = FetchConfigPart('el7-post.cfg')
     if byol:
       custom_post = '\n'.join([custom_post, rhel_byol_post])
     elif sap_hana:
-      utils.Status('Building RHEL 7 for SAP Hana')
+      utils.LogStatus('Building RHEL 7 for SAP Hana')
       custom_post = FetchConfigPart('rhel7-sap-hana-post.cfg')
     elif sap_apps:
-      utils.Status('Building RHEL 7 for SAP Apps')
+      utils.LogStatus('Building RHEL 7 for SAP Apps')
       custom_post = FetchConfigPart('rhel7-sap-apps-post.cfg')
     cleanup = FetchConfigPart('el7-cleanup.cfg')
     repo_version = 'el7'
   elif release == "centos7":
-    utils.Status('Building CentOS 7 image.')
+    utils.LogStatus('Building CentOS 7 image.')
     ks_options = FetchConfigPart('el7-options.cfg')
     custom_post = FetchConfigPart('co7-post.cfg')
     cleanup = FetchConfigPart('el7-cleanup.cfg')
     repo_version = 'el7'
   elif release == "oraclelinux6":
-    utils.Status('Building Oracle Linux 6 image.')
+    utils.LogStatus('Building Oracle Linux 6 image.')
     ks_options = FetchConfigPart('el6-options.cfg')
     custom_post = FetchConfigPart('ol6-post.cfg')
     cleanup = FetchConfigPart('el6-cleanup.cfg')
     repo_version = 'el6'
   elif release == "oraclelinux7":
-    utils.Status('Building Oracle Linux 7 image.')
+    utils.LogStatus('Building Oracle Linux 7 image.')
     ks_options = FetchConfigPart('el7-options.cfg')
     custom_post = FetchConfigPart('ol7-post.cfg')
     cleanup = FetchConfigPart('el7-cleanup.cfg')
     repo_version = 'el7'
   else:
-    utils.Fail('Unknown Image Name: %s' % release)
+    utils.LogFail('Unknown Image Name: %s' % release)
 
   ks_post = BuildPost(custom_post, cleanup, repo_version, google_cloud_repo)
 

daisy_workflows/image_build/enterprise_linux/save_logs.py

@@ -27,8 +27,8 @@ def main():
   # Mount the installer disk.
   utils.Execute(['mount', '-t', 'ext4', '/dev/sdb1', '/mnt'])
 
-  utils.Status('Installer root: %s' % os.listdir('/mnt'))
-  utils.Status('Build logs: %s' % os.listdir('/mnt/build-logs'))
+  utils.LogStatus('Installer root: %s' % os.listdir('/mnt'))
+  utils.LogStatus('Build logs: %s' % os.listdir('/mnt/build-logs'))
 
   # For some reason we need to remove the gsutil credentials.
   utils.Execute(['rm', '-Rf', '/root/.gsutil'])
@@ -42,6 +42,6 @@ def main():
 if __name__ == '__main__':
   try:
     main()
-    utils.Success('Build logs successfully saved.')
+    utils.LogSuccess('Build logs successfully saved.')
   except:
-    utils.Fail('Failed to save build logs.')
+    utils.LogFail('Failed to save build logs.')