From 08731960a83725804c6ce71802292bc154d0f726 Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Thu, 6 Jun 2019 11:18:23 +0200 Subject: [PATCH 01/16] Add readPlistFromString() function --- munkipkg | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/munkipkg b/munkipkg index 7b22009..945fc61 100755 --- a/munkipkg +++ b/munkipkg @@ -76,6 +76,15 @@ class PkgImportError(MunkiPkgError): pass +def readPlistFromString(data): + '''Wrapper for the differences between Python 2 and Python 3's plistlib''' + try: + return plistlib.load(data) + except AttributeError: + # plistlib module doesn't have a load function (as in Python 2) + return plistlib.readPlistFromString(data) + + def readPlist(filepath): '''Wrapper for the differences between Python 2 and Python 3's plistlib''' try: From a3f1e3b22a18baf6eaee63edf2138eab67eb9a56 Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Thu, 6 Jun 2019 11:19:22 +0200 Subject: [PATCH 02/16] Add run_subprocess() function --- munkipkg | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/munkipkg b/munkipkg index 945fc61..164140f 100755 --- a/munkipkg +++ b/munkipkg @@ -121,6 +121,22 @@ def display(message, quiet=False): print(("%s: %s" % (toolname, message))) +def run_subprocess(cmd): + '''Runs cmd with Popen''' + proc = subprocess.Popen( + cmd, + shell=False, + bufsize=1, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + proc_stdout, proc_stderr = proc.communicate() + retcode = proc.returncode + return (retcode, proc_stdout, proc_stderr) + + def validate_build_info_keys(build_info, file_path): '''Validates the data read from build_info.(plist|json|yaml|yml)''' valid_values = { From 03c5fe674e13f62a8ab7174179e82a25e16ac61a Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Thu, 6 Jun 2019 11:19:06 +0200 Subject: [PATCH 03/16] Allow passing toolname to display() --- munkipkg | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/munkipkg b/munkipkg index 164140f..fbbf42d 100755 --- a/munkipkg +++ b/munkipkg @@ -114,10 +114,11 @@ def unlink_if_possible(pathname): file=sys.stderr) -def display(message, quiet=False): +def display(message, quiet=False, toolname=None): '''Print message to stdout unless quiet is True''' if not quiet: - toolname = os.path.basename(sys.argv[0]) + if not toolname: + toolname = os.path.basename(sys.argv[0]) print(("%s: %s" % (toolname, message))) From 937a1482ba7e767d44f7e0f4409842b4c25d2198 Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Thu, 6 Jun 2019 11:17:53 +0200 Subject: [PATCH 04/16] Add notarization support --- README.md | 62 ++++++++++++++++++++++++++++++ munkipkg | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 169 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 56b87d1..e240859 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,9 @@ The value of this key is referenced in the default package name using `${version **signing_info** Dictionary of signing options. See below. +**notarization_info** +Dictionary of notarization options. See below. + ### Build directory @@ -256,6 +259,65 @@ The only required key/value in the signing_info dictionary is 'identity'. See the **SIGNED PACKAGES** section of the man page for `pkgbuild` or the **SIGNED PRODUCT ARCHIVES** section of the man page for `productbuild` for more information on the signing options. +### Package notarization + +**Important notes**: + +- Please read the [Customizing the Notarization Workflow](https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution/customizing_the_notarization_workflow) web page before you start notarizing your packages. +- Xcode 10 (or newer) is **required**. If you have more than one version of Xcode installed on your Mac, be sure to use the xcode-select utility to choose the appropriate version: `sudo xcode-select -s /path/to/Xcode10.app`. +- Unproxied network access to the Apple infrastructure (Usually `17.0.0.0/8` network) is required. +- Notarization tool tries to notarize not only the package but also the package payload. All code in the payload (including but not limited to app bundles, frameworks, kernel extensions) needs to be properly signed with the hardened runtime restrictions in order to be notarized. Please read Apple Developer documentation for more information. + +You may notarize **SIGNED PACKAGES** as part of the build process by adding a `notarization_info` dictionary to the build\_info.plist: + +```plist + notarization_info + + username + john.appleseed@apple.com + password + @keychain:AC_PASSWORD + stapler_timeout + 120 + +``` + +or, in JSON format in a build-info.json file: + +```json + "notarization_info": { + "username": "john.appleseed@apple.com", + "password": "@keychain:AC_PASSWORD", + "stapler_timeout": 120 + } +``` + +Keys/values of the `notarization_info` dictionary: + +| Key | Type | Required | Description | +| ----------------- | ------- | -------- | ----------- | +| username | String | Yes | Login email address of your developer Apple ID | +| password | String | Yes | 2FA app specific password. For information about the password and saving it to the login keychain see the web page [Customizing the Notarization Workflow](https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution/customizing_the_notarization_workflow) | +| stapler_timeout | Integer | No | See paragraph bellow | + +**About accessing password in keychain** + +If you configure `munki-pkg` to use the password from the login keychain user is going to be prompted to allow access to the password. +You can authorize this once clicking *Allow* or permanently clicking *Always Allow*. + +**About stapling** + +`munki-pkg` basically runs two commands: + +```shell +xcrun altool --notarize-app --primary-bundle-id "com.github.munki.pkg.munki-kickstart" --username "john.appleseed@apple.com" --password "@keychain:AC_PASSWORD" --file munki_kickstart.pkg +xcrun stapler staple munki_kickstart.pkg +``` + +There is a time delay between successfull upload of a signed package to the notary service using `altool` and availability of the staple from the notary service `stapler` can use. +`munki-pkg` will try to run `stapler` multiple times after sleeping for 15 seconds after each unsuccessul try. With `stapler_timeout` parameter you can specify timeout in seconds (default: 120) after which `munki-pkg` gives up. + + ### Additional options `--create` diff --git a/munkipkg b/munkipkg index fbbf42d..265a7fe 100755 --- a/munkipkg +++ b/munkipkg @@ -33,6 +33,9 @@ import stat import subprocess import sys import tempfile +import time +from xml.dom import minidom +from xml.parsers.expat import ExpatError try: import yaml @@ -49,6 +52,7 @@ LSBOM = "/usr/bin/lsbom" PKGBUILD = "/usr/bin/pkgbuild" PKGUTIL = "/usr/sbin/pkgutil" PRODUCTBUILD = "/usr/bin/productbuild" +XCRUN = "/usr/bin/xcrun" GITIGNORE_DEFAULT = """# .DS_Store files! .DS_Store @@ -61,6 +65,10 @@ BUILD_INFO_FILE = "build-info" REQUIREMENTS_PLIST = "product-requirements.plist" BOM_TEXT_FILE = "Bom.txt" +STAPLER_TIMEOUT = 120 +STAPLER_SLEEP = 15 + + class MunkiPkgError(Exception): '''Base Exception for errors in this domain''' pass @@ -262,10 +270,19 @@ def get_build_info(project_dir, options): info = default_build_info(project_dir) info['project_dir'] = project_dir # override default values with values from BUILD_INFO_PLIST - supported_keys = ['name', 'identifier', 'version', 'ownership', - 'install_location', 'postinstall_action', - 'preserve_xattr', 'suppress_bundle_relocation', - 'distribution_style', 'signing_info'] + supported_keys = [ + 'name', + 'identifier', + 'version', + 'ownership', + 'install_location', + 'postinstall_action', + 'preserve_xattr', + 'suppress_bundle_relocation', + 'distribution_style', + 'signing_info', + 'notarization_info', + ] build_file = os.path.join(project_dir, BUILD_INFO_FILE) file_type = None if not options.yaml and not options.json: @@ -650,6 +667,84 @@ def build_distribution_pkg(build_info, options): raise BuildError(err) +def notarize(build_info, options): + '''Use xcrun altool to notarize our package''' + if not ( + 'username' in build_info['notarization_info'] + and 'password' in build_info['notarization_info'] + ): + raise MunkiPkgError("notarization_info lacks username or password.") + + display("Uploading package to Apple notary service", options.quiet) + cmd = [ + XCRUN, + 'altool', + '--notarize-app', + '--primary-bundle-id', + build_info['identifier'], + '--username', + build_info['notarization_info']['username'], + '--password', + build_info['notarization_info']['password'], + '--output-format', + 'xml', + '--file', + os.path.join(build_info['build_dir'], build_info['name']), + ] + retcode, proc_stdout, proc_stderr = run_subprocess(cmd) + output = readPlistFromString(proc_stdout) + + if retcode: + for product_error in output['product-errors']: + print("altool: FAILURE " + product_error['message'], file=sys.stderr) + raise MunkiPkgError("Notarization failed") + else: + display( + "RequestUUID " + output['notarization-upload']['RequestUUID'], + options.quiet, + "altool", + ) + display("SUCCESS " + output['success-message'], options.quiet, "altool") + + +def staple(build_info, options): + '''Use xcrun staple to add staple to our package''' + display("Stapling package", options.quiet) + cmd = [ + XCRUN, + 'stapler', + 'staple', + os.path.join(build_info['build_dir'], build_info['name']), + ] + + if 'stapler_timeout' in build_info['notarization_info']: + timeout = build_info['notarization_info']['stapler_timeout'] + else: + timeout = STAPLER_TIMEOUT + + counter = 0 + while counter < timeout: + retcode, proc_stdout, proc_stderr = run_subprocess(cmd) + + # Stapler exited with code 0 + if not retcode: + break + else: + display( + "Staple not yet available. Trying again in {} seconds".format( + STAPLER_SLEEP + ), + options.quiet, + ) + time.sleep(STAPLER_SLEEP) + counter += STAPLER_SLEEP + + if retcode: + raise MunkiPkgError("Stapling failed") + else: + display("The staple and validate action worked!", options.quiet) + + def build(project_dir, options): '''Build our package''' @@ -712,6 +807,14 @@ def build(project_dir, options): if build_info['distribution_style']: build_distribution_pkg(build_info, options) + # notarize the pkg + if 'notarization_info' in build_info: + try: + notarize(build_info, options) + staple(build_info, options) + except MunkiPkgError as err: + print("ERROR: %s" % err, file=sys.stderr) + # cleanup temp dir _ = subprocess.call(["/bin/rm", "-rf", build_info['tmpdir']]) return 0 From bbd05088ace7c964b74e2154dcd4239d96070847 Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Thu, 6 Jun 2019 11:34:46 +0200 Subject: [PATCH 05/16] Add --skip-notarization and --skip-stapling options --- README.md | 6 ++++++ munkipkg | 11 +++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e240859..34a492b 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,12 @@ This option will import an existing package and convert it into a package projec `--export-bom-info` This option causes munkipkg to export bom info from the built package to a file named "Bom.txt" in the root of the package project directory. Since git does not normally track ownership, group, or mode of tracked files, and since the "ownership" option to `pkgbuild` can also result in different owner and group of files included in the package payload, exporting this info into a text file allows you to track this metadata in git (or other version control) as well. +`--skip-notarization` +Use this option to skip the whole notarization process when notarization is specified in the build-info. + +`--skip-stapling` +Use this option to skip only the stapling part of the notarization process when notarization is specified in the build-info. + `--sync` This option causes munkipkg to read the Bom.txt file, and use its information to create any missing empty directories and to set the permissions on files and directories. See [**Important git notes**](#important-git-notes) below. diff --git a/munkipkg b/munkipkg index 265a7fe..82a1e89 100755 --- a/munkipkg +++ b/munkipkg @@ -808,10 +808,11 @@ def build(project_dir, options): build_distribution_pkg(build_info, options) # notarize the pkg - if 'notarization_info' in build_info: + if 'notarization_info' in build_info and not options.skip_notarization: try: notarize(build_info, options) - staple(build_info, options) + if not options.skip_stapling: + staple(build_info, options) except MunkiPkgError as err: print("ERROR: %s" % err, file=sys.stderr) @@ -1139,6 +1140,12 @@ def main(): parser.add_option('-f', '--force', action='store_true', help='Forces creation of project directory if it already ' 'exists. ') + parser.add_option('--skip-notarization', action='store_true', + help='Skips whole notarization process when ' + 'notarization is specified in build-info') + parser.add_option('--skip-stapling', action='store_true', + help='Skips only stapling part of notarization process ' + 'when notarization is specified in build-info') options, arguments = parser.parse_args() if not arguments: From ab1f06f312a2b8b67f59c2ef64ed272eb96e096c Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Thu, 6 Jun 2019 14:23:14 +0200 Subject: [PATCH 06/16] Bump version to 0.8 --- munkipkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/munkipkg b/munkipkg index 82a1e89..c466608 100755 --- a/munkipkg +++ b/munkipkg @@ -46,7 +46,7 @@ except ImportError: from xml.dom import minidom from xml.parsers.expat import ExpatError -VERSION = "0.7" +VERSION = "0.8" DITTO = "/usr/bin/ditto" LSBOM = "/usr/bin/lsbom" PKGBUILD = "/usr/bin/pkgbuild" From 5ef6245c79d9d1b96045763c90e2ade9ba870217 Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Mon, 10 Jun 2019 09:37:17 +0200 Subject: [PATCH 07/16] Add support for asc_provider parameter --- README.md | 4 ++++ munkipkg | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 34a492b..ab167f3 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,8 @@ You may notarize **SIGNED PACKAGES** as part of the build process by adding a `n john.appleseed@apple.com password @keychain:AC_PASSWORD + asc_provider + JohnAppleseed1XXXXXX8 stapler_timeout 120 @@ -288,6 +290,7 @@ or, in JSON format in a build-info.json file: "notarization_info": { "username": "john.appleseed@apple.com", "password": "@keychain:AC_PASSWORD", + "asc_provider": "JohnAppleseed1XXXXXX8", "stapler_timeout": 120 } ``` @@ -298,6 +301,7 @@ Keys/values of the `notarization_info` dictionary: | ----------------- | ------- | -------- | ----------- | | username | String | Yes | Login email address of your developer Apple ID | | password | String | Yes | 2FA app specific password. For information about the password and saving it to the login keychain see the web page [Customizing the Notarization Workflow](https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution/customizing_the_notarization_workflow) | +| asc_provider | String | No | Only needed when a user account is associated with multiple providers | | stapler_timeout | Integer | No | See paragraph bellow | **About accessing password in keychain** diff --git a/munkipkg b/munkipkg index c466608..6d50c0d 100755 --- a/munkipkg +++ b/munkipkg @@ -691,6 +691,10 @@ def notarize(build_info, options): '--file', os.path.join(build_info['build_dir'], build_info['name']), ] + if 'asc_provider' in build_info['notarization_info']: + cmd.extend( + ['--asc-provider', build_info['notarization_info']['asc_provider']] + ) retcode, proc_stdout, proc_stderr = run_subprocess(cmd) output = readPlistFromString(proc_stdout) From dc15590622677f3169074405558109e3f84fd7bb Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Mon, 10 Jun 2019 10:00:27 +0200 Subject: [PATCH 08/16] Enhance error handling of xcrun altool --- munkipkg | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/munkipkg b/munkipkg index 6d50c0d..0910f73 100755 --- a/munkipkg +++ b/munkipkg @@ -696,7 +696,11 @@ def notarize(build_info, options): ['--asc-provider', build_info['notarization_info']['asc_provider']] ) retcode, proc_stdout, proc_stderr = run_subprocess(cmd) - output = readPlistFromString(proc_stdout) + try: + output = readPlistFromString(proc_stdout) + except ExpatError: + print(proc_stderr, file=sys.stderr) + raise MunkiPkgError("Notarization failed. Unable to run xcrun altool") if retcode: for product_error in output['product-errors']: From 127c0f0987625ebf687cb93f8cf4a2dc32c210e9 Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Thu, 13 Jun 2019 16:40:54 +0200 Subject: [PATCH 09/16] Improve dictionary key retrieval --- munkipkg | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/munkipkg b/munkipkg index 0910f73..5c0c134 100755 --- a/munkipkg +++ b/munkipkg @@ -703,16 +703,22 @@ def notarize(build_info, options): raise MunkiPkgError("Notarization failed. Unable to run xcrun altool") if retcode: - for product_error in output['product-errors']: - print("altool: FAILURE " + product_error['message'], file=sys.stderr) + for product_error in output.get('product-errors', []): + print( + "altool: FAILURE " + product_error.get('message', 'UNKNOWN ERROR'), + file=sys.stderr + ) raise MunkiPkgError("Notarization failed") - else: + + try: display( "RequestUUID " + output['notarization-upload']['RequestUUID'], options.quiet, "altool", ) display("SUCCESS " + output['success-message'], options.quiet, "altool") + except KeyError: + raise MunkiPkgError("Unexpected output from altool") def staple(build_info, options): @@ -725,10 +731,7 @@ def staple(build_info, options): os.path.join(build_info['build_dir'], build_info['name']), ] - if 'stapler_timeout' in build_info['notarization_info']: - timeout = build_info['notarization_info']['stapler_timeout'] - else: - timeout = STAPLER_TIMEOUT + timeout = build_info['notarization_info'].get('stapler_timeout', STAPLER_TIMEOUT) counter = 0 while counter < timeout: From 87aec47f48603b04602467c85798bebe13ee9f3f Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Thu, 13 Jun 2019 17:08:32 +0200 Subject: [PATCH 10/16] Add support for custom notarization primary-bundle-id --- README.md | 1 + munkipkg | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ab167f3..0b73bf6 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,7 @@ Keys/values of the `notarization_info` dictionary: | username | String | Yes | Login email address of your developer Apple ID | | password | String | Yes | 2FA app specific password. For information about the password and saving it to the login keychain see the web page [Customizing the Notarization Workflow](https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution/customizing_the_notarization_workflow) | | asc_provider | String | No | Only needed when a user account is associated with multiple providers | +| primary_bundle_id | String | No | Defaults to `identifier`. `primary_bundle_id` is useful when `identifier` contains characters such as '_' Apple notary service does not like | | stapler_timeout | Integer | No | See paragraph bellow | **About accessing password in keychain** diff --git a/munkipkg b/munkipkg index 5c0c134..9a5981f 100755 --- a/munkipkg +++ b/munkipkg @@ -681,7 +681,10 @@ def notarize(build_info, options): 'altool', '--notarize-app', '--primary-bundle-id', - build_info['identifier'], + build_info['notarization_info'].get( + 'primary_bundle_id', + build_info['identifier'] + ), '--username', build_info['notarization_info']['username'], '--password', From c747c3276b93a2d1318a4ef39c9124a5d00357db Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Fri, 14 Jun 2019 15:59:20 +0200 Subject: [PATCH 11/16] Implement altool --notarization-info check - Check result of the notary process with `altool --notarization-info` - Increase STAPLE_TIMEOUT to 300 seconds - Start using incremental time delay between notary process check - Update README.md to reflect the changes - Change wording of some docstrings --- README.md | 22 ++++----- munkipkg | 136 ++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 113 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 0b73bf6..a23c787 100644 --- a/README.md +++ b/README.md @@ -279,8 +279,8 @@ You may notarize **SIGNED PACKAGES** as part of the build process by adding a `n @keychain:AC_PASSWORD asc_provider JohnAppleseed1XXXXXX8 - stapler_timeout - 120 + staple_timeout + 600 ``` @@ -291,7 +291,7 @@ or, in JSON format in a build-info.json file: "username": "john.appleseed@apple.com", "password": "@keychain:AC_PASSWORD", "asc_provider": "JohnAppleseed1XXXXXX8", - "stapler_timeout": 120 + "stapler_timeout": 600 } ``` @@ -303,7 +303,7 @@ Keys/values of the `notarization_info` dictionary: | password | String | Yes | 2FA app specific password. For information about the password and saving it to the login keychain see the web page [Customizing the Notarization Workflow](https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution/customizing_the_notarization_workflow) | | asc_provider | String | No | Only needed when a user account is associated with multiple providers | | primary_bundle_id | String | No | Defaults to `identifier`. `primary_bundle_id` is useful when `identifier` contains characters such as '_' Apple notary service does not like | -| stapler_timeout | Integer | No | See paragraph bellow | +| staple_timeout | Integer | No | See paragraph bellow | **About accessing password in keychain** @@ -312,15 +312,15 @@ You can authorize this once clicking *Allow* or permanently clicking *Always All **About stapling** -`munki-pkg` basically runs two commands: +`munki-pkg` basically does following: -```shell -xcrun altool --notarize-app --primary-bundle-id "com.github.munki.pkg.munki-kickstart" --username "john.appleseed@apple.com" --password "@keychain:AC_PASSWORD" --file munki_kickstart.pkg -xcrun stapler staple munki_kickstart.pkg -``` +1. Uploads the package to Apple notary service using `xcrun altool --notarize-app --primary-bundle-id "com.github.munki.pkg.munki-kickstart" --username "john.appleseed@apple.com" --password "@keychain:AC_PASSWORD" --file munki_kickstart.pkg` +2. Checks periodically state of notarization process using `xcrun altool --notarization-info --username "john.appleseed@apple.com" --password "@keychain:AC_PASSWORD"` +3. If notarization was successful `munki-pkg` staples the package using `xcrun stapler staple munki_kickstart.pkg` -There is a time delay between successfull upload of a signed package to the notary service using `altool` and availability of the staple from the notary service `stapler` can use. -`munki-pkg` will try to run `stapler` multiple times after sleeping for 15 seconds after each unsuccessul try. With `stapler_timeout` parameter you can specify timeout in seconds (default: 120) after which `munki-pkg` gives up. +There is a time delay between successful upload of a signed package to the notary service and notarization result from the service. +`munki-pkg` checks multiple times if notarization process is done. There is sleep period between each try. Sleep period starts at 5 seconds and increases by increments of 5 (5s, 10s, 10s, etc.). +With `staple_timeout` parameter you can specify timeout in seconds (**default: 300 seconds**) after which `munki-pkg` gives up. ### Additional options diff --git a/munkipkg b/munkipkg index 9a5981f..4b5f70d 100755 --- a/munkipkg +++ b/munkipkg @@ -65,8 +65,8 @@ BUILD_INFO_FILE = "build-info" REQUIREMENTS_PLIST = "product-requirements.plist" BOM_TEXT_FILE = "Bom.txt" -STAPLER_TIMEOUT = 120 -STAPLER_SLEEP = 15 +STAPLE_TIMEOUT = 300 +STAPLE_SLEEP = 5 class MunkiPkgError(Exception): @@ -667,8 +667,8 @@ def build_distribution_pkg(build_info, options): raise BuildError(err) -def notarize(build_info, options): - '''Use xcrun altool to notarize our package''' +def upload_to_notary(build_info, options): + '''Use xcrun altool to upload the package to Apple notary service''' if not ( 'username' in build_info['notarization_info'] and 'password' in build_info['notarization_info'] @@ -682,8 +682,7 @@ def notarize(build_info, options): '--notarize-app', '--primary-bundle-id', build_info['notarization_info'].get( - 'primary_bundle_id', - build_info['identifier'] + 'primary_bundle_id', build_info['identifier'] ), '--username', build_info['notarization_info']['username'], @@ -699,11 +698,12 @@ def notarize(build_info, options): ['--asc-provider', build_info['notarization_info']['asc_provider']] ) retcode, proc_stdout, proc_stderr = run_subprocess(cmd) + try: output = readPlistFromString(proc_stdout) except ExpatError: print(proc_stderr, file=sys.stderr) - raise MunkiPkgError("Notarization failed. Unable to run xcrun altool") + raise MunkiPkgError("Notarization upload failed. Unable to run xcrun altool") if retcode: for product_error in output.get('product-errors', []): @@ -714,46 +714,112 @@ def notarize(build_info, options): raise MunkiPkgError("Notarization failed") try: - display( - "RequestUUID " + output['notarization-upload']['RequestUUID'], - options.quiet, - "altool", - ) + request_uuid = output['notarization-upload']['RequestUUID'] + display("RequestUUID " + request_uuid, options.quiet, "altool") display("SUCCESS " + output['success-message'], options.quiet, "altool") except KeyError: raise MunkiPkgError("Unexpected output from altool") + return request_uuid -def staple(build_info, options): - '''Use xcrun staple to add staple to our package''' - display("Stapling package", options.quiet) + +def get_notarization_state(request_uuid, build_info, options): + '''Checks for result of notarization process''' + state = {} cmd = [ XCRUN, - 'stapler', - 'staple', - os.path.join(build_info['build_dir'], build_info['name']), + 'altool', + '--notarization-info', + request_uuid, + '--username', + build_info['notarization_info']['username'], + '--password', + build_info['notarization_info']['password'], + '--output-format', + 'xml', ] + retcode, proc_stdout, proc_stderr = run_subprocess(cmd) + + try: + output = readPlistFromString(proc_stdout) + except ExpatError: + print(proc_stderr, file=sys.stderr) + raise MunkiPkgError("Notarization check failed. Unable to run xcrun altool") + + if retcode or 'notarization-info' not in output: + print("altool: " + output.get('success-message', 'Unexpected response')) + else: + state['log_url'] = output['notarization-info'].get('LogFileURL', '') + state['status'] = output['notarization-info'].get('Status', 'Unknown') + state['code'] = output['notarization-info'].get('Status Code', None) + state['message'] = output['notarization-info'].get('Status Message', '') + return state + + +def notarization_done(state, sleep_time, options): + '''Evaluates whether notarization is still in progress''' + if state['status'] == 'success': + display("Notarization successful. {}".format(state['message']), options.quiet) + return True + elif state['status'] == 'in progress': + display( + "Notarization still in progress. Trying again in {} seconds".format( + sleep_time + ), + options.quiet, + ) + return False + else: + display( + "Notarization unsuccessful:\n" + "\tStatus: {}\n" + "\tStatus Code: {}\n" + "\tStatus Message: {}\n" + "\tLogFileURL: {}".format( + state['status'], state['code'], state['message'], state['log_url'] + ), + options.quiet, + ) + raise MunkiPkgError("Notarization failed") - timeout = build_info['notarization_info'].get('stapler_timeout', STAPLER_TIMEOUT) +def wait_for_notarization(request_uuid, build_info, options): + '''Checks notarization state until it is done or we exceed the timeout value''' + display("Getting notarization state", options.quiet) + timeout = build_info['notarization_info'].get('staple_timeout', STAPLE_TIMEOUT) counter = 0 + sleep_time = STAPLE_SLEEP + while counter < timeout: - retcode, proc_stdout, proc_stderr = run_subprocess(cmd) + time.sleep(sleep_time) + state = get_notarization_state(request_uuid, build_info, options) + counter += sleep_time + sleep_time += STAPLE_SLEEP + + if notarization_done(state, sleep_time, options): + return True + + print( + "munkipkg: Timeout EXCEEDED when waiting for the notarization to complete. " + "You can manually staple the package later if notarization is successful.", + file=sys.stderr, + ) + return False - # Stapler exited with code 0 - if not retcode: - break - else: - display( - "Staple not yet available. Trying again in {} seconds".format( - STAPLER_SLEEP - ), - options.quiet, - ) - time.sleep(STAPLER_SLEEP) - counter += STAPLER_SLEEP + +def staple(build_info, options): + '''Use xcrun staple to add a staple to our package''' + display("Stapling package", options.quiet) + cmd = [ + XCRUN, + 'stapler', + 'staple', + os.path.join(build_info['build_dir'], build_info['name']), + ] + retcode, proc_stdout, proc_stderr = run_subprocess(cmd) if retcode: + print("stapler: FAILURE " + proc_stderr, file=sys.stderr) raise MunkiPkgError("Stapling failed") else: display("The staple and validate action worked!", options.quiet) @@ -824,8 +890,10 @@ def build(project_dir, options): # notarize the pkg if 'notarization_info' in build_info and not options.skip_notarization: try: - notarize(build_info, options) - if not options.skip_stapling: + request_uuid = upload_to_notary(build_info, options) + if not options.skip_stapling and wait_for_notarization( + request_uuid, build_info, options + ): staple(build_info, options) except MunkiPkgError as err: print("ERROR: %s" % err, file=sys.stderr) From 9f9ed7207594b1ae999f7b78ea880496b93d6d46 Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Mon, 17 Jun 2019 11:08:11 +0200 Subject: [PATCH 12/16] Enable automatic underscore replace for primary-bundle-id --- README.md | 2 +- munkipkg | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a23c787..c655ab2 100644 --- a/README.md +++ b/README.md @@ -302,7 +302,7 @@ Keys/values of the `notarization_info` dictionary: | username | String | Yes | Login email address of your developer Apple ID | | password | String | Yes | 2FA app specific password. For information about the password and saving it to the login keychain see the web page [Customizing the Notarization Workflow](https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution/customizing_the_notarization_workflow) | | asc_provider | String | No | Only needed when a user account is associated with multiple providers | -| primary_bundle_id | String | No | Defaults to `identifier`. `primary_bundle_id` is useful when `identifier` contains characters such as '_' Apple notary service does not like | +| primary_bundle_id | String | No | Defaults to `identifier`. Whether specified or not underscore characters are always automatically converted to hyphens since Apple notary service does not like underscores | | staple_timeout | Integer | No | See paragraph bellow | **About accessing password in keychain** diff --git a/munkipkg b/munkipkg index 4b5f70d..4528180 100755 --- a/munkipkg +++ b/munkipkg @@ -667,6 +667,19 @@ def build_distribution_pkg(build_info, options): raise BuildError(err) +def get_primary_bundle_id(build_info): + '''Gets primary bundle id for notarization''' + primary_bundle_id = build_info['notarization_info'].get( + 'primary_bundle_id', + build_info['identifier'], + ) + + # Apple notary service does not like underscores + primary_bundle_id = primary_bundle_id.replace('_', '-') + + return primary_bundle_id + + def upload_to_notary(build_info, options): '''Use xcrun altool to upload the package to Apple notary service''' if not ( @@ -681,9 +694,7 @@ def upload_to_notary(build_info, options): 'altool', '--notarize-app', '--primary-bundle-id', - build_info['notarization_info'].get( - 'primary_bundle_id', build_info['identifier'] - ), + get_primary_bundle_id(build_info), '--username', build_info['notarization_info']['username'], '--password', From 41f2158cc806fe774bdc43b9779372e283033c12 Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Tue, 18 Jun 2019 09:46:30 +0200 Subject: [PATCH 13/16] Add non zero exit for failed notarization --- munkipkg | 1 + 1 file changed, 1 insertion(+) diff --git a/munkipkg b/munkipkg index 4528180..dbf8bfb 100755 --- a/munkipkg +++ b/munkipkg @@ -908,6 +908,7 @@ def build(project_dir, options): staple(build_info, options) except MunkiPkgError as err: print("ERROR: %s" % err, file=sys.stderr) + return -1 # cleanup temp dir _ = subprocess.call(["/bin/rm", "-rf", build_info['tmpdir']]) From 826c9e848ae17988efc2ce44c97e38ff26bde649 Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Wed, 2 Oct 2019 13:59:44 +0200 Subject: [PATCH 14/16] Handle unexpected response in get_notarization_state() --- munkipkg | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/munkipkg b/munkipkg index dbf8bfb..8e6fa86 100755 --- a/munkipkg +++ b/munkipkg @@ -756,15 +756,17 @@ def get_notarization_state(request_uuid, build_info, options): except ExpatError: print(proc_stderr, file=sys.stderr) raise MunkiPkgError("Notarization check failed. Unable to run xcrun altool") - if retcode or 'notarization-info' not in output: print("altool: " + output.get('success-message', 'Unexpected response')) + print("altool: DEBUG output follows") + print(output) + state['status'] = 'Unknown' else: state['log_url'] = output['notarization-info'].get('LogFileURL', '') state['status'] = output['notarization-info'].get('Status', 'Unknown') state['code'] = output['notarization-info'].get('Status Code', None) state['message'] = output['notarization-info'].get('Status Message', '') - return state + return state def notarization_done(state, sleep_time, options): @@ -772,10 +774,11 @@ def notarization_done(state, sleep_time, options): if state['status'] == 'success': display("Notarization successful. {}".format(state['message']), options.quiet) return True - elif state['status'] == 'in progress': + elif state['status'] in ['in progress', 'Unknown']: display( - "Notarization still in progress. Trying again in {} seconds".format( - sleep_time + "Notarization state: {}. Trying again in {} seconds".format( + state['status'], + sleep_time, ), options.quiet, ) @@ -803,10 +806,11 @@ def wait_for_notarization(request_uuid, build_info, options): while counter < timeout: time.sleep(sleep_time) - state = get_notarization_state(request_uuid, build_info, options) counter += sleep_time sleep_time += STAPLE_SLEEP + state = get_notarization_state(request_uuid, build_info, options) + if notarization_done(state, sleep_time, options): return True From 83381db235373e3ba81c6e1b78aa03ea1681a89f Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Wed, 15 Jul 2020 16:38:29 +0200 Subject: [PATCH 15/16] Add App Store Connect API key support --- README.md | 28 ++++++++++++++++++++++------ munkipkg | 44 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index c655ab2..dc611b8 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ String. One of "recommended", "preserve", or "preserve-other". Defaults to "reco **postinstall_action** String. One of "none", "logout", or "restart". Defaults to "none". -**preserve_xattr** +**preserve_xattr** Boolean: true or false. Defaults to false. Setting this to true would preserve extended attributes, like codesigned flat files (e.g. script files), amongst other xattr's such as the apple quarantine warning (com.apple.quarantine). **product id** @@ -300,17 +300,33 @@ Keys/values of the `notarization_info` dictionary: | Key | Type | Required | Description | | ----------------- | ------- | -------- | ----------- | | username | String | Yes | Login email address of your developer Apple ID | -| password | String | Yes | 2FA app specific password. For information about the password and saving it to the login keychain see the web page [Customizing the Notarization Workflow](https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution/customizing_the_notarization_workflow) | +| password | String | (see authentication) | 2FA app specific password. | +| api_key | String | (see authentication) | App Store Connect API access key. | +| api_issuer | String | (see authentication) | App Store Connect API key issuer ID. | | asc_provider | String | No | Only needed when a user account is associated with multiple providers | | primary_bundle_id | String | No | Defaults to `identifier`. Whether specified or not underscore characters are always automatically converted to hyphens since Apple notary service does not like underscores | | staple_timeout | Integer | No | See paragraph bellow | -**About accessing password in keychain** +**Authentication** -If you configure `munki-pkg` to use the password from the login keychain user is going to be prompted to allow access to the password. -You can authorize this once clicking *Allow* or permanently clicking *Always Allow*. +To notarize the package you have to use Apple ID with access to App Store Connect. There are two possible authentication methods: App-specific password and API key. Either `password` or `api_key` + `api_issuer` keys(s) **must** be specified in the `notarization_info` dictionary. If you specify both `password` takes precedence. -**About stapling** +**Using the password** + +For information about the password and saving it to the login keychain see the web page [Customizing the Notarization Workflow](https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution/customizing_the_notarization_workflow). + +If you configure `munki-pkg` to use the password from the login keychain user is going to be prompted to allow access to the password. You can authorize this once clicking *Allow* or permanently clicking *Always Allow*. + +**Creating the API key** + +1. Log into [App Store Connect](https://appstoreconnect.apple.com) using developer Apple ID with access to API keys. +2. Go to Users and Access -> Keys. +3. Click + button to create a new key. +4. Name the key and select proper access - Developer. +5. Download the API key and save it to one of the following directories `./private_keys`, `~/private_keys`, `~/.private_keys`. Filename format is `AuthKey_.p8`. Use `` part when configuring `api_key` option. +6. Note the *Issuer ID* at the top of the web page. It must be provided using `api_issuer` option. + +**About stapling** `munki-pkg` basically does following: diff --git a/munkipkg b/munkipkg index 8e6fa86..733c029 100755 --- a/munkipkg +++ b/munkipkg @@ -680,13 +680,35 @@ def get_primary_bundle_id(build_info): return primary_bundle_id +def add_authentication_options(cmd, build_info): + '''Add --password or --apiKey + --apiIssuer options to the command''' + if 'password' in build_info['notarization_info']: + cmd.extend( + ['--password', build_info['notarization_info']['password']] + ) + elif ( + 'api_key' in build_info['notarization_info'] and + 'api_issuer' in build_info['notarization_info'] + ): + cmd.extend( + [ + '--apiKey', + build_info['notarization_info']['api_key'], + '--apiIssuer', + build_info['notarization_info']['api_issuer'], + ] + ) + else: + raise MunkiPkgError( + "password or api_key + api_issuer keys " + "must be specified in notarization_info." + ) + + def upload_to_notary(build_info, options): '''Use xcrun altool to upload the package to Apple notary service''' - if not ( - 'username' in build_info['notarization_info'] - and 'password' in build_info['notarization_info'] - ): - raise MunkiPkgError("notarization_info lacks username or password.") + if 'username' not in build_info['notarization_info']: + raise MunkiPkgError("notarization_info lacks username key.") display("Uploading package to Apple notary service", options.quiet) cmd = [ @@ -697,8 +719,6 @@ def upload_to_notary(build_info, options): get_primary_bundle_id(build_info), '--username', build_info['notarization_info']['username'], - '--password', - build_info['notarization_info']['password'], '--output-format', 'xml', '--file', @@ -708,7 +728,11 @@ def upload_to_notary(build_info, options): cmd.extend( ['--asc-provider', build_info['notarization_info']['asc_provider']] ) + add_authentication_options(cmd, build_info) + retcode, proc_stdout, proc_stderr = run_subprocess(cmd) + if proc_stdout.startswith('Generated JWT'): + proc_stdout = proc_stdout.split('\n',1)[1] try: output = readPlistFromString(proc_stdout) @@ -744,12 +768,14 @@ def get_notarization_state(request_uuid, build_info, options): request_uuid, '--username', build_info['notarization_info']['username'], - '--password', - build_info['notarization_info']['password'], '--output-format', 'xml', ] + add_authentication_options(cmd, build_info) + retcode, proc_stdout, proc_stderr = run_subprocess(cmd) + if proc_stdout.startswith('Generated JWT'): + proc_stdout = proc_stdout.split('\n',1)[1] try: output = readPlistFromString(proc_stdout) From c016cc13d92630c064b083e0e1a3ac9e4090a3d4 Mon Sep 17 00:00:00 2001 From: Michal Moravec Date: Tue, 18 Aug 2020 15:45:13 +0200 Subject: [PATCH 16/16] Make notarization code Python3 compatible --- munkipkg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/munkipkg b/munkipkg index 733c029..d6fb409 100755 --- a/munkipkg +++ b/munkipkg @@ -87,7 +87,7 @@ class PkgImportError(MunkiPkgError): def readPlistFromString(data): '''Wrapper for the differences between Python 2 and Python 3's plistlib''' try: - return plistlib.load(data) + return plistlib.loads(data) except AttributeError: # plistlib module doesn't have a load function (as in Python 2) return plistlib.readPlistFromString(data) @@ -135,6 +135,7 @@ def run_subprocess(cmd): proc = subprocess.Popen( cmd, shell=False, + universal_newlines=True, bufsize=1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -733,9 +734,8 @@ def upload_to_notary(build_info, options): retcode, proc_stdout, proc_stderr = run_subprocess(cmd) if proc_stdout.startswith('Generated JWT'): proc_stdout = proc_stdout.split('\n',1)[1] - try: - output = readPlistFromString(proc_stdout) + output = readPlistFromString(proc_stdout.encode("UTF-8")) except ExpatError: print(proc_stderr, file=sys.stderr) raise MunkiPkgError("Notarization upload failed. Unable to run xcrun altool") @@ -778,7 +778,7 @@ def get_notarization_state(request_uuid, build_info, options): proc_stdout = proc_stdout.split('\n',1)[1] try: - output = readPlistFromString(proc_stdout) + output = readPlistFromString(proc_stdout.encode("UTF-8")) except ExpatError: print(proc_stderr, file=sys.stderr) raise MunkiPkgError("Notarization check failed. Unable to run xcrun altool")