[containerapp] Add support for --artifact, bug fixes and tests (#6954)

src/containerapp/HISTORY.rst

@@ -10,6 +10,8 @@ upcoming
 * 'az containerapp patch apply': support image patching for java application
 * Upgrade api-version to 2023-08-01-preview
 * 'az container app create/update': support --logs-dynamic-json-columns/-j to configure whether to parse json string log into dynamic json columns
+* 'az container app create/update/up': Remove the region check for the Cloud Build feature
+* 'az container app create/update/up': Improve logs on the local buildpack source to cloud flow
 
 0.3.43
 ++++++

src/containerapp/azext_containerapp/_archive_utils.py

@@ -71,7 +71,7 @@ def _pack_source_code(source_location, tar_file_path, docker_file_path, docker_f
 
     original_docker_file_name = os.path.basename(docker_file_path.replace("\\", os.sep))
     ignore_list, ignore_list_size = _load_dockerignore_file(source_location, original_docker_file_name)
-    common_vcs_ignore_list = {'.git', '.gitignore', '.bzr', 'bzrignore', '.hg', '.hgignore', '.svn'}
+    common_vcs_ignore_list = {'.git', '.gitignore', '.bzr', 'bzrignore', '.hg', '.hgignore', '.svn', 'mvnw'}
 
     def _ignore_check(tarinfo, parent_ignored, parent_matching_rule_index):
         # ignore common vcs dir or file
@@ -186,6 +186,10 @@ def _load_dockerignore_file(source_location, original_docker_file_name):
 
 
 def _archive_file_recursively(tar, name, arcname, parent_ignored, parent_matching_rule_index, ignore_check):
+    if os.path.isfile(name) and (arcname == "" or arcname is None):
+        # If the file is in the root dir, use its name as the arcname
+        arcname = os.path.basename(name)
+
     # create a TarInfo object from the file
     tarinfo = tar.gettarinfo(name, arcname)
 

src/containerapp/azext_containerapp/_cloud_build_utils.py

@@ -2,7 +2,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
-# pylint: disable=line-too-long, too-many-locals, missing-timeout, too-many-statements, consider-using-with
+# pylint: disable=line-too-long, too-many-locals, missing-timeout, too-many-statements, consider-using-with, too-many-branches
 
 from threading import Thread
 import os
@@ -33,9 +33,6 @@ def run_cloud_build(cmd, source, location, resource_group_name, environment_name
     generated_build_name = f"build{run_full_id}"[:12]
     log_in_file(f"Starting the Cloud Build for build of id '{generated_build_name}'\n", logs_file, no_print=True)
 
-    if not os.path.exists(source):
-        raise ValidationError(f"Impossible to find the directory or file corresponding to {source}. Please make sure that this path exists.")
-
     try:
         done_spinner = False
         fail_spinner = False
@@ -61,7 +58,7 @@ def spin():
                     loop_counter = (loop_counter + 1) % 17
                     loading_bar_left_spaces_count = loop_counter - 9 if loop_counter > 9 else 0
                     loading_bar_right_spaces_count = 6 - loop_counter if loop_counter < 7 else 0
-                    spinner = f"[{' ' * loading_bar_left_spaces_count}{'=' * (7 - loading_bar_left_spaces_count - loading_bar_right_spaces_count)}{' ' * loading_bar_right_spaces_count}]"
+                    spinner = f"|{' ' * loading_bar_left_spaces_count}{'=' * (7 - loading_bar_left_spaces_count - loading_bar_right_spaces_count)}{' ' * loading_bar_right_spaces_count}|"
                     time_elapsed = time.time() - start_time
                     print(f"\r    {spinner} {task_title} ({time_elapsed:.1f}s)", end="", flush=True)
                     time.sleep(0.15)
@@ -177,12 +174,35 @@ def spin():
         done_spinner = False
         thread = display_spinner("Streaming Cloud Build logs")
         headers = {'Authorization': 'Bearer ' + token}
-        response_log_streaming = requests.get(
-            log_streaming_endpoint,
-            headers=headers,
-            stream=True)
-        if not response_log_streaming.ok:
-            raise ValidationError(f"Error when streaming the logs, request exited with {response_log_streaming.status_code}")
+        logs_stream_retries = 0
+        maximum_logs_stream_retries = 5
+        while logs_stream_retries < maximum_logs_stream_retries:
+            logs_stream_retries += 1
+            response_log_streaming = requests.get(
+                log_streaming_endpoint,
+                headers=headers,
+                stream=True)
+            if not response_log_streaming.ok:
+                raise ValidationError(f"Error when streaming the logs, request exited with {response_log_streaming.status_code}")
+            # Actually validate that we logs streams successfully
+            response_log_streaming_lines = response_log_streaming.iter_lines()
+            count_lines_check = 2
+            for line in response_log_streaming_lines:
+                log_line = remove_ansi_characters(line.decode("utf-8"))
+                log_in_file(log_line, logs_file, no_print=True)
+                if "Kubernetes error happened" in log_line:
+                    if logs_stream_retries >= maximum_logs_stream_retries:
+                        # We're getting an error when streaming logs and no retries remaining.
+                        raise CloudBuildError(log_line)
+                    # Wait for a bit, and then break to try again. Using "logs_stream_retries" as the number of seconds to wait is a primitive exponential retry.
+                    time.sleep(logs_stream_retries)
+                    break
+                count_lines_check -= 1
+                if count_lines_check <= 0:
+                    break
+            if count_lines_check <= 0:
+                # We checked the set number of lines and logs stream without error. Let's continue.
+                break
         done_spinner = True
         thread.join()
 
@@ -191,10 +211,10 @@ def spin():
         thread = display_spinner("Buildpack: Initializing")
         log_execution_phase_pattern = r"===== (.*) =====$"
         current_phase_logs = ""
-        for line in response_log_streaming.iter_lines():
+        for line in response_log_streaming_lines:
             log_line = remove_ansi_characters(line.decode("utf-8"))
             current_phase_logs += f"{log_line}\n{substatus_indentation}"
-            if "ERROR:" in log_line or "Kubernetes error happened" in log_line:
+            if "----- Cloud Build failed with exit code" in log_line or "Exiting with failure status due to previous errors" in log_line:
                 raise CloudBuildError(current_phase_logs)
 
             log_execution_phase_match = re.search(log_execution_phase_pattern, log_line)

src/containerapp/azext_containerapp/_constants.py

@@ -86,8 +86,8 @@
     timeout: 1800
 """
 
-ACA_BUILDER_BULLSEYE_IMAGE = "mcr.microsoft.com/oryx/builder:debian-bullseye-20231025.1"
-ACA_BUILDER_BOOKWORM_IMAGE = "mcr.microsoft.com/oryx/builder:debian-bookworm-20231025.1"
+ACA_BUILDER_BULLSEYE_IMAGE = "mcr.microsoft.com/oryx/builder:debian-bullseye-20231107.2"
+ACA_BUILDER_BOOKWORM_IMAGE = "mcr.microsoft.com/oryx/builder:debian-bookworm-20231107.2"
 
 DEFAULT_PORT = 8080  # used for no dockerfile scenario; not the hello world image
 

src/containerapp/azext_containerapp/_params.py

@@ -21,6 +21,7 @@ def load_arguments(self, _):
 
     with self.argument_context('containerapp create') as c:
         c.argument('source', help="Local directory path containing the application source and Dockerfile for building the container image. Preview: If no Dockerfile is present, a container image is generated using buildpacks. If Docker is not running or buildpacks cannot be used, Oryx will be used to generate the image. See the supported Oryx runtimes here: https://aka.ms/SourceToCloudSupportedVersions.", is_preview=True)
+        c.argument('artifact', help="Local path to the application artifact for building the container image. See the supported artifacts here: https://aka.ms/SourceToCloudSupportedArtifacts.", is_preview=True)
 
     # Springboard
     with self.argument_context('containerapp create', arg_group='Service Binding') as c:
@@ -37,9 +38,10 @@ def load_arguments(self, _):
         c.argument('service_principal_client_secret', help='The service principal client secret. Used by GitHub Actions to authenticate with Azure.', options_list=["--service-principal-client-secret", "--sp-sec"])
         c.argument('service_principal_tenant_id', help='The service principal tenant ID. Used by GitHub Actions to authenticate with Azure.', options_list=["--service-principal-tenant-id", "--sp-tid"])
 
-    # Source
+    # Source and Artifact
     with self.argument_context('containerapp update') as c:
         c.argument('source', help="Local directory path containing the application source and Dockerfile for building the container image. Preview: If no Dockerfile is present, a container image is generated using buildpacks. If Docker is not running or buildpacks cannot be used, Oryx will be used to generate the image. See the supported Oryx runtimes here: https://aka.ms/SourceToCloudSupportedVersions.", is_preview=True)
+        c.argument('artifact', help="Local path to the application artifact for building the container image. See the supported artifacts here: https://aka.ms/SourceToCloudSupportedArtifacts.", is_preview=True)
 
     # Springboard
     with self.argument_context('containerapp update', arg_group='Service Binding') as c:
@@ -76,6 +78,9 @@ def load_arguments(self, _):
         c.argument('statestore', help="The state store component and dev service to create.")
         c.argument('pubsub', help="The pubsub component and dev service to create.")
 
+    with self.argument_context('containerapp up') as c:
+        c.argument('artifact', help="Local path to the application artifact for building the container image. See the supported artifacts here: https://aka.ms/SourceToCloudSupportedArtifacts.", is_preview=True)
+
     with self.argument_context('containerapp up', arg_group='Github Repo') as c:
         c.argument('repo', help='Create an app via Github Actions. In the format: https://github.com/<owner>/<repository-name> or <owner>/<repository-name>')
         c.argument('token', help='A Personal Access Token with write access to the specified repository. For more information: https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line. If not provided or not found in the cache (and using --repo), a browser page will be opened to authenticate with Github.')

src/containerapp/azext_containerapp/_up_utils.py

@@ -533,19 +533,14 @@ def build_container_from_source_with_buildpack(self, image_name, source, cache_i
                 is_non_supported_platform = False
                 is_non_supported_os = False
                 with subprocess.Popen(command, stdout=subprocess.PIPE) as process:
-
-                    # Collect the standard output in a separate variable that will be printed when a builder
-                    # successfully builds the provided application source
-                    stdout_collection = []
-
                     # Stream output of 'pack build' to warning stream
                     while process.stdout.readable():
                         line = process.stdout.readline()
                         if not line:
                             break
 
                         stdout_line = str(line.strip(), 'utf-8')
-                        stdout_collection.append(stdout_line)
+                        logger.warning(stdout_line)
 
                         # Check if the application is targeting a platform that's found in the current builder,
                         # specifically, if none of the buildpacks in the current builder are able to detect a platform
@@ -572,9 +567,6 @@ def build_container_from_source_with_buildpack(self, image_name, source, cache_i
 
                     could_build_image = True
                     logger.debug(f"Successfully built image {image_name} using buildpacks.")
-
-                    # Flush the stdout we've collected to the warning stream
-                    logger.warning("\n".join(stdout_collection))
                     break
             except Exception as ex:
                 logger.warning(f"Unable to run 'pack build' command to produce runnable application image: {ex}")
@@ -661,22 +653,20 @@ def run_source_to_cloud_flow(self, source, dockerfile, can_create_acr_if_needed,
             )
             return False
 
-        # Only enable Cloud Build on Stage and Canary while the changes are deployed to all regions.
         location = "eastus"
         if self.env.location:
             location = self.env.location
-        is_cloud_build_enabled = any(location.lower() == region for region in ["northcentralusstage", "centraluseuap", "eastus2euap"])
-        if self.should_create_acr and is_cloud_build_enabled:
+        if self.should_create_acr:
             # No container registry provided. Let's use the default container registry through Cloud Build.
             self.image = self.build_container_from_source_with_cloud_build_service(source, location)
             return True
 
         if can_create_acr_if_needed:
             self.create_acr_if_needed()
         elif not registry_server:
-            raise RequiredArgumentMissingError("Usage error: --registry-server is required while using --source in this context")
+            raise RequiredArgumentMissingError("Usage error: --registry-server is required while using --source or --artifact in this context")
         elif ACR_IMAGE_SUFFIX not in registry_server:
-            raise InvalidArgumentValueError("Usage error: --registry-server: expected an ACR registry (*.azurecr.io) for --source in this context")
+            raise InvalidArgumentValueError("Usage error: --registry-server: expected an ACR registry (*.azurecr.io) for --source or --artifact in this context")
 
         # At this point in the logic, we know that the customer doesn't have a Dockerfile but has a container registry.
         # Cloud Build is not an option anymore as we don't support BYO container registry yet.
@@ -806,22 +796,23 @@ def _get_ingress_and_target_port(ingress, target_port, dockerfile_content: "list
     return ingress, target_port
 
 
-def _validate_up_args(cmd, source, image, repo, registry_server):
+def _validate_up_args(cmd, source, artifact, image, repo, registry_server):
     disallowed_params = ["--only-show-errors", "--output", "-o"]
     command_args = cmd.cli_ctx.data.get("safe_params", [])
     for a in disallowed_params:
         if a in command_args:
             raise ValidationError(f"Argument {a} is not allowed for 'az containerapp up'")
 
-    if not source and not image and not repo:
+    if not source and not artifact and not image and not repo:
         raise RequiredArgumentMissingError(
-            "You must specify either --source, --repo, or --image"
+            "You must specify either --source, --artifact, --repo, or --image"
         )
     if source and repo:
         raise MutuallyExclusiveArgumentError(
-            "Cannot use --source and --repo togther. "
+            "Cannot use --source and --repo together. "
             "Can either deploy from a local directory or a Github repo"
         )
+    _validate_source_artifact_args(source, artifact)
     if repo and registry_server and "azurecr.io" in registry_server:
         parsed = urlparse(registry_server)
         registry_name = (parsed.netloc if parsed.scheme else parsed.path).split(".")[0]
@@ -830,6 +821,23 @@ def _validate_up_args(cmd, source, image, repo, registry_server):
                                   "characters when using --repo")
 
 
+def _validate_source_artifact_args(source, artifact):
+    if source and artifact:
+        raise MutuallyExclusiveArgumentError(
+            "Cannot use --source and --artifact together."
+        )
+    if source:
+        if not os.path.exists(source):
+            raise ValidationError(f"Impossible to find the source directory corresponding to {source}. Please make sure that this path exists.")
+        if not os.path.isdir(source):
+            raise ValidationError(f"The path corresponding to {source} is not a directory. Please make sure that the path given to --source is a directory.")
+    if artifact:
+        if not os.path.exists(artifact):
+            raise ValidationError(f"Impossible to find the artifact file corresponding to {artifact}. Please make sure that this path exists.")
+        if not os.path.isfile(artifact):
+            raise ValidationError(f"The path corresponding to {artifact} is not a file. Please make sure that the path given to --artifact is a file.")
+
+
 def _reformat_image(source, repo, image):
     if source and (image or repo):
         image = image.split("/")[-1]  # if link is given

src/containerapp/azext_containerapp/_validators.py

@@ -21,7 +21,7 @@
 
 
 # called directly from custom method bc otherwise it disrupts the --environment auto RID functionality
-def validate_create(registry_identity, registry_pass, registry_user, registry_server, no_wait, source=None, repo=None, yaml=None, environment_type=None):
+def validate_create(registry_identity, registry_pass, registry_user, registry_server, no_wait, source=None, artifact=None, repo=None, yaml=None, environment_type=None):
     if source and repo:
         raise MutuallyExclusiveArgumentError("Usage error: --source and --repo cannot be used together. Can either deploy from a local directory or a GitHub repository")
     if (source or repo) and yaml: