[BEAM-13559][Playground] Remove tag in examples CD

playground/infrastructure/helper.py

@@ -63,6 +63,15 @@ class Example:
   output: str = ""
 
 
+@dataclass
+class ExampleTag:
+  """
+  Class which contains all information about beam playground tag
+  """
+  tag_as_dict: Dict[str, str]
+  tag_as_string: str
+
+
 def find_examples(work_dir: str, supported_categories: List[str],
                   sdk: Sdk) -> List[Example]:
   """
@@ -122,7 +131,7 @@ async def get_statuses(examples: List[Example]):
   await asyncio.gather(*tasks)
 
 
-def get_tag(filepath) -> Optional[Dict[str, str]]:
+def get_tag(filepath) -> Optional[ExampleTag]:
   """
   Parse file by filepath and find beam tag
 
@@ -135,27 +144,30 @@ def get_tag(filepath) -> Optional[Dict[str, str]]:
   """
   add_to_yaml = False
   yaml_string = ""
+  tag_string = ""
 
   with open(filepath, encoding="utf-8") as parsed_file:
     lines = parsed_file.readlines()
 
   for line in lines:
-    line = line.replace("//", "").replace("#", "")
+    formatted_line = line.replace("//", "").replace("#", "")
     if add_to_yaml is False:
-      if line.lstrip() == Config.BEAM_PLAYGROUND_TITLE:
+      if formatted_line.lstrip() == Config.BEAM_PLAYGROUND_TITLE:
         add_to_yaml = True
-        yaml_string += line.lstrip()
+        yaml_string += formatted_line.lstrip()
+        tag_string += line
     else:
-      yaml_with_new_string = yaml_string + line
+      yaml_with_new_string = yaml_string + formatted_line
       try:
         yaml.load(yaml_with_new_string, Loader=yaml.SafeLoader)
-        yaml_string += line
+        yaml_string += formatted_line
+        tag_string += line
       except YAMLError:
         break
 
   if add_to_yaml:
     tag_object = yaml.load(yaml_string, Loader=yaml.SafeLoader)
-    return tag_object[Config.BEAM_PLAYGROUND]
+    return ExampleTag(tag_object[Config.BEAM_PLAYGROUND], tag_string)
 
   return None
 
@@ -185,7 +197,7 @@ def _check_file(examples, filename, filepath, supported_categories, sdk: Sdk):
   if extension == Config.SDK_TO_EXTENSION[sdk]:
     tag = get_tag(filepath)
     if tag is not None:
-      if _validate(tag, supported_categories) is False:
+      if _validate(tag.tag_as_dict, supported_categories) is False:
         logging.error(
             "%s contains beam playground tag with incorrect format", filepath)
         has_error = True
@@ -210,8 +222,7 @@ def get_supported_categories(categories_path: str) -> List[str]:
 
 
 def _get_example(
-    filepath: str, filename: str, tag: Dict[str, Union[str,
-                                                       List[str]]]) -> Example:
+    filepath: str, filename: str, tag: ExampleTag) -> Example:
   """
   Return an Example by filepath and filename.
 
@@ -228,14 +239,15 @@ def _get_example(
   object_type = _get_object_type(filename, filepath)
   with open(filepath, encoding="utf-8") as parsed_file:
     content = parsed_file.read()
+  content = content.replace(tag.tag_as_string, "")
 
   return Example(
       name=name,
       sdk=sdk,
       filepath=filepath,
       code=content,
       status=STATUS_UNSPECIFIED,
-      tag=Tag(**tag),
+      tag=Tag(**tag.tag_as_dict),
       type=object_type)
 
 
@@ -336,7 +348,7 @@ async def _update_example_status(example: Example, client: GRPCClient):
       client: client to send requests to the server.
   """
   pipeline_id = await client.run_code(
-      example.code, example.sdk, example.tag.pipeline_options)
+      example.code, example.sdk, example.tag[TagFields.pipeline_options])
   example.pipeline_id = pipeline_id
   status = await client.check_status(pipeline_id)
   while status in [STATUS_VALIDATING,

playground/infrastructure/test_cd_helper.py

@@ -81,6 +81,7 @@ def test__write_to_local_fs(delete_temp_folder):
   expected_result = {
       "SDK_JAVA/name/name.java": "temp/pipeline_id/SDK_JAVA/name/name.java",
       "SDK_JAVA/name/name.output": "temp/pipeline_id/SDK_JAVA/name/name.output",
+      "SDK_JAVA/name/name.log": "temp/pipeline_id/SDK_JAVA/name/name.log",
       "SDK_JAVA/name/meta.info": "temp/pipeline_id/SDK_JAVA/name/meta.info"
   }
   assert CDHelper()._write_to_local_fs(example) == expected_result

playground/infrastructure/test_helper.py

@@ -27,7 +27,7 @@
 from helper import find_examples, Example, _get_example, _get_name, get_tag, \
   _validate, Tag, get_statuses, \
   _update_example_status, get_supported_categories, _check_file, \
-  _get_object_type
+  _get_object_type, ExampleTag
 
 
 @mock.patch("helper._check_file")
@@ -95,11 +95,12 @@ async def test_get_statuses(mock_update_example_status, mock_grpc_client):
 @mock.patch(
     "builtins.open",
     mock_open(
-        read_data="...\n# Beam-playground:\n#     name: Name\n\nimport ..."))
+        read_data="...\n# beam-playground:\n#     name: Name\n\nimport ..."))
 def test_get_tag_when_tag_is_exists():
   result = get_tag("")
 
-  assert result.get("name") == "Name"
+  assert result.tag_as_dict.get("name") == "Name"
+  assert result.tag_as_string == "# beam-playground:\n#     name: Name\n\n"
 
 
 @mock.patch("builtins.open", mock_open(read_data="...\n..."))
@@ -114,7 +115,7 @@ def test_get_tag_when_tag_does_not_exist():
 @mock.patch("helper.get_tag")
 def test__check_file_with_correct_tag(
     mock_get_tag, mock_validate, mock_get_example):
-  tag = {"name": "Name"}
+  tag = ExampleTag({"name": "Name"}, "")
   example = Example(
       name="filename",
       sdk=SDK_JAVA,
@@ -135,15 +136,15 @@ def test__check_file_with_correct_tag(
   assert len(examples) == 1
   assert examples[0] == example
   mock_get_tag.assert_called_once_with("/root/filename.java")
-  mock_validate.assert_called_once_with(tag, [])
+  mock_validate.assert_called_once_with(tag.tag_as_dict, [])
   mock_get_example.assert_called_once_with(
       "/root/filename.java", "filename.java", tag)
 
 
 @mock.patch("helper._validate")
 @mock.patch("helper.get_tag")
 def test__check_file_with_incorrect_tag(mock_get_tag, mock_validate):
-  tag = {"name": "Name"}
+  tag = ExampleTag({"name": "Name"}, "")
   examples = []
   sdk = SDK_JAVA
   mock_get_tag.return_value = tag
@@ -155,7 +156,7 @@ def test__check_file_with_incorrect_tag(mock_get_tag, mock_validate):
   assert result is True
   assert len(examples) == 0
   mock_get_tag.assert_called_once_with("/root/filename.java")
-  mock_validate.assert_called_once_with(tag, [])
+  mock_validate.assert_called_once_with(tag.tag_as_dict, [])
 
 
 @mock.patch("builtins.open", mock_open(read_data="categories:\n    - category"))
@@ -170,17 +171,22 @@ def test_get_supported_categories():
 @mock.patch("helper._get_name")
 def test__get_example(mock_get_name):
   mock_get_name.return_value = "filepath"
-
-  result = _get_example(
-      "/root/filepath.java",
-      "filepath.java",
+  tag = ExampleTag(
       {
           "name": "Name",
           "description": "Description",
           "multifile": "False",
           "categories": [""],
           "pipeline_options": "--option option"
-      })
+      },
+      ""
+  )
+
+  result = _get_example(
+      "/root/filepath.java",
+      "filepath.java",
+      tag
+  )
 
   assert result == Example(
       name="filepath",
@@ -267,7 +273,7 @@ async def test__update_example_status(
       code="code",
       output="output",
       status=STATUS_UNSPECIFIED,
-      tag={"name": "Name"})
+      tag={"pipeline_options": "--key value"})
 
   mock_grpc_client_run_code.return_value = "pipeline_id"
   mock_grpc_client_check_status.side_effect = [
@@ -278,7 +284,7 @@ async def test__update_example_status(
 
   assert example.pipeline_id == "pipeline_id"
   assert example.status == STATUS_FINISHED
-  mock_grpc_client_run_code.assert_called_once_with(example.code, example.sdk)
+  mock_grpc_client_run_code.assert_called_once_with(example.code, example.sdk, "--key value")
   mock_grpc_client_check_status.assert_has_calls([mock.call("pipeline_id")])