WIP: Add support for Go workspaces

This commit changes the resolve_gomod function so that it can handle a
json stream coming from `go list -m`, which happens when workspaces are
used in a Go project.

It also removes the previous restrictions on using workspaces.

New unit tests are still lacking.

Signed-off-by: Bruno Pimentel <bpimente@redhat.com>

cachito/workers/pkg_managers/gomod.py

@@ -375,7 +375,14 @@ def resolve_gomod(app_source_path, request, dep_replacements=None, git_dir_path=
             # Make Go ignore the vendor dir even if there is one
             go_list.extend(["-mod", "readonly"])
 
-        main_module_name = go([*go_list, "-m"], run_params).rstrip()
+        modules_json_stream = go([*go_list, "-m", "-json"], run_params).rstrip()
+
+        main_module, workspace_modules = _process_modules_json_stream(
+            app_source_path, modules_json_stream
+        )
+
+        main_module_name = main_module.path
+
         main_module_version = get_golang_version(
             main_module_name,
             git_dir_path,
@@ -387,6 +394,10 @@ def resolve_gomod(app_source_path, request, dep_replacements=None, git_dir_path=
                 else str(app_source_path).replace(f"{git_dir_path}/", "")
             ),
         )
+
+        for module in workspace_modules:
+            module.version = main_module_version
+
         main_module = {
             "type": "gomod",
             "name": main_module_name,
@@ -408,7 +419,7 @@ def go_list_deps(pattern: Literal["./...", "all"]) -> Iterator[GoPackage]:
             mod for pkg in go_list_deps("all") if (mod := pkg.module) and not mod.main
         )
         main_module_deps = _deduplicate_to_gomod_dicts(
-            chain(package_modules, downloaded_modules), deps_to_replace
+            chain(package_modules, downloaded_modules, workspace_modules), deps_to_replace
         )
 
         log.info("Retrieving the list of packages")
@@ -473,6 +484,34 @@ def go_list_deps(pattern: Literal["./...", "all"]) -> Iterator[GoPackage]:
         }
 
 
+def _process_modules_json_stream(
+    app_dir: Path, modules_json_stream: str
+) -> tuple[GoModule, list[GoModule]]:
+    """Process the json stream returned by "go list -m -json".
+
+    The stream will contain the module currently being processed, or a list of all workspaces in
+    case a go.work file is present in the repository.
+
+    :param app_dir: the path to the module directory
+    :param modules_json_stream: the json stream returned by "go list -m -json"
+    :return: A tuple containing the main module and a list of workspaces
+    """
+    module_list = []
+    main_module = None
+
+    for module in load_json_stream(modules_json_stream):
+        if module["Dir"] == str(app_dir):
+            main_module = GoModule.parse_obj(module)
+        else:
+            module_list.append(GoModule.parse_obj(module))
+
+    # should never happen, since the main module will always be a part of the json stream
+    if not main_module:
+        raise RuntimeError('Failed to find the main module info in the "go list -m" output.')
+
+    return main_module, module_list
+
+
 def _deduplicate_to_gomod_dicts(
     modules: Iterable[GoModule], user_specified_deps_to_replace: set[str]
 ) -> list[dict[str, Any]]:

cachito/workers/tasks/gomod.py

@@ -65,12 +65,6 @@ def _is_workspace(repo_root: Path, subpath: str):
     return False
 
 
-def _fail_if_bundle_dir_has_workspaces(bundle_dir: RequestBundleDir, subpaths: list[str]):
-    for subpath in subpaths:
-        if _is_workspace(bundle_dir.source_root_dir, subpath):
-            raise InvalidRepoStructure("Go workspaces are not supported by Cachito.")
-
-
 def _fail_if_parent_replacement_not_included(packages_json_data: PackagesData) -> None:
     """
     Fail if any dependency replacement refers to a parent dir that isn't included in this request.
@@ -139,8 +133,6 @@ def fetch_gomod_source(request_id, dep_replacements=None, package_configs=None):
         # Default to the root of the application source
         subpaths = [os.curdir]
 
-    _fail_if_bundle_dir_has_workspaces(bundle_dir, subpaths)
-
     invalid_gomod_files = _find_missing_gomod_files(bundle_dir, subpaths)
     if invalid_gomod_files:
         invalid_files_print = "; ".join(invalid_gomod_files)

tests/integration/test_gomod_packages.py

@@ -69,33 +69,6 @@ def test_gomod_vendor_check_fail(env_name, test_env):
     )
 
 
-def test_gomod_workspace_check(test_env):
-    """
-    Validate failing of gomod requests that contain workspaces.
-
-    Checks:
-    * The request fails with expected error message
-    """
-    env_data = utils.load_test_data("gomod_packages.yaml")["with_workspace"]
-    client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout"))
-    initial_response = client.create_new_request(
-        payload={
-            "repo": env_data["repo"],
-            "ref": env_data["ref"],
-            "pkg_managers": env_data["pkg_managers"],
-        },
-    )
-    completed_response = client.wait_for_complete_request(initial_response)
-    assert completed_response.status == 200
-    assert completed_response.data["state"] == "failed"
-    error_msg = "Go workspaces are not supported by Cachito."
-
-    assert error_msg in completed_response.data["state_reason"], (
-        f"#{completed_response.id}: Request failed correctly, but with unexpected message: "
-        f"{completed_response.data['state_reason']}. Expected message was: {error_msg}"
-    )
-
-
 def test_gomod_with_local_replacements_in_parent_dir_missing(test_env):
     """
     Test that a gomod local replacement from a parent directory includes the parent module.

tests/test_workers/test_pkg_managers/data/gomod-mocks/non-vendored/go_list_m.json

@@ -0,0 +1,7 @@
+{
+	"Path": "github.com/cachito-testing/gomod-pandemonium",
+	"Main": true,
+	"Dir": "{repo_dir}",
+	"GoMod": "{repo_dir}/go.mod",
+	"GoVersion": "1.19"
+}

tests/test_workers/test_pkg_managers/test_gomod.py

@@ -42,6 +42,18 @@ def go_mod_file(tmp_path: Path, request: pytest.FixtureRequest) -> None:
         f.write(request.param)
 
 
+def get_main_module_json(repo_dir: str) -> dict[str, str]:
+    return json.dumps(
+        {
+            "Path": "github.com/cachito-testing/gomod-pandemonium",
+            "Main": True,
+            "Dir": f"{repo_dir}",
+            "GoMod": f"{repo_dir}/go.mod",
+            "GoVersion": "1.19",
+        }
+    )
+
+
 RETRODEP_PRE_REPLACE = "github.com/release-engineering/retrodep/v2"
 RETRODEP_POST_REPLACE = "github.com/cachito-testing/retrodep/v2"
 
@@ -91,6 +103,8 @@ def test_resolve_gomod(
     force_gomod_tidy: bool,
     tmp_path: Path,
 ):
+    module_dir = str(tmp_path / "path/to/module")
+
     # Mock the tempfile.TemporaryDirectory context manager
     mock_temp_dir.return_value.__enter__.return_value = str(tmp_path)
 
@@ -105,8 +119,8 @@ def test_resolve_gomod(
     if force_gomod_tidy or dep_replacement:
         run_side_effects.append(mock.Mock(returncode=0, stdout=None))  # go mod tidy
     run_side_effects.append(
-        # go list -m
-        mock.Mock(returncode=0, stdout="github.com/cachito-testing/gomod-pandemonium")
+        # go list -m -json
+        mock.Mock(returncode=0, stdout=get_main_module_json(module_dir))
     )
     run_side_effects.append(
         # go list -deps -json all
@@ -221,6 +235,8 @@ def test_resolve_gomod_vendor_dependencies(
     force_gomod_tidy: bool,
     tmp_path: Path,
 ) -> None:
+    module_dir = tmp_path / "path/to/module"
+
     # Mock the tempfile.TemporaryDirectory context manager
     mock_temp_dir.return_value.__enter__.return_value = str(tmp_path)
 
@@ -230,8 +246,8 @@ def test_resolve_gomod_vendor_dependencies(
     if force_gomod_tidy:
         run_side_effects.append(mock.Mock(returncode=0, stdout=None))  # go mod tidy
     run_side_effects.append(
-        # go list -m
-        mock.Mock(returncode=0, stdout="github.com/cachito-testing/gomod-pandemonium")
+        # go list -m -json
+        mock.Mock(returncode=0, stdout=get_main_module_json(str(module_dir)))
     )
     run_side_effects.append(
         # go list -deps -json all
@@ -247,7 +263,6 @@ def test_resolve_gomod_vendor_dependencies(
     mock_go_release.return_value = "go0.1.0"
     mock_get_gomod_version.return_value = "0.1.1"
 
-    module_dir = tmp_path / "path/to/module"
     module_dir.joinpath("vendor").mkdir(parents=True)
     module_dir.joinpath("vendor/modules.txt").write_text(get_mocked_data("vendored/modules.txt"))
 
@@ -297,6 +312,8 @@ def test_resolve_gomod_strict_mode_raise_error(
     tmp_path: Path,
     strict_vendor: bool,
 ) -> None:
+    module_dir = str(tmp_path)
+
     # Mock the get_worker_config
     mock_config = mock.Mock()
     mock_config.cachito_gomod_strict_vendor = strict_vendor
@@ -311,12 +328,11 @@ def test_resolve_gomod_strict_mode_raise_error(
     # Mock the "subprocess.run" calls
     mock_run.side_effect = [
         mock.Mock(returncode=0, stdout=""),  # go mod download
-        mock.Mock(returncode=0, stdout="pizza"),  # go list -m
+        mock.Mock(returncode=0, stdout=get_main_module_json(module_dir)),  # go list -m -json
         mock.Mock(returncode=0, stdout=""),  # go list -deps -json all
         mock.Mock(returncode=0, stdout=""),  # go list -deps -json ./...
     ]
 
-    module_dir = str(tmp_path)
     tmp_path.joinpath("vendor").mkdir()
 
     request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"}
@@ -358,6 +374,7 @@ def test_resolve_gomod_no_deps(
         }
         """
     )
+    module_dir = str(tmp_path / "/path/to/module")
 
     # Mock the tempfile.TemporaryDirectory context manager
     mock_temp_dir.return_value.__enter__.return_value = str(tmp_path)
@@ -368,7 +385,7 @@ def test_resolve_gomod_no_deps(
     if force_gomod_tidy:
         run_side_effects.append(mock.Mock(returncode=0, stdout=None))  # go mod tidy
     run_side_effects.append(
-        mock.Mock(returncode=0, stdout="github.com/release-engineering/retrodep/v2")  # go list -m
+        mock.Mock(returncode=0, stdout=get_main_module_json(module_dir))  # go list -m -json
     )
     run_side_effects.append(
         # go list -deps -json all
@@ -384,8 +401,6 @@ def test_resolve_gomod_no_deps(
     mock_go_release.return_value = "go0.1.0"
     mock_get_gomod_version.return_value = "0.1.1"
 
-    module_dir = str(tmp_path / "/path/to/module")
-
     request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"}
     if force_gomod_tidy:
         request["flags"] = ["force-gomod-tidy"]
@@ -394,7 +409,7 @@ def test_resolve_gomod_no_deps(
 
     assert gomod_["module"] == {
         "type": "gomod",
-        "name": "github.com/release-engineering/retrodep/v2",
+        "name": "github.com/cachito-testing/gomod-pandemonium",
         "version": "v2.1.1",
     }
     assert not gomod_["module_deps"]
@@ -451,7 +466,7 @@ def test_go_list_cmd_failure(
 
     expect_error = "Go execution failed: "
     if go_mod_rc == 0:
-        expect_error += "`go list -e -mod readonly -m` failed with rc=1"
+        expect_error += "`go list -e -mod readonly -m -json` failed with rc=1"
     else:
         expect_error += "Cachito re-tried running `go mod download -json` command 1 times."
 

tests/test_workers/test_tasks/test_gomod.py

@@ -73,7 +73,6 @@
     ),
 )
 @pytest.mark.parametrize("has_pkg_lvl_deps", (True, False))
-@mock.patch("cachito.workers.tasks.gomod._fail_if_bundle_dir_has_workspaces")
 @mock.patch("cachito.workers.tasks.gomod.RequestBundleDir")
 @mock.patch("cachito.workers.tasks.gomod.update_request_env_vars")
 @mock.patch("cachito.workers.tasks.gomod.set_request_state")
@@ -85,7 +84,6 @@ def test_fetch_gomod_source(
     mock_set_request_state,
     mock_update_request_env_vars,
     mock_bundle_dir,
-    mock_fail_workspaces,
     dep_replacements,
     expect_state_update,
     pkg_config,
@@ -274,15 +272,13 @@ def directory_present(*args, **kwargs):
         ),
     ),
 )
-@mock.patch("cachito.workers.tasks.gomod._fail_if_bundle_dir_has_workspaces")
 @mock.patch("cachito.workers.tasks.gomod.get_worker_config")
 @mock.patch("cachito.workers.tasks.gomod.RequestBundleDir")
 @mock.patch("cachito.workers.tasks.gomod.resolve_gomod")
 def test_fetch_gomod_source_no_go_mod_file(
     mock_resolve_gomod,
     mock_bundle_dir,
     mock_gwc,
-    mock_fail_workspaces,
     ignore_missing_gomod_file,
     exception_expected,
     pkg_config,
@@ -357,22 +353,6 @@ def test_is_workspace(repo, subpath, expected_result, tmpdir):
     assert result == expected_result
 
 
-@pytest.mark.parametrize("add_go_work_file", [True, False])
-def test_fail_if_bundle_dir_has_workspaces(add_go_work_file, tmpdir):
-    tmpdir.mkdir("temp")
-    tmpdir.mkdir("temp/1")
-    tmpdir.mkdir("temp/1/app")
-
-    bundle_dir = RequestBundleDir(1, tmpdir)
-
-    if add_go_work_file:
-        Path(bundle_dir.source_root_dir / "go.work").touch()
-        with pytest.raises(InvalidRepoStructure):
-            gomod._fail_if_bundle_dir_has_workspaces(bundle_dir, ["."])
-    else:
-        gomod._fail_if_bundle_dir_has_workspaces(bundle_dir, ["."])
-
-
 @pytest.mark.parametrize(
     "packages",
     [