diff --git a/README.md b/README.md index 7ff2922..9a5ae35 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ The intermediate meta-manifest is generated to allow for the creation of custom Conda-vendor solves an environment with conda from an `environment.yaml` and determines all the packages that are required. The metadata for these required packages is stored in a file called `meta_manifest.yaml`. To create this file, run: - conda vendor create-meta-manifes + conda vendor meta-manifest t --environment-yaml environment.yaml The above command will output a `meta_manifest.yaml` file in the current directory. @@ -32,9 +32,9 @@ The above command will output a `meta_manifest.yaml` file in the current directo With a meta-manifest file created, conda-vendor can then create local channels. - conda vendor create-channels --meta-manifest-path ./meta_manifest.yaml + conda vendor channels --meta-manifest-path ./meta_manifest.yaml -This will create a directory called `local_channel` that will contain the same number of channels as were listed in the original `environment.yaml` file. These local channels will only contain the packages that are needed to satisfy the solved environment from the `create-meta-manifest` step. +This will create a directory called `local_channel` that will contain the same number of channels as were listed in the original `environment.yaml` file. These local channels will only contain the packages that are needed to satisfy the solved environment from the `meta-manifest` step. ### Using the Local channel @@ -53,7 +53,7 @@ This should show a list of all the packages in the environment the local paths t To generate a conda environment yaml that contains all the packages from the input `environment.yaml`, run the following: - conda vendor create-local-yaml --meta-manifest-path ./meta_manifest.yaml --channel-root + conda vendor local-yaml --meta-manifest-path ./meta_manifest.yaml --channel-root This will create a environment file inside the `local_channel` directory called `local_conda-vendor-env.yaml`. An environment can then be created with: @@ -67,6 +67,6 @@ The following functionality is only applicable if there is an organization that To generate an iron bank manifest from the meta-manifest, run: - conda vendor create-custom-manifest --meta-manifest-path ./meta-manifest.yaml --output-manifest-path ./custom_manifest.yaml + conda vendor custom-manifest --meta-manifest-path ./meta-manifest.yaml --output-manifest-path ./custom_manifest.yaml This will output a manifest file in the Iron Bank format. diff --git a/conda_vendor/__main__.py b/conda_vendor/__main__.py index aa39443..be9d406 100644 --- a/conda_vendor/__main__.py +++ b/conda_vendor/__main__.py @@ -2,10 +2,10 @@ import click from conda_vendor.cli import ( - create_ironbank_from_meta_manifest, - create_local_channels_from_meta_manifest, - create_meta_manifest_from_env_yml, - create_yaml_from_manifest, + ironbank_from_meta_manifest, + local_channels_from_meta_manifest, + meta_manifest_from_env_yml, + yaml_from_manifest, ) from conda_vendor.version import __version__ @@ -47,15 +47,24 @@ def cli(): default="meta_manifest.yaml", help="filename of output manifest", ) -def create_meta_manifest(verbose, environment_yaml, manifest_root, manifest_filename): +@click.option( + "--custom-platform", + "-c", + default=None, + type=click.Choice(["linux-64", "linux-32", "windows-64", "windows-32", "osx-64"]), + help="Specify an alternate platform to use for the conda solve.", +) +def meta_manifest( + verbose, environment_yaml, manifest_root, manifest_filename, custom_platform, +): set_logging_verbosity(verbose) click.echo(manifest_root) click.echo(manifest_filename) environment_yaml = Path(environment_yaml) manifest_root = Path(manifest_root) - create_meta_manifest_from_env_yml( - environment_yaml, manifest_root, manifest_filename + meta_manifest_from_env_yml( + environment_yaml, manifest_root, manifest_filename, custom_platform, ) @@ -73,11 +82,11 @@ def create_meta_manifest(verbose, environment_yaml, manifest_root, manifest_file default="./meta_manifest.yaml", help="path to meta manifest file", ) -def create_channels(verbose, channel_root, meta_manifest_path): +def channels(verbose, channel_root, meta_manifest_path): set_logging_verbosity(verbose) channel_root = Path(channel_root) meta_manifest_path = Path(meta_manifest_path) - create_local_channels_from_meta_manifest(channel_root, meta_manifest_path) + local_channels_from_meta_manifest(channel_root, meta_manifest_path) @click.command(help="custom manifest from meta-manifest file") @@ -87,21 +96,20 @@ def create_channels(verbose, channel_root, meta_manifest_path): ) @click.option( "--meta-manifest-path", + "-m", default="./meta_manifest.yaml", help="path to meta manifest file", ) @click.option( - "--output-manifest-path", default="./", help="output manifest path", + "--output-manifest-path", "-o", default="./", help="output manifest path", ) -def create_custom_manifest( - verbose, manifest_type, meta_manifest_path, output_manifest_path -): +def custom_manifest(verbose, manifest_type, meta_manifest_path, output_manifest_path): set_logging_verbosity(verbose) meta_manifest_path = Path(meta_manifest_path) output_manifest_path = Path(output_manifest_path) if manifest_type == "iron-bank": - create_ironbank_from_meta_manifest( + ironbank_from_meta_manifest( meta_manifest_path=meta_manifest_path, output_manifest_dir=output_manifest_path, ) @@ -122,25 +130,27 @@ def create_custom_manifest( ) @click.option( "--meta-manifest-path", + "-m", default="./meta_manifest.yaml", help="path to meta manifest file", ) @click.option( "--env-name", + "-n", default="conda-vendor-env", - help="name of conda environment defined in yaml", + help="name of conda environment to be defined in yaml", ) -def create_local_yaml(verbose, channel_root, meta_manifest_path, env_name): +def local_yaml(verbose, channel_root, meta_manifest_path, env_name): set_logging_verbosity(verbose) channel_root = Path(channel_root) meta_manifest_path = Path(meta_manifest_path) - create_yaml_from_manifest(channel_root, meta_manifest_path, env_name) + yaml_from_manifest(channel_root, meta_manifest_path, env_name) -cli.add_command(create_meta_manifest) -cli.add_command(create_channels) -cli.add_command(create_custom_manifest) -cli.add_command(create_local_yaml) +cli.add_command(meta_manifest) +cli.add_command(channels) +cli.add_command(custom_manifest) +cli.add_command(local_yaml) if __name__ == "__main__": diff --git a/conda_vendor/cli.py b/conda_vendor/cli.py index 496a394..1783379 100644 --- a/conda_vendor/cli.py +++ b/conda_vendor/cli.py @@ -6,14 +6,16 @@ from conda_vendor.manifest import MetaManifest -def create_meta_manifest_from_env_yml( - environment_yaml, manifest_root, manifest_filename +def meta_manifest_from_env_yml( + environment_yaml, manifest_root, manifest_filename, custom_platform=None, ): - conda_channel = MetaManifest(environment_yaml, manifest_root=manifest_root) + conda_channel = MetaManifest( + environment_yaml, manifest_root=manifest_root, custom_platform=custom_platform, + ) conda_channel.create_manifest(manifest_filename=manifest_filename) -def create_local_channels_from_meta_manifest(channel_root, meta_manifest_path): +def local_channels_from_meta_manifest(channel_root, meta_manifest_path): conda_channel = CondaChannel( channel_root=channel_root, meta_manifest_path=meta_manifest_path ) @@ -21,11 +23,11 @@ def create_local_channels_from_meta_manifest(channel_root, meta_manifest_path): conda_channel.download_binaries() -def create_ironbank_from_meta_manifest(meta_manifest_path, output_manifest_dir): +def ironbank_from_meta_manifest(meta_manifest_path, output_manifest_dir): custom_manifest = IBManifest(meta_manifest_path) custom_manifest.write_custom_manifest(output_manifest_dir) -def create_yaml_from_manifest(channel_root, meta_manifest_path, env_name): +def yaml_from_manifest(channel_root, meta_manifest_path, env_name): george_forge = YamlFromManifest(channel_root, meta_manifest_path=meta_manifest_path) george_forge.create_yaml(channel_root, env_name) diff --git a/conda_vendor/manifest.py b/conda_vendor/manifest.py index ff86ab0..e559732 100644 --- a/conda_vendor/manifest.py +++ b/conda_vendor/manifest.py @@ -28,7 +28,13 @@ def solve(*args, **kwargs): # see https://github.com/conda/conda/blob/248741a843e8ce9283fa94e6e4ec9c2fafeb76fd/conda/base/context.py#L51 -def get_conda_platform(platform=sys.platform): +def get_conda_platform( + platform=sys.platform, custom_platform=None, +): + + if custom_platform is not None: + return custom_platform + _platform_map = { "linux2": "linux", "linux": "linux", @@ -42,10 +48,12 @@ def get_conda_platform(platform=sys.platform): class MetaManifest: - def __init__(self, environment_yml, *, manifest_root=Path()): + def __init__( + self, environment_yml, *, manifest_root=Path(), custom_platform=None, + ): self.manifest_root = Path(manifest_root) logger.info(f"manifest_root : {self.manifest_root.absolute()}") - self.platform = get_conda_platform() + self.platform = get_conda_platform(custom_platform=custom_platform) self.valid_platforms = [self.platform, "noarch"] diff --git a/conda_vendor/version.py b/conda_vendor/version.py index 169f852..2765920 100644 --- a/conda_vendor/version.py +++ b/conda_vendor/version.py @@ -1,4 +1,4 @@ -__version__ = "0.1.8" +__version__ = "0.1.9" if __name__ == "__main__": print(__version__, end="") diff --git a/docs/source/user_guide.md b/docs/source/user_guide.md index b620886..1c6667a 100644 --- a/docs/source/user_guide.md +++ b/docs/source/user_guide.md @@ -17,7 +17,7 @@ The intermediate meta-manifest is generated to allow for the creation of custom Conda-vendor solves an environment with conda from an `environment.yaml` and determines all the packages that are required. The metadata for these required packages is stored in a file called `meta_manifest.yaml`. To create this file, run: - conda vendor create-meta-manifest --environment-yaml environment.yaml + conda vendor meta-manifest --environment-yaml environment.yaml The above command will output a `meta_manifest.yaml` file in the current directory. @@ -25,9 +25,9 @@ The above command will output a `meta_manifest.yaml` file in the current directo With a meta-manifest file created, conda-vendor can then create local channels. - conda vendor create-channels --meta-manifest-path ./meta_manifest.yaml + conda vendor channels --meta-manifest-path ./meta_manifest.yaml -This will create a directory called `local_channel` that will contain the same number of channels as were listed in the original `environment.yaml` file. These local channels will only contain the packages that are needed to satisfy the solved environment from the `create-meta-manifest` step. +This will create a directory called `local_channel` that will contain the same number of channels as were listed in the original `environment.yaml` file. These local channels will only contain the packages that are needed to satisfy the solved environment from the `meta-manifest` step. ### Using the Local channel @@ -46,7 +46,7 @@ This should show a list of all the packages in the environment the local paths t To generate a conda environment yaml that contains all the packages from the input `environment.yaml`, run the following: - conda vendor create-local-yaml --meta-manifest-path ./meta_manifest.yaml --channel-root + conda vendor local-yaml --meta-manifest-path ./meta_manifest.yaml --channel-root This will create a environment file inside the `local_channel` directory called `local_conda-vendor-env.yaml`. An environment can then be created with: @@ -60,6 +60,6 @@ The following functionality is only applicable if there is an organization that To generate an iron bank manifest from the meta-manifest, run: - conda vendor create-custom-manifest --meta-manifest-path ./meta-manifest.yaml --output-manifest-path ./custom_manifest.yaml + conda vendor custom-manifest --meta-manifest-path ./meta-manifest.yaml --output-manifest-path ./custom_manifest.yaml This will output a manifest file in the Iron Bank format. diff --git a/tests/test_conda_channel.py b/tests/test_conda_channel.py index 9334964..b282d46 100644 --- a/tests/test_conda_channel.py +++ b/tests/test_conda_channel.py @@ -70,11 +70,7 @@ def test_CondaChannel_fetch_and_filter_repodata(mock_download, conda_channel_fix fake_manifest_subset_metadata = { "repodata_url": "https://url1", - "entries": [ - {"fn": "file1"}, - {"fn": "file2"}, - {"fn": "file3"}, - ], + "entries": [{"fn": "file1"}, {"fn": "file2"}, {"fn": "file3"},], } fake_live_repo_data_json = { "info": {"subdir": "fake_subdir"}, @@ -97,10 +93,7 @@ def test_CondaChannel_fetch_and_filter_repodata(mock_download, conda_channel_fix expected = { "info": {"subdir": "fake_subdir"}, - "packages": { - "file1": {"id": 1}, - "file2": {"id": 3}, - }, + "packages": {"file1": {"id": 1}, "file2": {"id": 3},}, "packages.conda": {"file3": {"id": 5}}, } expected_mock_call = call("https://url1") diff --git a/tests/test_integration.py b/tests/test_integration.py index 01e214e..f1d0904 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -6,15 +6,15 @@ import yaml from conda_vendor.cli import ( - create_ironbank_from_meta_manifest, - create_local_channels_from_meta_manifest, - create_meta_manifest_from_env_yml, - create_yaml_from_manifest, + ironbank_from_meta_manifest, + local_channels_from_meta_manifest, + meta_manifest_from_env_yml, + yaml_from_manifest, ) from conda_vendor.custom_manifest import IBManifest -def test_create_meta_manifest_from_env_yml(tmp_path, minimal_conda_forge_environment): +def test_meta_manifest_from_env_yml(tmp_path, minimal_conda_forge_environment): test_manifest_filename = "test_metamanifest.yaml" expected_manifest_path = tmp_path / test_manifest_filename @@ -35,7 +35,7 @@ def test_get_packages_from_manifest(meta_manifest, expected_packages): i_bank_pkg_list.append(dep_entry) return i_bank_pkg_list - create_meta_manifest_from_env_yml( + meta_manifest_from_env_yml( minimal_conda_forge_environment, tmp_path, test_manifest_filename ) with open(expected_manifest_path) as f: @@ -49,26 +49,24 @@ def test_get_packages_from_manifest(meta_manifest, expected_packages): TestCase().assertCountEqual(result_packages, expected_packages) -def test_create_local_channels_from_meta_manifest( - tmp_path, minimal_conda_forge_environment -): +def test_local_channels_from_meta_manifest(tmp_path, minimal_conda_forge_environment): test_env_name = "the_test_env" test_manifest_filename = "test_metamanifest.yaml" channel_root = tmp_path test_manifest_path = tmp_path / test_manifest_filename path_to_env_yaml = tmp_path / f"local_{test_env_name}.yaml" - create_meta_manifest_from_env_yml( + meta_manifest_from_env_yml( minimal_conda_forge_environment, tmp_path, test_manifest_filename ) - create_yaml_from_manifest( + yaml_from_manifest( channel_root=tmp_path, meta_manifest_path=test_manifest_path, env_name=test_env_name, ) - create_local_channels_from_meta_manifest( + local_channels_from_meta_manifest( channel_root=tmp_path, meta_manifest_path=test_manifest_path ) @@ -114,21 +112,21 @@ def test_create_local_channels_from_meta_manifest( @patch("conda_vendor.custom_manifest.IBManifest.__init__") @patch("conda_vendor.custom_manifest.IBManifest.write_custom_manifest") -def test_create_ironbank_from_meta_manifest( +def test_ironbank_from_meta_manifest( mock_c, mock_i, tmp_path, get_path_location_for_manifest_fixture ): mock_i.return_value = None meta_manifest_path = get_path_location_for_manifest_fixture output_manifest_dir = tmp_path - create_ironbank_from_meta_manifest(meta_manifest_path, output_manifest_dir) + ironbank_from_meta_manifest(meta_manifest_path, output_manifest_dir) mock_c.assert_called_once_with(output_manifest_dir) mock_i.assert_called_once_with(meta_manifest_path) @patch("conda_vendor.env_yaml_from_manifest.YamlFromManifest.__init__") @patch("conda_vendor.env_yaml_from_manifest.YamlFromManifest.create_yaml") -def test_create_yaml_from_manifest( +def test_yaml_from_manifest( mock_c, mock_i, tmp_path, get_path_location_for_manifest_fixture ): mock_i.return_value = None @@ -137,7 +135,7 @@ def test_create_yaml_from_manifest( channel_root = tmp_path env_name = "forgin-georgin" - create_yaml_from_manifest(channel_root, meta_manifest_path, env_name) + yaml_from_manifest(channel_root, meta_manifest_path, env_name) mock_c.assert_called_once_with(channel_root, env_name) mock_i.assert_called_once_with(channel_root, meta_manifest_path=meta_manifest_path) @@ -146,16 +144,15 @@ def test_create_yaml_from_manifest( def test_smoke_cli(tmp_path, minimal_environment): test_environment_str = str(minimal_environment) test_output_metamanifest_root = str(tmp_path) - test_metamanifest_path = str(tmp_path / 'meta_manifest.yaml') + test_metamanifest_path = str(tmp_path / "meta_manifest.yaml") test_output_channel_root = str(tmp_path / "local_channel") - - cmd_str_clean = f"conda vendor create-meta-manifest --environment-yaml {test_environment_str} --manifest-root {test_output_metamanifest_root}" - subprocess.check_output( - cmd_str_clean, stderr=subprocess.STDOUT, shell=True - ).decode("utf-8") - cmd_str_clean = f"conda vendor create-channels --channel-root {test_output_channel_root} --meta-manifest-path {test_metamanifest_path} -v" - subprocess.check_output( - cmd_str_clean, stderr=subprocess.STDOUT, shell=True - ).decode("utf-8") - + cmd_str_clean = f"conda vendor meta-manifest --environment-yaml {test_environment_str} --manifest-root {test_output_metamanifest_root}" + subprocess.check_output(cmd_str_clean, stderr=subprocess.STDOUT, shell=True).decode( + "utf-8" + ) + cmd_str_clean = f"conda vendor channels --channel-root {test_output_channel_root} --meta-manifest-path {test_metamanifest_path} -v" + + subprocess.check_output(cmd_str_clean, stderr=subprocess.STDOUT, shell=True).decode( + "utf-8" + ) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index f10a543..eaf86b1 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -27,6 +27,14 @@ def test_get_conda_platform(mock_struct) -> None: assert mock_struct.call_count == 1 +def test_get_conda_platform_custom(): + test_platforms = ["linux-64", "linux-32", "win-64", "win-32", "osx-64"] + expected_returns = ["linux-64", "linux-32", "win-64", "win-32", "osx-64"] + + actual_returns = [get_conda_platform(custom_platform=p) for p in test_platforms] + assert set(actual_returns) == set(expected_returns) + + def test_LockWrapper_init(): lw = LockWrapper() assert isinstance(lw, LockWrapper) @@ -118,12 +126,7 @@ def test_get_manifest(meta_manifest_fixture): }, ] - test_env_deps_solution = { - "actions": { - "FETCH": test_fetch_entries, - "LINK": [], - } - } + test_env_deps_solution = {"actions": {"FETCH": test_fetch_entries, "LINK": [],}} test_meta_manifest.env_deps["solution"] = test_env_deps_solution