diff --git a/pyproject.toml b/pyproject.toml index e35a43c163..02b89e975a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,10 +39,10 @@ dependencies = [ "google-pasta", "importlib-metadata>=1.4.0,<7.0", "jsonschema", - "numpy==1.26.4", + "numpy>=1.26.4,<3.0", "omegaconf>=2.2,<3", "packaging>=23.0,<25", - "pandas", + "pandas>=2.3.0", "pathos", "platformdirs", "protobuf>=3.12,<6.32", diff --git a/requirements/extras/scipy_requirements.txt b/requirements/extras/scipy_requirements.txt index 44ce1d9331..f89caf8c2b 100644 --- a/requirements/extras/scipy_requirements.txt +++ b/requirements/extras/scipy_requirements.txt @@ -1 +1 @@ -scipy==1.11.3 +scipy==1.13.0 diff --git a/requirements/extras/test_requirements.txt b/requirements/extras/test_requirements.txt index d66235d84a..393fbac589 100644 --- a/requirements/extras/test_requirements.txt +++ b/requirements/extras/test_requirements.txt @@ -1,5 +1,5 @@ tox==3.24.5 -numpy==1.26.4 +numpy>=2.0.0, <3.0 build[virtualenv]==1.2.1 flake8==7.1.2 pytest==6.2.5 @@ -23,8 +23,8 @@ requests==2.32.2 sagemaker-experiments==0.1.35 Jinja2==3.1.6 pyvis==0.2.1 -pandas==1.4.4 -scikit-learn==1.3.0 +pandas>=2.3.0 +scikit-learn==1.6.1 cloudpickle==2.2.1 jsonpickle<4.0.0 PyYAML>=6.0.1 @@ -44,7 +44,7 @@ onnx==1.17.0 nbformat>=5.9,<6 accelerate>=0.24.1,<=0.27.0 schema==0.7.5 -tensorflow>=2.16.2,<=2.18.0 +tensorflow>=2.16.2,<=2.19.0 mlflow>=2.14.2,<3 huggingface_hub==0.26.2 uvicorn>=0.30.1 diff --git a/src/sagemaker/image_uri_config/sklearn.json b/src/sagemaker/image_uri_config/sklearn.json index 85114a11d2..0087f9fb14 100644 --- a/src/sagemaker/image_uri_config/sklearn.json +++ b/src/sagemaker/image_uri_config/sklearn.json @@ -388,6 +388,54 @@ "us-west-2": "246618743249" }, "repository": "sagemaker-scikit-learn" + }, + "1.4-2": { + "processors": [ + "cpu" + ], + "py_versions": [ + "py3" + ], + "registries": { + "af-south-1": "510948584623", + "ap-east-1": "651117190479", + "ap-northeast-1": "354813040037", + "ap-northeast-2": "366743142698", + "ap-northeast-3": "867004704886", + "ap-south-1": "720646828776", + "ap-south-2": "628508329040", + "ap-southeast-1": "121021644041", + "ap-southeast-2": "783357654285", + "ap-southeast-3": "951798379941", + "ap-southeast-4": "106583098589", + "ca-central-1": "341280168497", + "ca-west-1": "190319476487", + "cn-north-1": "450853457545", + "cn-northwest-1": "451049120500", + "eu-central-1": "492215442770", + "eu-central-2": "680994064768", + "eu-north-1": "662702820516", + "eu-south-1": "978288397137", + "eu-south-2": "104374241257", + "eu-west-1": "141502667606", + "eu-west-2": "764974769150", + "eu-west-3": "659782779980", + "il-central-1": "898809789911", + "me-central-1": "272398656194", + "me-south-1": "801668240914", + "sa-east-1": "737474898029", + "us-east-1": "683313688378", + "us-east-2": "257758044811", + "us-gov-east-1": "237065988967", + "us-gov-west-1": "414596584902", + "us-iso-east-1": "833128469047", + "us-isob-east-1": "281123927165", + "us-isof-east-1": "108575199400", + "us-isof-south-1": "124985052026", + "us-west-1": "746614075791", + "us-west-2": "246618743249" + }, + "repository": "sagemaker-scikit-learn" } } }, diff --git a/src/sagemaker/serve/utils/conda_in_process.yml b/src/sagemaker/serve/utils/conda_in_process.yml index d51754ec5a..740c798bd5 100644 --- a/src/sagemaker/serve/utils/conda_in_process.yml +++ b/src/sagemaker/serve/utils/conda_in_process.yml @@ -12,7 +12,7 @@ dependencies: - boto3>=1.34.142,<2.0 - cloudpickle==2.2.1 - google-pasta - - numpy==1.26.4 + - numpy>=2.0.0,<3.0 - protobuf>=3.12,<5.0 - smdebug_rulesconfig==1.0.1 - importlib-metadata>=1.4.0,<7.0 @@ -64,7 +64,7 @@ dependencies: - multiprocess>=0.70.14 - networkx>=3.1 - packaging>=23.1 - - pandas>=1.5.3 + - pandas>=2.3.0 - pathos>=0.3.0 - pillow>=9.5.0 - platformdirs>=3.2.0 diff --git a/tests/data/remote_function/requirements.txt b/tests/data/remote_function/requirements.txt index 44ce1d9331..f89caf8c2b 100644 --- a/tests/data/remote_function/requirements.txt +++ b/tests/data/remote_function/requirements.txt @@ -1 +1 @@ -scipy==1.11.3 +scipy==1.13.0 diff --git a/tests/data/serve_resources/mlflow/pytorch/conda.yaml b/tests/data/serve_resources/mlflow/pytorch/conda.yaml index b740d25b70..93c33b4cf8 100644 --- a/tests/data/serve_resources/mlflow/pytorch/conda.yaml +++ b/tests/data/serve_resources/mlflow/pytorch/conda.yaml @@ -2,23 +2,23 @@ channels: - conda-forge dependencies: - python=3.10.13 -- pip<=23.3.1 +- pip<=24.3 - pip: - - mlflow==2.10.2 + - mlflow>=2.16.1 - astunparse==1.6.3 - cffi==1.16.0 - cloudpickle==2.2.1 - defusedxml==0.7.1 - dill==0.3.9 - gmpy2==2.1.2 - - numpy==1.26.4 + - numpy>=2.0.0,<3.0 - opt-einsum==3.3.0 - packaging==24.0 - - pandas==2.2.1 + - pandas>=2.3.0 - pyyaml==6.0.1 - requests==2.31.0 - torch>=2.6.0 - torchvision>=0.17.0 - tqdm==4.66.2 - - scikit-learn==1.3.2 + - scikit-learn==1.6.1 name: mlflow-env diff --git a/tests/data/serve_resources/mlflow/pytorch/requirements.txt b/tests/data/serve_resources/mlflow/pytorch/requirements.txt index eabe5e8e82..bb2dc9293c 100644 --- a/tests/data/serve_resources/mlflow/pytorch/requirements.txt +++ b/tests/data/serve_resources/mlflow/pytorch/requirements.txt @@ -5,10 +5,10 @@ cloudpickle==2.2.1 defusedxml==0.7.1 dill==0.3.9 gmpy2==2.1.2 -numpy==1.26.4 +numpy>=2.0.0,<3.0 opt-einsum==3.3.0 packaging>=23.0,<25 -pandas==2.2.1 +pandas>=2.3.0 pyyaml==6.0.1 requests==2.32.4 torch>=2.6.0 diff --git a/tests/data/serve_resources/mlflow/tensorflow/MLmodel b/tests/data/serve_resources/mlflow/tensorflow/MLmodel index 6a961f3612..f00412149d 100644 --- a/tests/data/serve_resources/mlflow/tensorflow/MLmodel +++ b/tests/data/serve_resources/mlflow/tensorflow/MLmodel @@ -10,7 +10,7 @@ flavors: code: null model_type: tf2-module saved_model_dir: tf2model -mlflow_version: 2.20.3 +mlflow_version: 2.11.1 model_size_bytes: 23823 model_uuid: 40d2323944294fce898d8693455f60e8 run_id: 592132312fb84935b201de2c027c54c6 diff --git a/tests/data/serve_resources/mlflow/tensorflow/conda.yaml b/tests/data/serve_resources/mlflow/tensorflow/conda.yaml index 90d8c300a0..2f60ba6451 100644 --- a/tests/data/serve_resources/mlflow/tensorflow/conda.yaml +++ b/tests/data/serve_resources/mlflow/tensorflow/conda.yaml @@ -2,7 +2,7 @@ channels: - conda-forge dependencies: - python=3.10.13 -- pip<=23.3.1 +- pip<=24.3 - pip: - mlflow==2.11.1 - cloudpickle==2.2.1 diff --git a/tests/data/serve_resources/mlflow/tensorflow_numpy2/MLmodel b/tests/data/serve_resources/mlflow/tensorflow_numpy2/MLmodel new file mode 100644 index 0000000000..694ab87f3d --- /dev/null +++ b/tests/data/serve_resources/mlflow/tensorflow_numpy2/MLmodel @@ -0,0 +1,13 @@ +artifact_path: model +flavors: + python_function: + env: + conda: conda.yaml + virtualenv: python_env.yaml + loader_module: mlflow.tensorflow + python_version: 3.10.0 + tensorflow: + saved_model_dir: tf2model + model_type: tf2-module +mlflow_version: 2.20.3 +model_uuid: test-uuid-numpy2 diff --git a/tests/data/serve_resources/mlflow/tensorflow_numpy2/conda.yaml b/tests/data/serve_resources/mlflow/tensorflow_numpy2/conda.yaml new file mode 100644 index 0000000000..079d4cb62e --- /dev/null +++ b/tests/data/serve_resources/mlflow/tensorflow_numpy2/conda.yaml @@ -0,0 +1,11 @@ +channels: +- conda-forge +dependencies: +- python=3.10 +- pip +- pip: + - numpy>=2.0.0 + - tensorflow==2.19.0 + - scikit-learn + - mlflow +name: mlflow-env diff --git a/tests/data/serve_resources/mlflow/tensorflow_numpy2/data/keras_module.txt b/tests/data/serve_resources/mlflow/tensorflow_numpy2/data/keras_module.txt new file mode 100644 index 0000000000..5445ce90f6 --- /dev/null +++ b/tests/data/serve_resources/mlflow/tensorflow_numpy2/data/keras_module.txt @@ -0,0 +1 @@ +tensorflow.keras diff --git a/tests/data/serve_resources/mlflow/tensorflow_numpy2/data/model.keras b/tests/data/serve_resources/mlflow/tensorflow_numpy2/data/model.keras new file mode 100644 index 0000000000..582536ce65 Binary files /dev/null and b/tests/data/serve_resources/mlflow/tensorflow_numpy2/data/model.keras differ diff --git a/tests/data/serve_resources/mlflow/tensorflow_numpy2/data/save_format.txt b/tests/data/serve_resources/mlflow/tensorflow_numpy2/data/save_format.txt new file mode 100644 index 0000000000..f6afb303b0 --- /dev/null +++ b/tests/data/serve_resources/mlflow/tensorflow_numpy2/data/save_format.txt @@ -0,0 +1 @@ +tf diff --git a/tests/data/serve_resources/mlflow/tensorflow_numpy2/python_env.yaml b/tests/data/serve_resources/mlflow/tensorflow_numpy2/python_env.yaml new file mode 100644 index 0000000000..511b585ede --- /dev/null +++ b/tests/data/serve_resources/mlflow/tensorflow_numpy2/python_env.yaml @@ -0,0 +1,5 @@ +python: 3.10.0 +build_dependencies: +- pip +dependencies: +- -r requirements.txt diff --git a/tests/data/serve_resources/mlflow/tensorflow_numpy2/requirements.txt b/tests/data/serve_resources/mlflow/tensorflow_numpy2/requirements.txt new file mode 100644 index 0000000000..ad108e44f1 --- /dev/null +++ b/tests/data/serve_resources/mlflow/tensorflow_numpy2/requirements.txt @@ -0,0 +1,3 @@ +numpy>=2.0.0 +tensorflow==2.19.0 +scikit-learn diff --git a/tests/data/serve_resources/mlflow/xgboost/conda.yaml b/tests/data/serve_resources/mlflow/xgboost/conda.yaml index 44ca3c4c2e..033b91d969 100644 --- a/tests/data/serve_resources/mlflow/xgboost/conda.yaml +++ b/tests/data/serve_resources/mlflow/xgboost/conda.yaml @@ -2,14 +2,14 @@ channels: - conda-forge dependencies: - python=3.10.13 -- pip<=23.3.1 +- pip<=24.3 - pip: - - mlflow==2.11.1 + - mlflow>=2.16.1 - lz4==4.3.2 - - numpy==1.26.4 - - pandas==2.2.1 + - numpy>=1.26.4,<3.0 + - pandas>=2.3.0 - psutil==5.9.8 - - scikit-learn==1.3.2 - - scipy==1.11.3 + - scikit-learn==1.6.1 + - scipy==1.13.0 - xgboost==1.7.1 name: mlflow-env diff --git a/tests/data/serve_resources/mlflow/xgboost/requirements.txt b/tests/data/serve_resources/mlflow/xgboost/requirements.txt index 78c7a1afda..8907600722 100644 --- a/tests/data/serve_resources/mlflow/xgboost/requirements.txt +++ b/tests/data/serve_resources/mlflow/xgboost/requirements.txt @@ -1,8 +1,8 @@ mlflow==3.1.0 lz4==4.3.2 -numpy==1.26.4 -pandas==2.0.3 +numpy>=1.26.4,<3.0 +pandas>=2.3.0 psutil==5.9.8 -scikit-learn==1.5.1 -scipy==1.11.3 +scikit-learn==1.6.1 +scipy==1.13.0 xgboost==1.7.1 diff --git a/tests/data/workflow/requirements.txt b/tests/data/workflow/requirements.txt index 44ce1d9331..f89caf8c2b 100644 --- a/tests/data/workflow/requirements.txt +++ b/tests/data/workflow/requirements.txt @@ -1 +1 @@ -scipy==1.11.3 +scipy==1.13.0 diff --git a/tests/integ/sagemaker/experiments/test_run.py b/tests/integ/sagemaker/experiments/test_run.py index f00f53a5ad..7493cc5036 100644 --- a/tests/integ/sagemaker/experiments/test_run.py +++ b/tests/integ/sagemaker/experiments/test_run.py @@ -693,7 +693,7 @@ def _generate_estimator( sagemaker_client_config=sagemaker_client_config, ) return SKLearn( - framework_version="1.2-1", + framework_version="1.4-2", entry_point=_ENTRY_POINT_PATH, dependencies=[sdk_tar], role=execution_role, diff --git a/tests/integ/sagemaker/feature_store/feature_processor/test_feature_processor_integ.py b/tests/integ/sagemaker/feature_store/feature_processor/test_feature_processor_integ.py index fb69bb1b3f..14030534a2 100644 --- a/tests/integ/sagemaker/feature_store/feature_processor/test_feature_processor_integ.py +++ b/tests/integ/sagemaker/feature_store/feature_processor/test_feature_processor_integ.py @@ -1108,15 +1108,15 @@ def get_expected_dataframe(): expected_dataframe = pd.read_csv(os.path.join(_FEATURE_PROCESSOR_DIR, "car-data.csv")) expected_dataframe["Model"].replace("^\d\d\d\d\s", "", regex=True, inplace=True) # noqa: W605 expected_dataframe["Mileage"].replace("(,)|(mi\.)", "", regex=True, inplace=True) # noqa: W605 - expected_dataframe["Mileage"].replace("Not available", np.NaN, inplace=True) + expected_dataframe["Mileage"].replace("Not available", np.nan, inplace=True) expected_dataframe["Price"].replace("\$", "", regex=True, inplace=True) # noqa: W605 expected_dataframe["Price"].replace(",", "", regex=True, inplace=True) expected_dataframe["MSRP"].replace( "(^MSRP\s\\$)|(,)", "", regex=True, inplace=True # noqa: W605 ) - expected_dataframe["MSRP"].replace("Not specified", np.NaN, inplace=True) + expected_dataframe["MSRP"].replace("Not specified", np.nan, inplace=True) expected_dataframe["MSRP"].replace( - "\\$\d+[a-zA-Z\s]+", np.NaN, regex=True, inplace=True # noqa: W605 + "\\$\d+[a-zA-Z\s]+", np.nan, regex=True, inplace=True # noqa: W605 ) expected_dataframe["Mileage"] = expected_dataframe["Mileage"].astype(float) expected_dataframe["Price"] = expected_dataframe["Price"].astype(float) diff --git a/tests/integ/sagemaker/remote_function/test_decorator.py b/tests/integ/sagemaker/remote_function/test_decorator.py index fa55d7dfa7..33b3a6bdc8 100644 --- a/tests/integ/sagemaker/remote_function/test_decorator.py +++ b/tests/integ/sagemaker/remote_function/test_decorator.py @@ -315,6 +315,9 @@ def divide(x, y): divide(10, 2) +@pytest.mark.skip( + reason="Test only valid for numpy < 2.0 due to serialization compatibility changes", +) def test_with_incompatible_dependencies( sagemaker_session, dummy_container_without_error, cpu_instance_type ): diff --git a/tests/integ/sagemaker/serve/test_serve_mlflow_tensorflow_flavor_happy.py b/tests/integ/sagemaker/serve/test_serve_mlflow_tensorflow_flavor_happy.py index c25cbd7e18..9c0257d44c 100644 --- a/tests/integ/sagemaker/serve/test_serve_mlflow_tensorflow_flavor_happy.py +++ b/tests/integ/sagemaker/serve/test_serve_mlflow_tensorflow_flavor_happy.py @@ -105,7 +105,9 @@ def tensorflow_schema_builder(custom_request_translator, custom_response_transla @pytest.mark.skipif( PYTHON_VERSION_IS_NOT_310, - reason="The goal of these test are to test the serving components of our feature", + np.__version__ >= "2.0.0", + reason="The goal of these test are to test the serving components of our feature and \ + the input model artifacts used in this specific test are generated with py310 and numpy<2.", ) def test_happy_tensorflow_sagemaker_endpoint_with_tensorflow_serving( sagemaker_session, diff --git a/tests/integ/sagemaker/serve/test_tensorflow_serving_numpy2.py b/tests/integ/sagemaker/serve/test_tensorflow_serving_numpy2.py new file mode 100644 index 0000000000..4575639eda --- /dev/null +++ b/tests/integ/sagemaker/serve/test_tensorflow_serving_numpy2.py @@ -0,0 +1,203 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Simple integration test for TensorFlow Serving builder with numpy 2.0 compatibility.""" + +from __future__ import absolute_import + +import pytest +import io +import os +import numpy as np +import logging +from tests.integ import DATA_DIR + +from sagemaker.serve.builder.model_builder import ModelBuilder, Mode +from sagemaker.serve.builder.schema_builder import SchemaBuilder, CustomPayloadTranslator +from sagemaker.serve.utils.types import ModelServer + +logger = logging.getLogger(__name__) + + +class TestTensorFlowServingNumpy2: + """Simple integration tests for TensorFlow Serving with numpy 2.0.""" + + def test_tensorflow_serving_validation_with_numpy2(self, sagemaker_session): + """Test TensorFlow Serving validation works with numpy 2.0.""" + logger.info(f"Testing TensorFlow Serving validation with numpy {np.__version__}") + + # Create a simple schema builder with numpy 2.0 arrays + input_data = np.array([[1.0, 2.0, 3.0]], dtype=np.float32) + output_data = np.array([4.0], dtype=np.float32) + + schema_builder = SchemaBuilder(sample_input=input_data, sample_output=output_data) + + # Test without MLflow model - should raise validation error + model_builder = ModelBuilder( + mode=Mode.SAGEMAKER_ENDPOINT, + model_server=ModelServer.TENSORFLOW_SERVING, + schema_builder=schema_builder, + sagemaker_session=sagemaker_session, + ) + + with pytest.raises( + ValueError, match="Tensorflow Serving is currently only supported for mlflow models" + ): + model_builder._validate_for_tensorflow_serving() + + logger.info("TensorFlow Serving validation test passed") + + def test_tensorflow_serving_with_sample_mlflow_model(self, sagemaker_session): + """Test TensorFlow Serving builder initialization with sample MLflow model.""" + logger.info("Testing TensorFlow Serving with sample MLflow model") + + # Use constant MLflow model structure from test data + mlflow_model_dir = os.path.join( + DATA_DIR, "serve_resources", "mlflow", "tensorflow_numpy2_removed" + ) + + # Create schema builder with numpy 2.0 arrays + input_data = np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float32) + output_data = np.array([5.0], dtype=np.float32) + + schema_builder = SchemaBuilder(sample_input=input_data, sample_output=output_data) + + # Create ModelBuilder - this should not raise validation errors + model_builder = ModelBuilder( + mode=Mode.SAGEMAKER_ENDPOINT, + model_server=ModelServer.TENSORFLOW_SERVING, + schema_builder=schema_builder, + sagemaker_session=sagemaker_session, + model_metadata={"MLFLOW_MODEL_PATH": mlflow_model_dir}, + role_arn="arn:aws:iam::123456789012:role/SageMakerRole", + ) + + # Initialize MLflow handling to set _is_mlflow_model flag + model_builder._handle_mlflow_input() + + # Test validation passes + model_builder._validate_for_tensorflow_serving() + logger.info("TensorFlow Serving with sample MLflow model test passed") + + def test_numpy2_custom_payload_translators(self): + """Test custom payload translators work with numpy 2.0.""" + logger.info(f"Testing custom payload translators with numpy {np.__version__}") + + class Numpy2RequestTranslator(CustomPayloadTranslator): + def serialize_payload_to_bytes(self, payload: object) -> bytes: + buffer = io.BytesIO() + np.save(buffer, payload, allow_pickle=False) + return buffer.getvalue() + + def deserialize_payload_from_stream(self, stream) -> object: + return np.load(io.BytesIO(stream.read()), allow_pickle=False) + + class Numpy2ResponseTranslator(CustomPayloadTranslator): + def serialize_payload_to_bytes(self, payload: object) -> bytes: + buffer = io.BytesIO() + np.save(buffer, np.array(payload), allow_pickle=False) + return buffer.getvalue() + + def deserialize_payload_from_stream(self, stream) -> object: + return np.load(io.BytesIO(stream.read()), allow_pickle=False) + + # Test data + test_input = np.array([[1.0, 2.0, 3.0]], dtype=np.float32) + test_output = np.array([4.0], dtype=np.float32) + + # Create translators + request_translator = Numpy2RequestTranslator() + response_translator = Numpy2ResponseTranslator() + + # Test request translator + serialized_input = request_translator.serialize_payload_to_bytes(test_input) + assert isinstance(serialized_input, bytes) + + deserialized_input = request_translator.deserialize_payload_from_stream( + io.BytesIO(serialized_input) + ) + np.testing.assert_array_equal(test_input, deserialized_input) + + # Test response translator + serialized_output = response_translator.serialize_payload_to_bytes(test_output) + assert isinstance(serialized_output, bytes) + + deserialized_output = response_translator.deserialize_payload_from_stream( + io.BytesIO(serialized_output) + ) + np.testing.assert_array_equal(test_output, deserialized_output) + + logger.info("Custom payload translators test passed") + + def test_numpy2_schema_builder_creation(self): + """Test SchemaBuilder creation with numpy 2.0 arrays.""" + logger.info(f"Testing SchemaBuilder with numpy {np.__version__}") + + # Create test data with numpy 2.0 + input_data = np.array([[1.0, 2.0, 3.0, 4.0, 5.0]], dtype=np.float32) + output_data = np.array([10.0], dtype=np.float32) + + # Create SchemaBuilder + schema_builder = SchemaBuilder(sample_input=input_data, sample_output=output_data) + + # Verify schema builder properties + assert schema_builder.sample_input is not None + assert schema_builder.sample_output is not None + + # Test with custom translators + class TestTranslator(CustomPayloadTranslator): + def serialize_payload_to_bytes(self, payload: object) -> bytes: + buffer = io.BytesIO() + np.save(buffer, payload, allow_pickle=False) + return buffer.getvalue() + + def deserialize_payload_from_stream(self, stream) -> object: + return np.load(io.BytesIO(stream.read()), allow_pickle=False) + + translator = TestTranslator() + schema_builder_with_translator = SchemaBuilder( + sample_input=input_data, + sample_output=output_data, + input_translator=translator, + output_translator=translator, + ) + + assert schema_builder_with_translator.custom_input_translator is not None + assert schema_builder_with_translator.custom_output_translator is not None + + logger.info("SchemaBuilder creation test passed") + + def test_numpy2_basic_operations(self): + """Test basic numpy 2.0 operations used in TensorFlow Serving.""" + logger.info(f"Testing basic numpy 2.0 operations. Version: {np.__version__}") + + # Test array creation + arr = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32) + assert arr.dtype == np.float32 + assert arr.shape == (4,) + + # Test array operations + arr_2d = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32) + assert arr_2d.shape == (2, 2) + + # Test serialization without pickle (numpy 2.0 safe) + buffer = io.BytesIO() + np.save(buffer, arr_2d, allow_pickle=False) + buffer.seek(0) + loaded_arr = np.load(buffer, allow_pickle=False) + + np.testing.assert_array_equal(arr_2d, loaded_arr) + + # Test dtype preservation + assert loaded_arr.dtype == np.float32 + + logger.info("Basic numpy 2.0 operations test passed") diff --git a/tests/integ/test_dependency_compatibility.py b/tests/integ/test_dependency_compatibility.py new file mode 100644 index 0000000000..185d7fb021 --- /dev/null +++ b/tests/integ/test_dependency_compatibility.py @@ -0,0 +1,85 @@ +from __future__ import absolute_import + +"""Integration test to verify dependency compatibility.""" + +import subprocess +import sys +import tempfile +import os +import pytest +from pathlib import Path + + +def test_dependency_compatibility(): + """Test that all dependencies in pyproject.toml are compatible.""" + # Get project root + project_root = Path(__file__).parent.parent.parent + pyproject_path = project_root / "pyproject.toml" + + assert pyproject_path.exists(), "pyproject.toml not found" + + with tempfile.TemporaryDirectory() as temp_dir: + # Create a fresh virtual environment + venv_path = os.path.join(temp_dir, "test_env") + subprocess.run([sys.executable, "-m", "venv", venv_path], check=True) + + # Get pip path for the virtual environment + if sys.platform == "win32": + pip_path = os.path.join(venv_path, "Scripts", "pip") + else: + pip_path = os.path.join(venv_path, "bin", "pip") + + # Install dependencies + result = subprocess.run( + [pip_path, "install", "-e", str(project_root)], capture_output=True, text=True + ) + + if result.returncode != 0: + pytest.fail(f"Dependency installation failed:\n{result.stderr}") + + # Check for conflicts + check_result = subprocess.run([pip_path, "check"], capture_output=True, text=True) + + if check_result.returncode != 0: + pytest.fail(f"Dependency conflicts found:\n{check_result.stdout}") + + +def test_numpy_pandas_compatibility(): + """Test specific NumPy-pandas compatibility.""" + try: + import numpy as np + import pandas as pd + + # Test basic operations + arr = np.array([1, 2, 3]) + df = pd.DataFrame({"col": arr}) + + # This should not raise the dtype size error + result = df.values + assert isinstance(result, np.ndarray) + + except ImportError: + pytest.skip("NumPy or pandas not available") + except ValueError as e: + if "numpy.dtype size changed" in str(e): + pytest.fail(f"NumPy-pandas compatibility issue: {e}") + raise + + +def test_critical_imports(): + """Test that critical packages can be imported without conflicts.""" + critical_packages = ["numpy", "pandas", "boto3", "sagemaker_core", "protobuf", "cloudpickle"] + + failed_imports = [] + + for package in critical_packages: + try: + __import__(package) + except ImportError: + # Skip if package not installed + continue + except Exception as e: + failed_imports.append(f"{package}: {e}") + + if failed_imports: + pytest.fail("Import failures:\n" + "\n".join(failed_imports)) diff --git a/tests/integ/test_dependency_resolution.py b/tests/integ/test_dependency_resolution.py new file mode 100644 index 0000000000..cda4cdd3d7 --- /dev/null +++ b/tests/integ/test_dependency_resolution.py @@ -0,0 +1,70 @@ +from __future__ import absolute_import + +"""Test dependency resolution using pip-tools.""" + +import subprocess +import sys +import tempfile +import pytest +from pathlib import Path + + +def test_pip_compile_resolution(): + """Test that pip-compile can resolve all dependencies without conflicts.""" + project_root = Path(__file__).parent.parent.parent + pyproject_path = project_root / "pyproject.toml" + + with tempfile.TemporaryDirectory() as temp_dir: + # Install pip-tools + subprocess.run( + [sys.executable, "-m", "pip", "install", "pip-tools"], check=True, capture_output=True + ) + + # Try to compile dependencies + result = subprocess.run( + [ + sys.executable, + "-m", + "piptools", + "compile", + str(pyproject_path), + "--dry-run", + "--quiet", + ], + capture_output=True, + text=True, + cwd=temp_dir, + ) + + if result.returncode != 0: + # Check for specific conflict patterns + stderr = result.stderr.lower() + if "could not find a version" in stderr or "incompatible" in stderr: + pytest.fail(f"Dependency resolution failed:\n{result.stderr}") + # Other errors might be acceptable (missing extras, etc.) + + +def test_pipdeptree_conflicts(): + """Test using pipdeptree to detect conflicts.""" + try: + subprocess.run( + [sys.executable, "-m", "pip", "install", "pipdeptree"], check=True, capture_output=True + ) + + result = subprocess.run( + [sys.executable, "-m", "pipdeptree", "--warn", "conflict"], + capture_output=True, + text=True, + ) + + if "Warning!!" in result.stdout: + pytest.fail(f"Dependency conflicts detected:\n{result.stdout}") + + except subprocess.CalledProcessError: + pytest.skip("pipdeptree installation failed") + + +if __name__ == "__main__": + test_pip_compile_resolution() + test_pipdeptree_conflicts() + print("✅ Dependency resolution tests passed") diff --git a/tests/unit/sagemaker/jumpstart/constants.py b/tests/unit/sagemaker/jumpstart/constants.py index ae02c597da..f288dfd6e4 100644 --- a/tests/unit/sagemaker/jumpstart/constants.py +++ b/tests/unit/sagemaker/jumpstart/constants.py @@ -5361,7 +5361,7 @@ "safetensors==0.3.1", "sagemaker_jumpstart_huggingface_script_utilities==1.1.3", "sagemaker_jumpstart_script_utilities==1.1.9", - "scipy==1.11.1", + "scipy==1.13.0", "termcolor==2.3.0", "texttable==1.6.7", "tokenize-rt==5.1.0", @@ -7870,7 +7870,7 @@ "safetensors==0.3.1", "sagemaker_jumpstart_huggingface_script_utilities==1.1.3", "sagemaker_jumpstart_script_utilities==1.1.9", - "scipy==1.11.1", + "scipy==1.13.0", "termcolor==2.3.0", "texttable==1.6.7", "tokenize-rt==5.1.0", @@ -8346,7 +8346,7 @@ "safetensors==0.3.1", "sagemaker_jumpstart_huggingface_script_utilities==1.1.3", "sagemaker_jumpstart_script_utilities==1.1.9", - "scipy==1.11.1", + "scipy==1.13.0", "termcolor==2.3.0", "texttable==1.6.7", "tokenize-rt==5.1.0", @@ -12095,7 +12095,7 @@ "inference_vulnerabilities": [], "training_vulnerable": False, "training_dependencies": [ - "numpy==1.23.1", + "numpy>=2.0.0", "opencv_python==4.7.0.68", "sagemaker_jumpstart_prepack_script_utilities==1.0.0", ], @@ -14360,10 +14360,10 @@ "jmespath==1.0.1", "jsonschema==4.17.3", "multiprocess==0.70.14", - "numpy==1.26.4", + "numpy>=2.0.0", "oscrypto==1.3.0", "packaging==23.1", - "pandas==2.0.2", + "pandas>=2.3.0", "pathos==0.3.0", "pkgutil-resolve-name==1.3.10", "platformdirs==3.8.0", @@ -14884,10 +14884,10 @@ "jmespath==1.0.1", "jsonschema==4.17.3", "multiprocess==0.70.14", - "numpy==1.24.3", + "numpy>==2.0.0", "oscrypto==1.3.0", "packaging==23.1", - "pandas==2.0.2", + "pandas>=2.3.0", "pathos==0.3.0", "pkgutil-resolve-name==1.3.10", "platformdirs==3.8.0", @@ -17400,7 +17400,7 @@ "safetensors==0.3.1", "sagemaker_jumpstart_huggingface_script_utilities==1.1.4", "sagemaker_jumpstart_script_utilities==1.1.9", - "scipy==1.11.1", + "scipy==1.13.0", "termcolor==2.3.0", "texttable==1.6.7", "tokenize-rt==5.1.0", diff --git a/tests/unit/sagemaker/serve/detector/test_dependency_manager.py b/tests/unit/sagemaker/serve/detector/test_dependency_manager.py index 52e9822e57..2cbc93422c 100644 --- a/tests/unit/sagemaker/serve/detector/test_dependency_manager.py +++ b/tests/unit/sagemaker/serve/detector/test_dependency_manager.py @@ -21,8 +21,8 @@ DEPENDENCY_LIST = [ "requests==2.26.0", - "numpy==1.26.4", - "pandas<=1.3.3", + "numpy>=2.0.0", + "pandas>=2.3.0", "matplotlib<3.5.0", "scikit-learn>0.24.1", "Django!=4.0.0", @@ -34,8 +34,8 @@ EXPECTED_DEPENDENCY_MAP = { "requests": "==2.26.0", - "numpy": "==1.26.4", - "pandas": "<=1.3.3", + "numpy": ">=2.0.0", + "pandas": ">=2.3.0", "matplotlib": "<3.5.0", "scikit-learn": ">0.24.1", "Django": "!=4.0.0",