From 3cb0ec964ff5f6ae36afba273570c65d89315f47 Mon Sep 17 00:00:00 2001 From: Yu Date: Thu, 20 Sep 2018 16:04:56 -0700 Subject: [PATCH 1/2] Update gov cloud account number --- .../sparksdk/algorithms/ImageURIProvider.scala | 6 ++++-- ...orizationMachinesSageMakerEstimatorTests.scala | 10 ++++++++++ .../KMeansSageMakerEstimatorTests.scala | 4 ++++ .../LinearLearnerSageMakerEstimatorTests.scala | 15 +++++++++++++++ .../algorithms/PCASageMakerEstimatorTests.scala | 4 ++++ .../XGBoostSageMakerEstimatorTests.scala | 4 ++++ 6 files changed, 41 insertions(+), 2 deletions(-) diff --git a/sagemaker-spark-sdk/src/main/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/ImageURIProvider.scala b/sagemaker-spark-sdk/src/main/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/ImageURIProvider.scala index e32f016..454ce51 100644 --- a/sagemaker-spark-sdk/src/main/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/ImageURIProvider.scala +++ b/sagemaker-spark-sdk/src/main/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/ImageURIProvider.scala @@ -40,7 +40,8 @@ private[algorithms] object SagerMakerRegionAccountMaps { Regions.AP_NORTHEAST_1.getName -> "351501993468", Regions.AP_NORTHEAST_2.getName -> "835164637446", Regions.EU_CENTRAL_1.getName -> "664544806723", - Regions.AP_SOUTHEAST_2.getName -> "712309505854" + Regions.AP_SOUTHEAST_2.getName -> "712309505854", + Regions.GovCloud.getName -> "226302683700" ) // For LDA @@ -64,7 +65,8 @@ private[algorithms] object SagerMakerRegionAccountMaps { Regions.AP_NORTHEAST_1.getName -> "501404015308", Regions.AP_NORTHEAST_2.getName -> "306986355934", Regions.EU_CENTRAL_1.getName -> "813361260812", - Regions.AP_SOUTHEAST_2.getName -> "544295431143" + Regions.AP_SOUTHEAST_2.getName -> "544295431143", + Regions.GovCloud.getName -> "226302683700" ) } diff --git a/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/FactorizationMachinesSageMakerEstimatorTests.scala b/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/FactorizationMachinesSageMakerEstimatorTests.scala index 23752e5..5850d10 100644 --- a/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/FactorizationMachinesSageMakerEstimatorTests.scala +++ b/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/FactorizationMachinesSageMakerEstimatorTests.scala @@ -96,6 +96,11 @@ class FactorizationMachinesSageMakerEstimatorTests extends FlatSpec with Mockito createFactorizationMachinesBinaryClassifier(region = Regions.AP_SOUTHEAST_2.getName) assert(estimatorAPSouthEast2.trainingImage == "712309505854.dkr.ecr.ap-southeast-2.amazonaws.com/factorization-machines:1") + + val estimatorGovCloud = + createFactorizationMachinesBinaryClassifier(region = Regions.GovCloud.getName) + assert(estimatorGovCloud.trainingImage == + "226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/factorization-machines:1") } it should "use the correct defaults for regressor" in { @@ -142,6 +147,11 @@ class FactorizationMachinesSageMakerEstimatorTests extends FlatSpec with Mockito createFactorizationMachinesRegressor(region = Regions.AP_SOUTHEAST_2.getName) assert(estimatorAPSouthEast2.trainingImage == "712309505854.dkr.ecr.ap-southeast-2.amazonaws.com/factorization-machines:1") + + val estimatorGovCloud = + createFactorizationMachinesRegressor(region = Regions.GovCloud.getName) + assert(estimatorGovCloud.trainingImage == + "226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/factorization-machines:1") } it should "setFeatureDim" in { diff --git a/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/KMeansSageMakerEstimatorTests.scala b/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/KMeansSageMakerEstimatorTests.scala index 6725a77..0dabbf5 100644 --- a/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/KMeansSageMakerEstimatorTests.scala +++ b/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/KMeansSageMakerEstimatorTests.scala @@ -84,6 +84,10 @@ class KMeansSageMakerEstimatorTests extends FlatSpec with Matchers with MockitoS val estimatorAPSouthEast2 = createKMeansEstimator(region = Regions.AP_SOUTHEAST_2.getName) assert(estimatorAPSouthEast2.trainingImage == "712309505854.dkr.ecr.ap-southeast-2.amazonaws.com/kmeans:1") + + val estimatorGovCloud = createKMeansEstimator(region = Regions.GovCloud.getName) + assert(estimatorGovCloud.trainingImage == + "226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/kmeans:1") } it should "setK" in { diff --git a/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/LinearLearnerSageMakerEstimatorTests.scala b/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/LinearLearnerSageMakerEstimatorTests.scala index 7a6edf4..56a962e 100644 --- a/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/LinearLearnerSageMakerEstimatorTests.scala +++ b/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/LinearLearnerSageMakerEstimatorTests.scala @@ -101,6 +101,11 @@ class LinearLearnerSageMakerEstimatorTests extends FlatSpec with MockitoSugar { createLinearLearnerBinaryClassifier(region = Regions.AP_SOUTHEAST_2.getName) assert(estimatorAPSouthEast2.trainingImage == "712309505854.dkr.ecr.ap-southeast-2.amazonaws.com/linear-learner:1") + + val estimatorGovCloud = + createLinearLearnerBinaryClassifier(region = Regions.GovCloud.getName) + assert(estimatorGovCloud.trainingImage == + "226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/linear-learner:1") } it should "use the correct defaults for multiclass classifier" in { @@ -151,6 +156,11 @@ class LinearLearnerSageMakerEstimatorTests extends FlatSpec with MockitoSugar { createLinearLearnerMultiClassClassifier(region = Regions.AP_SOUTHEAST_2.getName) assert(estimatorAPSouthEast2.trainingImage == "712309505854.dkr.ecr.ap-southeast-2.amazonaws.com/linear-learner:1") + + val estimatorGovCloud = + createLinearLearnerMultiClassClassifier(region = Regions.GovCloud.getName) + assert(estimatorGovCloud.trainingImage == + "226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/linear-learner:1") } it should "use the correct defaults for regressor" in { @@ -197,6 +207,11 @@ class LinearLearnerSageMakerEstimatorTests extends FlatSpec with MockitoSugar { createLinearLearnerRegressor(region = Regions.AP_SOUTHEAST_2.getName) assert(estimatorAPSouthEast2.trainingImage == "712309505854.dkr.ecr.ap-southeast-2.amazonaws.com/linear-learner:1") + + val estimatorGovCloud = + createLinearLearnerRegressor(region = Regions.GovCloud.getName) + assert(estimatorGovCloud.trainingImage == + "226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/linear-learner:1") } it should "setFeatureDim" in { diff --git a/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/PCASageMakerEstimatorTests.scala b/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/PCASageMakerEstimatorTests.scala index d3c568e..eb78953 100644 --- a/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/PCASageMakerEstimatorTests.scala +++ b/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/PCASageMakerEstimatorTests.scala @@ -64,6 +64,10 @@ class PCASageMakerEstimatorTests extends FlatSpec with MockitoSugar { val estimatorAPSouthEast2 = createPCAEstimator(region = Regions.AP_SOUTHEAST_2.getName) assert(estimatorAPSouthEast2.trainingImage == "712309505854.dkr.ecr.ap-southeast-2.amazonaws.com/pca:1") + + val estimatorGovCloud = createPCAEstimator(region = Regions.GovCloud.getName) + assert(estimatorGovCloud.trainingImage == + "226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/pca:1") } it should "use the correct defaults" in { diff --git a/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/XGBoostSageMakerEstimatorTests.scala b/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/XGBoostSageMakerEstimatorTests.scala index 2b5f300..23ef35d 100644 --- a/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/XGBoostSageMakerEstimatorTests.scala +++ b/sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/XGBoostSageMakerEstimatorTests.scala @@ -83,6 +83,10 @@ class XGBoostSageMakerEstimatorTests extends FlatSpec with Matchers with Mockito val estimatorAPSouthEast2 = createXGBoostEstimator(region = Regions.AP_SOUTHEAST_2.getName) assert(estimatorAPSouthEast2.trainingImage == "544295431143.dkr.ecr.ap-southeast-2.amazonaws.com/xgboost:1") + + val estimatorGovCloud = createXGBoostEstimator(region = Regions.GovCloud.getName) + assert(estimatorGovCloud.trainingImage == + "226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/xgboost:1") } it should "setBooster" in { From 4fcd25d5de8503290ad1f9cdac1ac89facd601be Mon Sep 17 00:00:00 2001 From: Yu Date: Thu, 20 Sep 2018 16:25:41 -0700 Subject: [PATCH 2/2] update changelog --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 52667cc..4ecc40a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,11 @@ CHANGELOG ========= +1.1.5dev +======== + +* spark/pyspark: Enable PDT region support for spark SDK + 1.1.4 =====