Skip to content

Commit 865bebb

Browse files
marfriazmarfriaz-aws
marfriaz
and
marfriaz-aws
authored
Update default_bucket and default_bucket_prefix to derive from sagemaker rather than sagemaker_core AND Append the default_bucket_prefix to s3 paths if one exists to sample notebooks (#4812)
* Append the default_bucket_prefix to s3 paths if one exists to sample notebooks **Description** This change checks the sagemaker session if there is a default_bucket_prefix set. If a default bucket prefix is specified, it is appended it to the s3 path. This ensures sample notebooks that utilize s3, works in SageMaker Unified Studio. * Update default_bucket and default_bucket_prefix to derive from sagemaker rather than sagemaker_core **Description** In order to be compatible with SageMaker Unified Studio, the default_bucket and default_bucket_prefix must be derived from the sagemaker python sdk rather than the sagemaker_core python SDK. --------- Co-authored-by: Marco Friaz <marfriaz@amazon.com>
1 parent cf8f7dd commit 865bebb

File tree

35 files changed

+372
-96
lines changed

35 files changed

+372
-96
lines changed

‎ sagemaker-core/get_started.ipynb

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,18 +36,21 @@
3636
"outputs": [],
3737
"source": [
3838
"import time\n",
39+
"import sagemaker\n",
40+
"\n",
3941
"from sagemaker_core.helper.session_helper import Session, get_execution_role\n",
4042
"\n",
4143
"# Set up region, role and bucket parameters used throughout the notebook.\n",
4244
"sagemaker_session = Session()\n",
4345
"region = sagemaker_session.boto_region_name\n",
4446
"role = get_execution_role()\n",
45-
"bucket = sagemaker_session.default_bucket()\n",
46-
"default_bucket_prefix = sagemaker_session.default_bucket_prefix\n",
47+
"bucket = sagemaker.Session().default_bucket()\n",
48+
"default_bucket_prefix = sagemaker.Session().default_bucket_prefix\n",
4749
"\n",
4850
"print(f\"AWS region: {region}\")\n",
4951
"print(f\"Execution role: {role}\")\n",
50-
"print(f\"Default S3 bucket: {bucket}\")"
52+
"print(f\"Default S3 bucket: {bucket}\")\n",
53+
"print(f\"Default S3 bucket prefix: {default_bucket_prefix}\")"
5154
]
5255
},
5356
{
@@ -152,11 +155,15 @@
152155
"# Upload each dataset to S3\n",
153156
"# If a default bucket prefix is specified, append it to the s3 path\n",
154157
"if default_bucket_prefix:\n",
155-
" s3_train_input = sagemaker_session.upload_data(\"train.csv\", key_prefix=default_bucket_prefix)\n",
158+
" s3_train_input = sagemaker_session.upload_data(\n",
159+
" \"train.csv\", bucket, key_prefix=default_bucket_prefix\n",
160+
" )\n",
156161
" s3_validation_input = sagemaker_session.upload_data(\n",
157-
" \"validation.csv\", key_prefix=default_bucket_prefix\n",
162+
" \"validation.csv\", bucket, key_prefix=default_bucket_prefix\n",
163+
" )\n",
164+
" s3_test_input = sagemaker_session.upload_data(\n",
165+
" \"test.csv\", bucket, key_prefix=default_bucket_prefix\n",
158166
" )\n",
159-
" s3_test_input = sagemaker_session.upload_data(\"test.csv\", key_prefix=default_bucket_prefix)\n",
160167
"else:\n",
161168
" s3_train_input = sagemaker_session.upload_data(\"train.csv\", bucket)\n",
162169
" s3_validation_input = sagemaker_session.upload_data(\"validation.csv\", bucket)\n",

‎ sagemaker-core/inference_and_resource_chaining.ipynb

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,8 @@
135135
"metadata": {},
136136
"outputs": [],
137137
"source": [
138+
"import sagemaker\n",
139+
"\n",
138140
"from sagemaker_core.helper.session_helper import get_execution_role, Session\n",
139141
"from rich import print\n",
140142
"\n",
@@ -143,8 +145,8 @@
143145
"sagemaker_session = Session()\n",
144146
"region = sagemaker_session.boto_region_name\n",
145147
"role = get_execution_role()\n",
146-
"bucket = sagemaker_session.default_bucket()\n",
147-
"default_bucket_prefix = sagemaker_session.default_bucket_prefix\n",
148+
"bucket = sagemaker.Session().default_bucket()\n",
149+
"default_bucket_prefix = sagemaker.Session().default_bucket_prefix\n",
148150
"print(role)"
149151
]
150152
},

‎ sagemaker-core/intelligent_defaults_and_logging.ipynb

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,8 @@
167167
"metadata": {},
168168
"outputs": [],
169169
"source": [
170+
"import sagemaker\n",
171+
"\n",
170172
"from sagemaker_core.helper.session_helper import Session, get_execution_role\n",
171173
"from rich import print\n",
172174
"\n",
@@ -175,8 +177,8 @@
175177
"sagemaker_session = Session()\n",
176178
"region = sagemaker_session.boto_region_name\n",
177179
"role = get_execution_role()\n",
178-
"bucket = sagemaker_session.default_bucket()\n",
179-
"default_bucket_prefix = sagemaker_session.default_bucket_prefix\n",
180+
"bucket = sagemaker.Session().default_bucket()\n",
181+
"default_bucket_prefix = sagemaker.Session().default_bucket_prefix\n",
180182
"print(role)"
181183
]
182184
},

‎ sagemaker-core/sagemaker-core-feature-store/sagemaker_core_feature_store_introduction.ipynb

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,14 +63,15 @@
6363
"import pandas as pd\n",
6464
"import numpy as np\n",
6565
"import io\n",
66+
"import sagemaker\n",
6667
"from sagemaker_core.helper.session_helper import get_execution_role, Session\n",
6768
"\n",
6869
"sagemaker_session = Session()\n",
6970
"REGION_NAME = sagemaker_session._region_name\n",
7071
"role = get_execution_role()\n",
71-
"s3_bucket_name = sagemaker_session.default_bucket()\n",
72+
"s3_bucket_name = sagemaker.Session().default_bucket()\n",
7273
"prefix = \"sagemaker-featurestore-introduction\"\n",
73-
"default_bucket_prefix = sagemaker_session.default_bucket_prefix\n",
74+
"default_bucket_prefix = sagemaker.Session().default_bucket_prefix\n",
7475
"\n",
7576
"# If a default bucket prefix is specified, append it to the s3 path\n",
7677
"if default_bucket_prefix:\n",

‎ sagemaker-core/sagemaker-core-llama-3-8B-speculative-decoding.ipynb

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
},
4444
"outputs": [],
4545
"source": [
46+
"import sagemaker\n",
4647
"import json\n",
4748
"import os\n",
4849
"\n",
@@ -76,8 +77,8 @@
7677
" f\"763104351884.dkr.ecr.{REGION}.amazonaws.com/djl-inference:0.29.0-tensorrtllm0.11.0-cu124\"\n",
7778
")\n",
7879
"\n",
79-
"default_bucket = SM_SESSION.default_bucket()\n",
80-
"default_bucket_prefix = SM_SESSION.default_bucket_prefix\n",
80+
"default_bucket = sagemaker.Session().default_bucket()\n",
81+
"default_bucket_prefix = sagemaker.Session().default_bucket_prefix\n",
8182
"\n",
8283
"TARGET_MODEL = \"meta-llama/Meta-Llama-3-8B\"\n",
8384
"DRAFT_MODEL = \"sagemaker\""

‎ sagemaker-core/sagemaker-core-llama-3-8B.ipynb

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
"source": [
4343
"import json\n",
4444
"import os\n",
45+
"import sagemaker\n",
4546
"\n",
4647
"from sagemaker_core.helper.session_helper import get_execution_role, Session\n",
4748
"import pathlib\n",
@@ -71,8 +72,8 @@
7172
" f\"763104351884.dkr.ecr.{REGION}.amazonaws.com/djl-inference:0.29.0-tensorrtllm0.11.0-cu124\"\n",
7273
")\n",
7374
"\n",
74-
"default_bucket = SM_SESSION.default_bucket()\n",
75-
"default_bucket_prefix = SM_SESSION.default_bucket_prefix\n",
75+
"default_bucket = sagemaker.Session().default_bucket()\n",
76+
"default_bucket_prefix = sagemaker.Session().default_bucket_prefix\n",
7677
"\n",
7778
"MODEL = \"meta-llama/Meta-Llama-3-8B\""
7879
]

‎ sagemaker-core/sagemaker-core-pyspark-processing.ipynb

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@
9191
"outputs": [],
9292
"source": [
9393
"import logging\n",
94+
"import sagemaker\n",
9495
"from time import gmtime, strftime\n",
9596
"from sagemaker_core.helper.session_helper import get_execution_role, Session\n",
9697
"\n",
@@ -101,8 +102,8 @@
101102
"sagemaker_session = Session()\n",
102103
"REGION_NAME = sagemaker_session._region_name\n",
103104
"role = get_execution_role()\n",
104-
"s3_bucket_name = sagemaker_session.default_bucket()\n",
105-
"default_bucket_prefix = sagemaker_session.default_bucket_prefix\n",
105+
"s3_bucket_name = sagemaker.Session().default_bucket()\n",
106+
"default_bucket_prefix = sagemaker.Session().default_bucket_prefix\n",
106107
"default_bucket_prefix_path = \"\"\n",
107108
"\n",
108109
"# If a default bucket prefix is specified, append it to the s3 path\n",

‎ sagemaker-core/sagemaker_core_overview.ipynb

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,8 @@
153153
"metadata": {},
154154
"outputs": [],
155155
"source": [
156+
"import sagemaker\n",
157+
"\n",
156158
"from sagemaker_core.helper.session_helper import Session, get_execution_role\n",
157159
"from rich import print\n",
158160
"\n",
@@ -161,8 +163,8 @@
161163
"sagemaker_session = Session()\n",
162164
"region = sagemaker_session.boto_region_name\n",
163165
"role = get_execution_role()\n",
164-
"bucket = sagemaker_session.default_bucket()\n",
165-
"default_bucket_prefix = sagemaker_session.default_bucket_prefix\n",
166+
"bucket = sagemaker.Session().default_bucket()\n",
167+
"default_bucket_prefix = sagemaker.Session().default_bucket_prefix\n",
166168
"print(role)"
167169
]
168170
},

‎ prepare_data/sm-ground_truth_chained_streaming_labeling_job.ipynb

Lines changed: 38 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,14 @@
178178
"\n",
179179
"if BUCKET == \"<< YOUR S3 BUCKET NAME >>\":\n",
180180
" BUCKET = sess.default_bucket()\n",
181+
"\n",
182+
"default_bucket_prefix = sess.default_bucket_prefix\n",
183+
"default_bucket_prefix_path = \"\"\n",
184+
"\n",
185+
"# If a default bucket prefix is specified, append it to the s3 path\n",
186+
"if default_bucket_prefix:\n",
187+
" default_bucket_prefix_path = f\"{default_bucket_prefix}/\"\n",
188+
"\n",
181189
"region = boto3.session.Session().region_name\n",
182190
"bucket_region = s3.head_bucket(Bucket=BUCKET)[\"ResponseMetadata\"][\"HTTPHeaders\"][\n",
183191
" \"x-amz-bucket-region\"\n",
@@ -449,7 +457,7 @@
449457
"metadata": {},
450458
"outputs": [],
451459
"source": [
452-
"s3.upload_file(\"class_labels.json\", BUCKET, \"class_labels.json\")"
460+
"s3.upload_file(\"class_labels.json\", BUCKET, f\"{default_bucket_prefix_path}class_labels.json\")"
453461
]
454462
},
455463
{
@@ -458,7 +466,7 @@
458466
"metadata": {},
459467
"outputs": [],
460468
"source": [
461-
"LABEL_CATEGORIES_S3_URI = f\"s3://{BUCKET}/class_labels.json\"\n",
469+
"LABEL_CATEGORIES_S3_URI = f\"s3://{BUCKET}/{default_bucket_prefix_path}class_labels.json\"\n",
462470
"print(f\"You should now see class_labels.json in {LABEL_CATEGORIES_S3_URI}\")"
463471
]
464472
},
@@ -547,7 +555,9 @@
547555
"outputs": [],
548556
"source": [
549557
"if DEFAULT:\n",
550-
" result = s3.upload_file(\"instructions.template\", BUCKET, \"instructions.template\")"
558+
" result = s3.upload_file(\n",
559+
" \"instructions.template\", BUCKET, f\"{default_bucket_prefix_path}instructions.template\"\n",
560+
" )"
551561
]
552562
},
553563
{
@@ -593,7 +603,9 @@
593603
"# does not use video frames or 3D point clouds.\n",
594604
"if not DEFAULT and not video_frame_task and not point_cloud_task:\n",
595605
" make_template(save_fname=\"instructions.html\")\n",
596-
" s3.upload_file(\"instructions.template\", BUCKET, \"instructions.template\")"
606+
" s3.upload_file(\n",
607+
" \"instructions.template\", BUCKET, f\"{default_bucket_prefix_path}instructions.template\"\n",
608+
" )"
597609
]
598610
},
599611
{
@@ -695,7 +707,7 @@
695707
" HUMAN_UI.append(HUMAN_TASK_UI_ARN)\n",
696708
" UI_CONFIG_PARAM = \"HumanTaskUiArn\"\n",
697709
"else:\n",
698-
" UI_TEMPLATE_S3_URI = f\"s3://{BUCKET}/instructions.template\"\n",
710+
" UI_TEMPLATE_S3_URI = f\"s3://{BUCKET}/{default_bucket_prefix_path}instructions.template\"\n",
699711
" HUMAN_UI.append(UI_TEMPLATE_S3_URI)\n",
700712
" UI_CONFIG_PARAM = \"UiTemplateS3Uri\"\n",
701713
"\n",
@@ -716,7 +728,7 @@
716728
"\n",
717729
"# If you want to store your output manifest in a different folder, provide an OUTPUT_PATH.\n",
718730
"OUTPUT_FOLDER_PREFIX = \"/gt-streaming-demo-output\"\n",
719-
"OUTPUT_BUCKET = \"s3://\" + BUCKET + OUTPUT_FOLDER_PREFIX\n",
731+
"OUTPUT_BUCKET = \"s3://\" + BUCKET + f\"{default_bucket_prefix_path}{OUTPUT_FOLDER_PREFIX}\"\n",
720732
"print(\"Your output data will be stored in:\", OUTPUT_BUCKET)\n",
721733
"\n",
722734
"# An IAM role with AmazonGroundTruthExecution policies attached.\n",
@@ -1183,7 +1195,7 @@
11831195
"metadata": {},
11841196
"outputs": [],
11851197
"source": [
1186-
"s3.upload_file(\"class_labels2.json\", BUCKET, \"class_labels2.json\")"
1198+
"s3.upload_file(\"class_labels2.json\", BUCKET, \"{default_bucket_prefix_path}class_labels2.json\")"
11871199
]
11881200
},
11891201
{
@@ -1192,7 +1204,7 @@
11921204
"metadata": {},
11931205
"outputs": [],
11941206
"source": [
1195-
"LABEL_CATEGORIES_S3_URI2 = f\"s3://{BUCKET}/class_labels2.json\"\n",
1207+
"LABEL_CATEGORIES_S3_URI2 = f\"s3://{BUCKET}/{default_bucket_prefix_path}class_labels2.json\"\n",
11961208
"print(f\"You should now see class_labels2.json in {LABEL_CATEGORIES_S3_URI2}\")"
11971209
]
11981210
},
@@ -1296,7 +1308,9 @@
12961308
"outputs": [],
12971309
"source": [
12981310
"if DEFAULT:\n",
1299-
" result = s3.upload_file(\"instructions2.template\", BUCKET, \"instructions2.template\")"
1311+
" result = s3.upload_file(\n",
1312+
" \"instructions2.template\", BUCKET, f\"{default_bucket_prefix_path}instructions2.template\"\n",
1313+
" )"
13001314
]
13011315
},
13021316
{
@@ -1369,7 +1383,9 @@
13691383
"# does not use video frames or 3D point clouds.\n",
13701384
"if not DEFAULT and not video_frame_task and not point_cloud_task:\n",
13711385
" make_template(save_fname=\"instructions2.template\")\n",
1372-
" s3.upload_file(\"instructions2.template\", BUCKET, \"instructions2.template\")"
1386+
" s3.upload_file(\n",
1387+
" \"instructions2.template\", BUCKET, f\"{default_bucket_prefix_path}instructions2.template\"\n",
1388+
" )"
13731389
]
13741390
},
13751391
{
@@ -1476,7 +1492,7 @@
14761492
" HUMAN_UI2.append(HUMAN_TASK_UI_ARN2)\n",
14771493
" UI_CONFIG_PARAM = \"HumanTaskUiArn\"\n",
14781494
"else:\n",
1479-
" UI_TEMPLATE_S3_URI2 = f\"s3://{BUCKET}/instructions2.template\"\n",
1495+
" UI_TEMPLATE_S3_URI2 = f\"s3://{BUCKET}/{default_bucket_prefix_path}instructions2.template\"\n",
14801496
" HUMAN_UI2.append(UI_TEMPLATE_S3_URI2)\n",
14811497
" UI_CONFIG_PARAM = \"UiTemplateS3Uri\"\n",
14821498
"\n",
@@ -1491,8 +1507,13 @@
14911507
"source": [
14921508
"# If you want to store your output manifest in a different folder, provide an OUTPUT_PATH.\n",
14931509
"OUTPUT_FOLDER_PREFIX = \"/gt-streaming-demo-output\"\n",
1494-
"OUTPUT_BUCKET = \"s3://\" + BUCKET + OUTPUT_FOLDER_PREFIX\n",
1495-
"print(\"Your output data will be stored in:\", OUTPUT_BUCKET)\n",
1510+
"\n",
1511+
"# If a default bucket prefix is specified, append it to the s3 path\n",
1512+
"if default_bucket_prefix:\n",
1513+
" OUTPUT_FOLDER_PREFIX = f\"/{default_bucket_prefix}{OUTPUT_FOLDER_PREFIX}\"\n",
1514+
"\n",
1515+
"OUTPUT_BUCKET = \"s3://\" + BUCKET + f\"{OUTPUT_FOLDER_PREFIX}\"\n",
1516+
"print(\"Your output data will be stored in:\", f\"{OUTPUT_FOLDER_PREFIX}\")\n",
14961517
"\n",
14971518
"# An IAM role with AmazonGroundTruthExecution policies attached.\n",
14981519
"# This must be the same role that you used to create this notebook instance.\n",
@@ -1670,11 +1691,11 @@
16701691
"metadata": {},
16711692
"outputs": [],
16721693
"source": [
1673-
"if(DEFAULT): \n",
1694+
"if DEFAULT:\n",
16741695
" !wget https://aws-ml-blog.s3.amazonaws.com/artifacts/gt-labeling-job-resources/example-image.jpg\n",
1675-
" s3.upload_file('example-image.jpg', BUCKET, 'example-image.jpg')\n",
1676-
" REQUEST = str({\"source-ref\": f\"s3://{BUCKET}/example-image.jpg\"})\n",
1677-
"print(f'Your request: {REQUEST}')"
1696+
" s3.upload_file(\"example-image.jpg\", BUCKET, f\"{default_bucket_prefix_path}example-image.jpg\")\n",
1697+
" REQUEST = str({\"source-ref\": f\"s3://{BUCKET}/{default_bucket_prefix_path}example-image.jpg\"})\n",
1698+
"print(f\"Your request: {REQUEST}\")"
16781699
]
16791700
},
16801701
{

0 commit comments

Comments
 (0)