Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: add filter and timestamp splits #549

Closed
wants to merge 16 commits into from
Closed

feat: add filter and timestamp splits #549

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
5a72c86
initial commit
sirtorry Jul 16, 2021
0a5971e
add tests
sirtorry Jul 16, 2021
4875cd1
fix arrays
sirtorry Jul 16, 2021
9315fea
fix tests
sirtorry Jul 16, 2021
b04dca6
fix test
sirtorry Jul 17, 2021
5b78ee6
change default split behaviour
sirtorry Jul 23, 2021
c0b38f8
modify tests
sirtorry Jul 23, 2021
99aa367
fix tests
sirtorry Jul 27, 2021
2ef9cbd
tests for multi-split
sirtorry Jul 27, 2021
5f8a7b1
split counter bug fix
sirtorry Jul 27, 2021
d43ef98
fixes multi split issue
sirtorry Jul 27, 2021
a45d837
bug fixes
sirtorry Jul 28, 2021
3577e9e
bug fix
sirtorry Jul 28, 2021
864d0e2
fix non-sync tests
sirtorry Jul 29, 2021
e31531f
change yield to return
sirtorry Jul 29, 2021
20ea593
fix tabular test
sirtorry Jul 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
851 changes: 691 additions & 160 deletions google/cloud/aiplatform/training_jobs.py
View file
Open in desktop

Large diffs are not rendered by default.

22 changes: 11 additions & 11 deletions tests/unit/aiplatform/test_automl_image_training_jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@
_TEST_FRACTION_SPLIT_TRAINING = 0.6
_TEST_FRACTION_SPLIT_VALIDATION = 0.2
_TEST_FRACTION_SPLIT_TEST = 0.2
_TEST_FILTER_SPLIT_TRAINING = "train"
_TEST_FILTER_SPLIT_VALIDATION = "validate"
_TEST_FILTER_SPLIT_TEST = "test"

_TEST_MODEL_NAME = (
f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/models/{_TEST_MODEL_ID}"
Expand Down Expand Up @@ -260,6 +263,9 @@ def test_run_call_pipeline_service_create(
training_fraction_split=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction_split=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction_split=_TEST_FRACTION_SPLIT_TEST,
training_filter_split=_TEST_FILTER_SPLIT_TRAINING,
validation_filter_split=_TEST_FILTER_SPLIT_VALIDATION,
test_filter_split=_TEST_FILTER_SPLIT_TEST,
budget_milli_node_hours=_TEST_TRAINING_BUDGET_MILLI_NODE_HOURS,
disable_early_stopping=_TEST_TRAINING_DISABLE_EARLY_STOPPING,
sync=sync,
Expand All @@ -268,10 +274,10 @@ def test_run_call_pipeline_service_create(
if not sync:
model_from_job.wait()

true_fraction_split = gca_training_pipeline.FractionSplit(
training_fraction=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction=_TEST_FRACTION_SPLIT_TEST,
true_filter_split = gca_training_pipeline.FilterSplit(
training_filter=_TEST_FILTER_SPLIT_TRAINING,
validation_filter=_TEST_FILTER_SPLIT_VALIDATION,
test_filter=_TEST_FILTER_SPLIT_TEST,
)

true_managed_model = gca_model.Model(
Expand All @@ -281,7 +287,7 @@ def test_run_call_pipeline_service_create(
)

true_input_data_config = gca_training_pipeline.InputDataConfig(
fraction_split=true_fraction_split, dataset_id=mock_dataset_image.name,
filter_split=true_filter_split, dataset_id=mock_dataset_image.name,
)

true_training_pipeline = gca_training_pipeline.TrainingPipeline(
Expand Down Expand Up @@ -377,9 +383,6 @@ def test_run_called_twice_raises(self, mock_dataset_image, sync):
job.run(
dataset=mock_dataset_image,
model_display_name=_TEST_MODEL_DISPLAY_NAME,
training_fraction_split=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction_split=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction_split=_TEST_FRACTION_SPLIT_TEST,
disable_early_stopping=_TEST_TRAINING_DISABLE_EARLY_STOPPING,
sync=sync,
)
Expand All @@ -388,9 +391,6 @@ def test_run_called_twice_raises(self, mock_dataset_image, sync):
job.run(
dataset=mock_dataset_image,
model_display_name=_TEST_MODEL_DISPLAY_NAME,
training_fraction_split=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction_split=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction_split=_TEST_FRACTION_SPLIT_TEST,
sync=sync,
)

Expand Down
53 changes: 11 additions & 42 deletions tests/unit/aiplatform/test_automl_tabular_training_jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@
_TEST_VALIDATION_FRACTION_SPLIT = 0.2
_TEST_TEST_FRACTION_SPLIT = 0.2
_TEST_PREDEFINED_SPLIT_COLUMN_NAME = "split"
_TEST_TIMESTAMP_SPLIT_COLUMN_NAME = "timestamp"

_TEST_OUTPUT_PYTHON_PACKAGE_PATH = "gs://test/ouput/python/trainer.tar.gz"

Expand Down Expand Up @@ -299,6 +300,7 @@ def test_run_call_pipeline_service_create(
validation_fraction_split=_TEST_VALIDATION_FRACTION_SPLIT,
test_fraction_split=_TEST_TEST_FRACTION_SPLIT,
predefined_split_column_name=_TEST_PREDEFINED_SPLIT_COLUMN_NAME,
timestamp_split_column_name=_TEST_TIMESTAMP_SPLIT_COLUMN_NAME,
weight_column=_TEST_TRAINING_WEIGHT_COLUMN,
budget_milli_node_hours=_TEST_TRAINING_BUDGET_MILLI_NODE_HOURS,
disable_early_stopping=_TEST_TRAINING_DISABLE_EARLY_STOPPING,
Expand All @@ -308,10 +310,11 @@ def test_run_call_pipeline_service_create(
if not sync:
model_from_job.wait()

true_fraction_split = gca_training_pipeline.FractionSplit(
true_timestamp_split = gca_training_pipeline.TimestampSplit(
training_fraction=_TEST_TRAINING_FRACTION_SPLIT,
validation_fraction=_TEST_VALIDATION_FRACTION_SPLIT,
test_fraction=_TEST_TEST_FRACTION_SPLIT,
key=_TEST_TIMESTAMP_SPLIT_COLUMN_NAME,
)

true_managed_model = gca_model.Model(
Expand All @@ -320,10 +323,10 @@ def test_run_call_pipeline_service_create(
)

true_input_data_config = gca_training_pipeline.InputDataConfig(
fraction_split=true_fraction_split,
predefined_split=gca_training_pipeline.PredefinedSplit(
key=_TEST_PREDEFINED_SPLIT_COLUMN_NAME
),
timestamp_split=true_timestamp_split,
dataset_id=mock_dataset_tabular.name,
)

Expand Down Expand Up @@ -378,9 +381,6 @@ def test_run_call_pipeline_if_no_model_display_name(
model_from_job = job.run(
dataset=mock_dataset_tabular,
target_column=_TEST_TRAINING_TARGET_COLUMN,
training_fraction_split=_TEST_TRAINING_FRACTION_SPLIT,
validation_fraction_split=_TEST_VALIDATION_FRACTION_SPLIT,
test_fraction_split=_TEST_TEST_FRACTION_SPLIT,
weight_column=_TEST_TRAINING_WEIGHT_COLUMN,
budget_milli_node_hours=_TEST_TRAINING_BUDGET_MILLI_NODE_HOURS,
disable_early_stopping=_TEST_TRAINING_DISABLE_EARLY_STOPPING,
Expand Down Expand Up @@ -450,7 +450,6 @@ def test_run_call_pipeline_service_create_if_no_column_transformations(
training_fraction_split=_TEST_TRAINING_FRACTION_SPLIT,
validation_fraction_split=_TEST_VALIDATION_FRACTION_SPLIT,
test_fraction_split=_TEST_TEST_FRACTION_SPLIT,
predefined_split_column_name=_TEST_PREDEFINED_SPLIT_COLUMN_NAME,
weight_column=_TEST_TRAINING_WEIGHT_COLUMN,
budget_milli_node_hours=_TEST_TRAINING_BUDGET_MILLI_NODE_HOURS,
disable_early_stopping=_TEST_TRAINING_DISABLE_EARLY_STOPPING,
Expand All @@ -472,11 +471,7 @@ def test_run_call_pipeline_service_create_if_no_column_transformations(
)

true_input_data_config = gca_training_pipeline.InputDataConfig(
fraction_split=true_fraction_split,
predefined_split=gca_training_pipeline.PredefinedSplit(
key=_TEST_PREDEFINED_SPLIT_COLUMN_NAME
),
dataset_id=mock_dataset_tabular.name,
fraction_split=true_fraction_split, dataset_id=mock_dataset_tabular.name,
)

true_training_pipeline = gca_training_pipeline.TrainingPipeline(
Expand Down Expand Up @@ -524,10 +519,6 @@ def test_run_call_pipeline_service_create_if_set_additional_experiments(
dataset=mock_dataset_tabular,
target_column=_TEST_TRAINING_TARGET_COLUMN,
model_display_name=_TEST_MODEL_DISPLAY_NAME,
training_fraction_split=_TEST_TRAINING_FRACTION_SPLIT,
validation_fraction_split=_TEST_VALIDATION_FRACTION_SPLIT,
test_fraction_split=_TEST_TEST_FRACTION_SPLIT,
predefined_split_column_name=_TEST_PREDEFINED_SPLIT_COLUMN_NAME,
weight_column=_TEST_TRAINING_WEIGHT_COLUMN,
budget_milli_node_hours=_TEST_TRAINING_BUDGET_MILLI_NODE_HOURS,
disable_early_stopping=_TEST_TRAINING_DISABLE_EARLY_STOPPING,
Expand All @@ -549,11 +540,7 @@ def test_run_call_pipeline_service_create_if_set_additional_experiments(
)

true_input_data_config = gca_training_pipeline.InputDataConfig(
fraction_split=true_fraction_split,
predefined_split=gca_training_pipeline.PredefinedSplit(
key=_TEST_PREDEFINED_SPLIT_COLUMN_NAME
),
dataset_id=mock_dataset_tabular.name,
fraction_split=true_fraction_split, dataset_id=mock_dataset_tabular.name,
)

true_training_pipeline = gca_training_pipeline.TrainingPipeline(
Expand Down Expand Up @@ -601,9 +588,6 @@ def test_run_call_pipeline_service_create_with_column_specs(
dataset=mock_dataset_tabular_alternative,
target_column=_TEST_TRAINING_TARGET_COLUMN,
model_display_name=_TEST_MODEL_DISPLAY_NAME,
training_fraction_split=_TEST_TRAINING_FRACTION_SPLIT,
validation_fraction_split=_TEST_VALIDATION_FRACTION_SPLIT,
test_fraction_split=_TEST_TEST_FRACTION_SPLIT,
predefined_split_column_name=_TEST_PREDEFINED_SPLIT_COLUMN_NAME,
weight_column=_TEST_TRAINING_WEIGHT_COLUMN,
budget_milli_node_hours=_TEST_TRAINING_BUDGET_MILLI_NODE_HOURS,
Expand All @@ -614,16 +598,9 @@ def test_run_call_pipeline_service_create_with_column_specs(
if not sync:
model_from_job.wait()

true_fraction_split = gca_training_pipeline.FractionSplit(
training_fraction=_TEST_TRAINING_FRACTION_SPLIT,
validation_fraction=_TEST_VALIDATION_FRACTION_SPLIT,
test_fraction=_TEST_TEST_FRACTION_SPLIT,
)

true_managed_model = gca_model.Model(display_name=_TEST_MODEL_DISPLAY_NAME)

true_input_data_config = gca_training_pipeline.InputDataConfig(
fraction_split=true_fraction_split,
predefined_split=gca_training_pipeline.PredefinedSplit(
key=_TEST_PREDEFINED_SPLIT_COLUMN_NAME
),
Expand Down Expand Up @@ -716,7 +693,7 @@ def test_run_call_pipeline_service_create_with_column_specs_not_auto(
training_fraction_split=_TEST_TRAINING_FRACTION_SPLIT,
validation_fraction_split=_TEST_VALIDATION_FRACTION_SPLIT,
test_fraction_split=_TEST_TEST_FRACTION_SPLIT,
predefined_split_column_name=_TEST_PREDEFINED_SPLIT_COLUMN_NAME,
timestamp_split_column_name=_TEST_TIMESTAMP_SPLIT_COLUMN_NAME,
weight_column=_TEST_TRAINING_WEIGHT_COLUMN,
budget_milli_node_hours=_TEST_TRAINING_BUDGET_MILLI_NODE_HOURS,
disable_early_stopping=_TEST_TRAINING_DISABLE_EARLY_STOPPING,
Expand All @@ -726,19 +703,17 @@ def test_run_call_pipeline_service_create_with_column_specs_not_auto(
if not sync:
model_from_job.wait()

true_fraction_split = gca_training_pipeline.FractionSplit(
true_timestamp_split = gca_training_pipeline.TimestampSplit(
training_fraction=_TEST_TRAINING_FRACTION_SPLIT,
validation_fraction=_TEST_VALIDATION_FRACTION_SPLIT,
test_fraction=_TEST_TEST_FRACTION_SPLIT,
key=_TEST_TIMESTAMP_SPLIT_COLUMN_NAME,
)

true_managed_model = gca_model.Model(display_name=_TEST_MODEL_DISPLAY_NAME)

true_input_data_config = gca_training_pipeline.InputDataConfig(
fraction_split=true_fraction_split,
predefined_split=gca_training_pipeline.PredefinedSplit(
key=_TEST_PREDEFINED_SPLIT_COLUMN_NAME
),
timestamp_split=true_timestamp_split,
dataset_id=mock_dataset_tabular_alternative.name,
)

Expand Down Expand Up @@ -778,9 +753,6 @@ def test_run_called_twice_raises(self, mock_dataset_tabular, sync):
dataset=mock_dataset_tabular,
target_column=_TEST_TRAINING_TARGET_COLUMN,
model_display_name=_TEST_MODEL_DISPLAY_NAME,
training_fraction_split=_TEST_TRAINING_FRACTION_SPLIT,
validation_fraction_split=_TEST_VALIDATION_FRACTION_SPLIT,
test_fraction_split=_TEST_TEST_FRACTION_SPLIT,
sync=sync,
)

Expand All @@ -789,9 +761,6 @@ def test_run_called_twice_raises(self, mock_dataset_tabular, sync):
dataset=mock_dataset_tabular,
target_column=_TEST_TRAINING_TARGET_COLUMN,
model_display_name=_TEST_MODEL_DISPLAY_NAME,
training_fraction_split=_TEST_TRAINING_FRACTION_SPLIT,
validation_fraction_split=_TEST_VALIDATION_FRACTION_SPLIT,
test_fraction_split=_TEST_TEST_FRACTION_SPLIT,
sync=sync,
)

Expand Down
44 changes: 19 additions & 25 deletions tests/unit/aiplatform/test_automl_text_training_jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@
_TEST_FRACTION_SPLIT_TRAINING = 0.6
_TEST_FRACTION_SPLIT_VALIDATION = 0.2
_TEST_FRACTION_SPLIT_TEST = 0.2
_TEST_FILTER_SPLIT_TRAINING = "train"
_TEST_FILTER_SPLIT_VALIDATION = "validate"
_TEST_FILTER_SPLIT_TEST = "test"

_TEST_MODEL_NAME = (
f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/models/{_TEST_MODEL_ID}"
Expand Down Expand Up @@ -266,9 +269,6 @@ def test_init_aiplatform_with_encryption_key_name_and_create_training_job(
model_from_job = job.run(
dataset=mock_dataset_text,
model_display_name=_TEST_MODEL_DISPLAY_NAME,
training_fraction_split=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction_split=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction_split=_TEST_FRACTION_SPLIT_TEST,
sync=sync,
)

Expand Down Expand Up @@ -331,16 +331,19 @@ def test_run_call_pipeline_service_create_classification(
training_fraction_split=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction_split=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction_split=_TEST_FRACTION_SPLIT_TEST,
training_filter_split=_TEST_FILTER_SPLIT_TRAINING,
validation_filter_split=_TEST_FILTER_SPLIT_VALIDATION,
test_filter_split=_TEST_FILTER_SPLIT_TEST,
sync=sync,
)

if not sync:
model_from_job.wait()

true_fraction_split = gca_training_pipeline.FractionSplit(
training_fraction=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction=_TEST_FRACTION_SPLIT_TEST,
true_filter_split = gca_training_pipeline.FilterSplit(
training_filter=_TEST_FILTER_SPLIT_TRAINING,
validation_filter=_TEST_FILTER_SPLIT_VALIDATION,
test_filter=_TEST_FILTER_SPLIT_TEST,
)

true_managed_model = gca_model.Model(
Expand All @@ -349,7 +352,7 @@ def test_run_call_pipeline_service_create_classification(
)

true_input_data_config = gca_training_pipeline.InputDataConfig(
fraction_split=true_fraction_split, dataset_id=mock_dataset_text.name,
filter_split=true_filter_split, dataset_id=mock_dataset_text.name,
)

true_training_pipeline = gca_training_pipeline.TrainingPipeline(
Expand Down Expand Up @@ -457,25 +460,25 @@ def test_run_call_pipeline_service_create_sentiment(
model_from_job = job.run(
dataset=mock_dataset_text,
model_display_name=_TEST_MODEL_DISPLAY_NAME,
training_fraction_split=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction_split=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction_split=_TEST_FRACTION_SPLIT_TEST,
training_filter_split=_TEST_FILTER_SPLIT_TRAINING,
validation_filter_split=_TEST_FILTER_SPLIT_VALIDATION,
test_filter_split=_TEST_FILTER_SPLIT_TEST,
sync=sync,
)

if not sync:
model_from_job.wait()

true_fraction_split = gca_training_pipeline.FractionSplit(
training_fraction=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction=_TEST_FRACTION_SPLIT_TEST,
true_filter_split = gca_training_pipeline.FilterSplit(
training_filter=_TEST_FILTER_SPLIT_TRAINING,
validation_filter=_TEST_FILTER_SPLIT_VALIDATION,
test_filter=_TEST_FILTER_SPLIT_TEST,
)

true_managed_model = gca_model.Model(display_name=_TEST_MODEL_DISPLAY_NAME)

true_input_data_config = gca_training_pipeline.InputDataConfig(
fraction_split=true_fraction_split, dataset_id=mock_dataset_text.name,
filter_split=true_filter_split, dataset_id=mock_dataset_text.name,
)

true_training_pipeline = gca_training_pipeline.TrainingPipeline(
Expand Down Expand Up @@ -518,9 +521,6 @@ def test_run_call_pipeline_if_no_model_display_name(

model_from_job = job.run(
dataset=mock_dataset_text,
training_fraction_split=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction_split=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction_split=_TEST_FRACTION_SPLIT_TEST,
model_display_name=None, # Omit model_display_name
sync=sync,
)
Expand Down Expand Up @@ -572,19 +572,13 @@ def test_run_called_twice_raises(self, mock_dataset_text, sync):
job.run(
dataset=mock_dataset_text,
model_display_name=_TEST_MODEL_DISPLAY_NAME,
training_fraction_split=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction_split=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction_split=_TEST_FRACTION_SPLIT_TEST,
sync=sync,
)

with pytest.raises(RuntimeError):
job.run(
dataset=mock_dataset_text,
model_display_name=_TEST_MODEL_DISPLAY_NAME,
training_fraction_split=_TEST_FRACTION_SPLIT_TRAINING,
validation_fraction_split=_TEST_FRACTION_SPLIT_VALIDATION,
test_fraction_split=_TEST_FRACTION_SPLIT_TEST,
sync=sync,
)

Expand Down