diff --git a/automl_zero/BUILD b/automl_zero/BUILD index 1a060eebd5b..7247c2419c5 100644 --- a/automl_zero/BUILD +++ b/automl_zero/BUILD @@ -43,6 +43,7 @@ cc_library( deps = [ "datasets_cc_proto", ":definitions", + "@com_google_googletest//:gtest_prod", ], ) @@ -71,6 +72,7 @@ cc_library( ":memory", ":random_generator", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", @@ -109,6 +111,7 @@ cc_library( ":instruction", ":random_generator", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", ], ) @@ -151,6 +154,7 @@ cc_library( deps = [ ":instruction_cc_proto", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", "@com_google_glog//:glog", "@com_google_protobuf//:protobuf", "@eigen_archive//:eigen", @@ -183,6 +187,7 @@ cc_library( ":train_budget", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", @@ -223,6 +228,7 @@ cc_library( ":instruction_cc_proto", ":memory", ":random_generator", + "@com_google_googletest//:gtest_prod", ], ) @@ -455,6 +461,7 @@ cc_library( deps = [ ":definitions", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", ], ) @@ -481,6 +488,7 @@ cc_library( ":random_generator", ":randomizer", "@com_google_absl//absl/memory", + "@com_google_googletest//:gtest_prod", ], ) @@ -578,8 +586,10 @@ cc_library( ":mutator", ":random_generator", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", "@com_google_absl//absl/memory", "@com_google_absl//absl/time", + "@com_google_googletest//:gtest_prod", ], ) @@ -600,6 +610,7 @@ cc_test( ":regularized_evolution", ":test_util", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", "@com_google_absl//absl/memory", "@com_google_absl//absl/time", "@com_google_googletest//:gtest_main", @@ -625,6 +636,7 @@ cc_binary( ":regularized_evolution", ":train_budget", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", "@com_google_absl//absl/time", "@com_google_protobuf//:protobuf", ], diff --git a/automl_zero/algorithm_test_util.h b/automl_zero/algorithm_test_util.h index 59aafb80fe7..e8b3f70a184 100644 --- a/automl_zero/algorithm_test_util.h +++ b/automl_zero/algorithm_test_util.h @@ -19,7 +19,7 @@ #include "algorithm.h" #include "algorithm.pb.h" -#include "testing/base/public/gmock.h" +#include "gmock/gmock.h" namespace automl_zero { diff --git a/automl_zero/definitions_test.cc b/automl_zero/definitions_test.cc index 1a10269d505..ccf8fcb9572 100644 --- a/automl_zero/definitions_test.cc +++ b/automl_zero/definitions_test.cc @@ -19,7 +19,7 @@ #include #include "testing/base/public/benchmark.h" -#include "testing/base/public/gmock.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" namespace automl_zero { diff --git a/automl_zero/executor.h b/automl_zero/executor.h index 93bc8e8f9da..01afa1b8630 100644 --- a/automl_zero/executor.h +++ b/automl_zero/executor.h @@ -34,6 +34,7 @@ #include "instruction.h" #include "memory.h" #include "random_generator.h" +#include "gtest/gtest_prod.h" namespace automl_zero { diff --git a/automl_zero/executor_test.cc b/automl_zero/executor_test.cc index 985cee0f7db..11d4d5c52b7 100644 --- a/automl_zero/executor_test.cc +++ b/automl_zero/executor_test.cc @@ -31,7 +31,7 @@ #include "memory.h" #include "random_generator.h" #include "random_generator_test_util.h" -#include "testing/base/public/gmock.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/strings/str_cat.h" #include "Eigen/Core" diff --git a/automl_zero/generator_test.cc b/automl_zero/generator_test.cc index 4a3b743439c..56af52a0b38 100644 --- a/automl_zero/generator_test.cc +++ b/automl_zero/generator_test.cc @@ -31,7 +31,7 @@ #include "random_generator_test_util.h" #include "test_util.h" #include "util.h" -#include "testing/base/public/gmock.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/strings/str_cat.h" diff --git a/automl_zero/generator_test_util.h b/automl_zero/generator_test_util.h index 59d8bbed338..9660d657dc9 100644 --- a/automl_zero/generator_test_util.h +++ b/automl_zero/generator_test_util.h @@ -17,7 +17,7 @@ #include "algorithm.h" #include "generator.h" -#include "testing/base/public/gmock.h" +#include "gmock/gmock.h" namespace automl_zero { diff --git a/automl_zero/mutator.h b/automl_zero/mutator.h index 1ccc4fb76bd..1848cdb0842 100644 --- a/automl_zero/mutator.h +++ b/automl_zero/mutator.h @@ -24,6 +24,7 @@ #include "mutator.pb.h" #include "random_generator.h" #include "randomizer.h" +#include "gtest/gtest_prod.h" namespace automl_zero { diff --git a/automl_zero/mutator_test_util.h b/automl_zero/mutator_test_util.h index 10647522cc0..a0f0089ed48 100644 --- a/automl_zero/mutator_test_util.h +++ b/automl_zero/mutator_test_util.h @@ -16,7 +16,7 @@ #define MUTATOR_TEST_UTIL_H_ #include "mutator.h" -#include "testing/base/public/gmock.h" +#include "gmock/gmock.h" namespace automl_zero { diff --git a/automl_zero/regularized_evolution.h b/automl_zero/regularized_evolution.h index a3961023fe1..0beee00b383 100644 --- a/automl_zero/regularized_evolution.h +++ b/automl_zero/regularized_evolution.h @@ -29,6 +29,7 @@ #include "random_generator.h" #include "absl/flags/flag.h" #include "absl/time/time.h" +#include "gtest/gtest_prod.h" namespace automl_zero { diff --git a/automl_zero/regularized_evolution_test.cc b/automl_zero/regularized_evolution_test.cc index 31d4c4e94c0..042f4be2fe4 100644 --- a/automl_zero/regularized_evolution_test.cc +++ b/automl_zero/regularized_evolution_test.cc @@ -28,7 +28,7 @@ #include "mutator_test_util.h" #include "random_generator.h" #include "test_util.h" -#include "testing/base/public/gmock.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/flags/flag.h" #include "absl/memory/memory.h" diff --git a/automl_zero/run_demo.sh b/automl_zero/run_demo.sh new file mode 100755 index 00000000000..6c82e03822b --- /dev/null +++ b/automl_zero/run_demo.sh @@ -0,0 +1,78 @@ +# Copyright 2020 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +# Usage: +# ./run_demo.sh + +bazel run -c opt --copt=-mavx --dynamic_mode=off \ + --copt=-DMAX_SCALAR_ADDRESSES=4 \ + --copt=-DMAX_VECTOR_ADDRESSES=3 \ + --copt=-DMAX_MATRIX_ADDRESSES=1 \ + :run_search_experiment -- \ + --search_experiment_spec=" \ + search_tasks { \ + tasks { \ + scalar_linear_regression_task {} \ + features_size: 4 \ + num_train_examples: 100 \ + num_valid_examples: 100 \ + num_tasks: 10 \ + eval_type: RMS_ERROR \ + } \ + } \ + setup_ops: [SCALAR_CONST_SET_OP, VECTOR_INNER_PRODUCT_OP, SCALAR_DIFF_OP, SCALAR_PRODUCT_OP, SCALAR_VECTOR_PRODUCT_OP, VECTOR_SUM_OP] \ + predict_ops: [SCALAR_CONST_SET_OP, VECTOR_INNER_PRODUCT_OP, SCALAR_DIFF_OP, SCALAR_PRODUCT_OP, SCALAR_VECTOR_PRODUCT_OP, VECTOR_SUM_OP] \ + learn_ops: [SCALAR_CONST_SET_OP, VECTOR_INNER_PRODUCT_OP, SCALAR_DIFF_OP, SCALAR_PRODUCT_OP, SCALAR_VECTOR_PRODUCT_OP, VECTOR_SUM_OP] \ + learn_size_init: 8 \ + setup_size_init: 10 \ + predict_size_init: 2 \ + fec {num_train_examples: 10 num_valid_examples: 10} \ + fitness_combination_mode: MEAN_FITNESS_COMBINATION \ + population_size: 1000 \ + tournament_size: 10 \ + initial_population: RANDOM_ALGORITHM \ + max_individuals: 1000000 \ + allowed_mutation_types { + mutation_types: [ALTER_PARAM_MUTATION_TYPE, RANDOMIZE_INSTRUCTION_MUTATION_TYPE, RANDOMIZE_COMPONENT_FUNCTION_MUTATION_TYPE] \ + } \ + mutate_prob: 0.9 \ + progress_every: 10000 \ + " \ + --final_tasks=" \ + tasks { \ + scalar_linear_regression_task {} \ + features_size: 4 \ + num_train_examples: 1000 \ + num_valid_examples: 100 \ + num_tasks: 100 \ + eval_type: RMS_ERROR \ + data_seeds: [1000000] \ + param_seeds: [2000000] \ + } \ + " \ + --max_experiments=0 \ + --randomize_task_seeds \ + --select_tasks=" \ + tasks { \ + scalar_linear_regression_task {} \ + features_size: 4 \ + num_train_examples: 1000 \ + num_valid_examples: 100 \ + num_tasks: 100 \ + eval_type: RMS_ERROR \ + } \ + " \ + --sufficient_fitness=0.9999 diff --git a/automl_zero/run_integration_test_nonlinear.sh b/automl_zero/run_integration_test_nonlinear.sh new file mode 100755 index 00000000000..bb4d5878436 --- /dev/null +++ b/automl_zero/run_integration_test_nonlinear.sh @@ -0,0 +1,83 @@ +# Copyright 2020 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +# TODO(ereal): fix mutate_*_size_* values. Affects random number generation. + +# Nonlinear example with Neural Network as init model: +bazel run -c opt --copt=-mavx --dynamic_mode=off \ + --copt=-DMAX_SCALAR_ADDRESSES=5 \ + --copt=-DMAX_VECTOR_ADDRESSES=9 \ + --copt=-DMAX_MATRIX_ADDRESSES=2 \ + :run_search_experiment -- \ + --search_experiment_spec=" \ + search_tasks { \ + tasks { \ + scalar_2layer_nn_regression_task {} \ + features_size: 4 \ + num_train_examples: 1000 \ + num_valid_examples: 100 \ + num_tasks: 1 \ + eval_type: RMS_ERROR \ + } \ + } \ + setup_ops: [VECTOR_GAUSSIAN_SET_OP, MATRIX_GAUSSIAN_SET_OP] \ + predict_ops: [] \ + learn_ops: [] \ + setup_size_init: 2 \ + predict_size_init: 1 \ + learn_size_init: 4 \ + train_budget {train_budget_baseline: NEURAL_NET_ALGORITHM} \ + fitness_combination_mode: MEAN_FITNESS_COMBINATION \ + population_size: 1000 \ + tournament_size: 10 \ + initial_population: INTEGRATION_TEST_DAMAGED_NEURAL_NET_ALGORITHM \ + max_individuals: 100000 \ + allowed_mutation_types { + mutation_types: [ALTER_PARAM_MUTATION_TYPE, INSERT_INSTRUCTION_MUTATION_TYPE, REMOVE_INSTRUCTION_MUTATION_TYPE, TRADE_INSTRUCTION_MUTATION_TYPE] \ + } \ + mutate_prob: 1.0 \ + mutate_setup_size_min: 6 \ + mutate_setup_size_max: 7 \ + mutate_predict_size_min: 3 \ + mutate_predict_size_max: 4 \ + mutate_learn_size_min: 9 \ + mutate_learn_size_max: 10 \ + progress_every: 10000 \ + " \ + --final_tasks=" \ + tasks { \ + scalar_2layer_nn_regression_task {} \ + features_size: 4 \ + num_train_examples: 1000 \ + num_valid_examples: 100 \ + num_tasks: 1 \ + eval_type: RMS_ERROR \ + param_seeds: [9000] \ + data_seeds: [19000] \ + } \ + " \ + --random_seed=100001 \ + --select_tasks=" \ + tasks { \ + scalar_2layer_nn_regression_task {} \ + features_size: 4 \ + num_train_examples: 1000 \ + num_valid_examples: 100 \ + num_tasks: 1 eval_type: \ + RMS_ERROR param_seeds: [9000] \ + data_seeds: [19000] \ + } \ + " diff --git a/automl_zero/run_integration_tests.sh b/automl_zero/run_integration_tests.sh new file mode 100755 index 00000000000..728457b25aa --- /dev/null +++ b/automl_zero/run_integration_tests.sh @@ -0,0 +1,39 @@ +# Copyright 2020 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +# Run this file as an integrated test that will generate outputs to copy to +# critique. + +# Usage: +# ./run_integration_tests.sh + +echo "Running integrated tests. You need to copy the output below to the 'Tested' section of the CL." +echo +echo "LOCAL_LINEAR_TEST:" +echo "output is" +./run_integration_test_linear.sh 2>&1 | grep "indivs=[2|1]1000\|Final evaluation fitness" +echo "expected is +indivs=11000, elapsed_secs=0, mean=0.240699, stdev=0.090653, best fit=0.305231, +indivs=21000, elapsed_secs=0, mean=0.481513, stdev=0.401085, best fit=0.999813, +Final evaluation fitness (on unseen data) = 1.000000" +echo +echo "LOCAL_NONLINEAR_TEST:" +echo "output is" +./run_integration_test_nonlinear.sh 2>&1 | grep "indivs=[1|9]1000\|Final evaluation fitness" +echo "expected is +indivs=11000, elapsed_secs=X, mean=0.855631, stdev=0.277759, best fit=0.999835, +indivs=91000, elapsed_secs=X, mean=0.724830, stdev=0.423565, best fit=1.000000, +Final evaluation fitness (on unseen data) = 0.999680" diff --git a/automl_zero/run_search_experiment.cc b/automl_zero/run_search_experiment.cc index 88f93fb711f..5581d2eb95a 100644 --- a/automl_zero/run_search_experiment.cc +++ b/automl_zero/run_search_experiment.cc @@ -37,6 +37,7 @@ #include "train_budget.h" #include "google/protobuf/text_format.h" #include "absl/flags/flag.h" +#include "absl/flags/parse.h" #include "absl/time/time.h" typedef automl_zero::IntegerT IntegerT; @@ -251,7 +252,7 @@ void run() { } // namespace automl_zero int main(int argc, char** argv) { - gflags::ParseCommandLineFlags(&argc, &argv, true); + absl::ParseCommandLine(argc, argv); automl_zero::run(); return 0; } diff --git a/automl_zero/task.h b/automl_zero/task.h index b26c0041cc9..7d18484bee9 100644 --- a/automl_zero/task.h +++ b/automl_zero/task.h @@ -25,6 +25,7 @@ #include "task.pb.h" #include "definitions.h" +#include "gtest/gtest_prod.h" namespace automl_zero { diff --git a/automl_zero/task_util.cc b/automl_zero/task_util.cc index 73f4e4c0ff4..c22b263ca1d 100644 --- a/automl_zero/task_util.cc +++ b/automl_zero/task_util.cc @@ -138,14 +138,12 @@ void RandomizeTaskSeeds(TaskCollection* task_collection, RandomSeedT base_param_seed = HashMix(static_cast(85652777), seed); mt19937 param_seed_bit_gen(base_param_seed); - RandomGenerator param_seed_gen = RandomGenerator( - ¶m_seed_bit_gen); + RandomGenerator param_seed_gen(¶m_seed_bit_gen); RandomSeedT base_data_seed = HashMix(static_cast(38272328), seed); mt19937 data_seed_bit_gen(base_data_seed); - RandomGenerator data_seed_gen = RandomGenerator( - &data_seed_bit_gen); + RandomGenerator data_seed_gen(&data_seed_bit_gen); for (TaskSpec& task : *task_collection->mutable_tasks()) { task.clear_param_seeds(); diff --git a/automl_zero/task_util.h b/automl_zero/task_util.h index 02794da6b9a..839af75053d 100644 --- a/automl_zero/task_util.h +++ b/automl_zero/task_util.h @@ -132,9 +132,9 @@ struct ScalarLinearRegressionTaskCreator { RandomSeedT data_seed, TaskBuffer* buffer) { ClearAndResize(num_train_examples, num_valid_examples, buffer); std::mt19937 data_bit_gen(data_seed + 939723201); - RandomGenerator data_gen = RandomGenerator(&data_bit_gen); + RandomGenerator data_gen(&data_bit_gen); std::mt19937 param_bit_gen(param_seed + 997958712); - RandomGenerator weights_gen = RandomGenerator(¶m_bit_gen); + RandomGenerator weights_gen(¶m_bit_gen); Generator generator(NO_OP_ALGORITHM, 0, 0, 0, {}, {}, {}, nullptr, nullptr); @@ -169,9 +169,9 @@ struct Scalar2LayerNnRegressionTaskCreator { RandomSeedT data_seed, TaskBuffer* buffer) { ClearAndResize(num_train_examples, num_valid_examples, buffer); std::mt19937 data_bit_gen(data_seed + 865546086); - RandomGenerator data_gen = RandomGenerator(&data_bit_gen); + RandomGenerator data_gen(&data_bit_gen); std::mt19937 param_bit_gen(param_seed + 174299604); - RandomGenerator weights_gen = RandomGenerator(¶m_bit_gen); + RandomGenerator weights_gen(¶m_bit_gen); Generator generator(NO_OP_ALGORITHM, 0, 0, 0, {}, {}, {}, nullptr, nullptr); diff --git a/automl_zero/task_util_test.cc b/automl_zero/task_util_test.cc index 9663cfee293..88806bf7f8c 100644 --- a/automl_zero/task_util_test.cc +++ b/automl_zero/task_util_test.cc @@ -31,7 +31,7 @@ #include "random_generator.h" #include "random_generator_test_util.h" #include "test_util.h" -#include "testing/base/public/gmock.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/strings/str_cat.h" #include "Eigen/Core" diff --git a/automl_zero/test_util_test.cc b/automl_zero/test_util_test.cc index f581a5ae666..857c6ee42fa 100644 --- a/automl_zero/test_util_test.cc +++ b/automl_zero/test_util_test.cc @@ -18,7 +18,7 @@ #include #include "definitions.h" -#include "testing/base/public/gmock.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" namespace automl_zero {