Skip to content

Commit

Permalink
rerun test to make sure they work in jupyter
Browse files Browse the repository at this point in the history
  • Loading branch information
@ajlee21
ajlee21 committed Jun 20, 2022
1 parent 38028a4 commit f49b51c
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 68 deletions.
3 changes: 2 additions & 1 deletion tests/config_test_1.tsv
View file
Expand Up @@ -6,9 +6,10 @@ metadata_filename "data/metadata/recount2_metadata.tsv"
experiment_ids_filename "data/metadata/experiment_ids.txt"
scaler_transform_filename "../test_results/scaler_transform.pickle"
vae_model_dir "models/"
training_stats_dir "logs/"
learning_rate 0.001
batch_size 5
epochs 10
epochs 3
kappa 0.01
intermediate_dim 2500
latent_dim 30
Expand Down
1 change: 1 addition & 0 deletions tests/config_test_2.tsv
View file
Expand Up @@ -24,3 +24,4 @@ metadata_delimiter "\t"
metadata_experiment_colname "project"
metadata_sample_colname "run"
project_id "SRP016140"
simulated_data_dir "test_results/"
4 changes: 4 additions & 0 deletions tests/data/test_vae_log.tsv
View file
@@ -0,0 +1,4 @@
val_loss val_kl_loss val_recons_loss loss kl_loss recons_loss learning_rate batch_size epochs kappa
3356.395222981771 112.98992919921875 3356.395263671875 2954.725341796875 14.532898902893066 2954.725341796875 0.001 5 3 0.01
3487.342244466146 180.31304931640625 3487.342041015625 2655.964013671875 14.915770530700684 2655.9638671875 0.001 5 3 0.01
3444.1155598958335 214.6160430908203 3444.115478515625 2599.88212890625 14.356307029724121 2599.882080078125 0.001 5 3 0.01
88 changes: 62 additions & 26 deletions tests/simulation_tests.ipynb
View file
Expand Up @@ -23,11 +23,11 @@
"name": "stdout",
"output_type": "stream",
"text": [
"WARNING:tensorflow:From /home/alexandra/Documents/Sandbox/ponyo_LOCAL/ponyo/helper_vae.py:21: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
"WARNING:tensorflow:From /home/alexandra/Documents/Repos/ponyo/ponyo/helper_vae.py:21: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
"\n",
"WARNING:tensorflow:From /home/alexandra/Documents/Sandbox/ponyo_LOCAL/ponyo/helper_vae.py:25: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
"WARNING:tensorflow:From /home/alexandra/Documents/Repos/ponyo/ponyo/helper_vae.py:25: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
"\n",
"WARNING:tensorflow:From /home/alexandra/Documents/Sandbox/ponyo_LOCAL/ponyo/helper_vae.py:25: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
"WARNING:tensorflow:From /home/alexandra/Documents/Repos/ponyo/ponyo/helper_vae.py:25: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
"\n"
]
}
Expand Down Expand Up @@ -59,7 +59,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"WARNING:tensorflow:From /home/alexandra/Documents/Sandbox/ponyo_LOCAL/ponyo/train_vae_modules.py:56: The name tf.set_random_seed is deprecated. Please use tf.compat.v1.set_random_seed instead.\n",
"WARNING:tensorflow:From /home/alexandra/Documents/Repos/ponyo/ponyo/train_vae_modules.py:56: The name tf.set_random_seed is deprecated. Please use tf.compat.v1.set_random_seed instead.\n",
"\n"
]
}
Expand Down Expand Up @@ -102,8 +102,7 @@
"sample_id_colname = params['metadata_sample_colname']\n",
"project_id = params['project_id']\n",
"training_stats_dir = params[\"training_stats_dir\"]\n",
"vae_model_dir = params[\"vae_model_dir\"]\n",
"simulated_data_dir = params[\"simulated_data_dir\"]"
"vae_model_dir = params[\"vae_model_dir\"]"
]
},
{
Expand All @@ -113,6 +112,13 @@
"## 1. Random simulation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 5,
Expand All @@ -125,7 +131,7 @@
"WARNING:tensorflow:From /home/alexandra/anaconda3/envs/test_ponyo/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"If using Keras pass *_constraint arguments to layers.\n",
"Normalized gene expression data contains 273 samples and 5000 genes\n",
"Normalized gene expression data contains 56 samples and 5000 genes\n",
"WARNING:tensorflow:From /home/alexandra/anaconda3/envs/test_ponyo/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
"\n",
"Return: simulated gene expression data containing 100 samples and 5000 genes\n"
Expand All @@ -135,7 +141,7 @@
"source": [
"# Run simulation\n",
"simulated_data = simulate_expression_data.simulate_by_random_sampling(\n",
" normalized_data_filename,\n",
" normalized_compendium_filename,\n",
" num_simulated_samples,\n",
" vae_model_dir,\n",
")"
Expand All @@ -154,13 +160,14 @@
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"There are 3220 experiments in the compendium\n",
"There are 9 experiments with gene expression data\n",
"9 experiment ids saved to file\n"
]
"data": {
"text/plain": [
"'utils.create_experiment_id_file(metadata_filename,\\n normalized_compendium_filename,\\n experiment_id_filename,\\n config_filename)'"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Expand All @@ -176,12 +183,14 @@
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Normalized gene expression data contains 273 samples and 5000 genes\n",
"Return: simulated gene expression data containing 1537 samples and 5001 genes\n"
]
"data": {
"text/plain": [
"'# Run simulation\\nsimulated_data = simulate_expression_data.simulate_by_latent_transformation(\\n num_simulated_experiments,\\n normalized_compendium_filename,\\n vae_model_dir,\\n latent_dim,\\n metadata_filename,\\n metadata_delimiter,\\n experiment_id_colname,\\n sample_id_colname,\\n experiment_id_filename,\\n local_dir,\\n)'"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Expand Down Expand Up @@ -221,7 +230,18 @@
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"'# Run simulation\\nsimulate_expression_data.shift_template_experiment(\\n normalized_compendium_filename,\\n vae_model_dir,\\n latent_dim,\\n scaler_filename,\\n metadata_filename,\\n metadata_delimiter,\\n experiment_id_colname,\\n sample_id_colname,\\n project_id,\\n local_dir,\\n simulated_data_dir,\\n num_runs)'"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\"\"\"# Run simulation\n",
"simulate_expression_data.shift_template_experiment(\n",
Expand Down Expand Up @@ -266,7 +286,7 @@
"# Load parameters\n",
"local_dir = params[\"local_dir\"]\n",
"raw_compendium_filename = params[\"raw_compendium_filename\"]\n",
"normalized_compendium_filename = params[\"normalized_comepndium_filename\"]\n",
"normalized_compendium_filename = params[\"normalized_compendium_filename\"]\n",
"raw_template_filename = params[\"raw_template_filename\"]\n",
"mapped_template_filename = params[\"mapped_template_filename\"]\n",
"normalized_template_filename = params[\"normalized_template_filename\"]\n",
Expand Down Expand Up @@ -305,7 +325,7 @@
"output_type": "stream",
"text": [
"(5, 5000)\n",
"(273, 5000)\n"
"(56, 5000)\n"
]
}
],
Expand Down Expand Up @@ -574,11 +594,27 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 14,
"metadata": {
"scrolled": false
},
"outputs": [],
"outputs": [
{
"ename": "FileNotFoundError",
"evalue": "[Errno 2] No such file or directory: 'test_results/selected_simulated_data_SRP016140_0.tsv'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-14-b15d2915362d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mlatent_dim\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0mnum_runs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0msimulated_data_dir\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 12\u001b[0m )\n",
"\u001b[0;32m~/Documents/Repos/ponyo/ponyo/simulate_expression_data.py\u001b[0m in \u001b[0;36membed_shift_template_experiment\u001b[0;34m(normalized_compendium_filename, normalized_template_filename, vae_model_dir, selected_experiment_id, scaler_filename, local_dir, latent_dim, num_simulated_experiments, simulated_data_dir)\u001b[0m\n\u001b[1;32m 1000\u001b[0m )\n\u001b[1;32m 1001\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1002\u001b[0;31m \u001b[0msimulated_data_scaled_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_filename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfloat_format\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"%.3f\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"\\t\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1003\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1004\u001b[0m out_encoded_filename = os.path.join(\n",
"\u001b[0;32m~/anaconda3/envs/test_ponyo/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mto_csv\u001b[0;34m(self, path_or_buf, sep, na_rep, float_format, columns, header, index, index_label, mode, encoding, compression, quoting, quotechar, line_terminator, chunksize, date_format, doublequote, escapechar, decimal, errors)\u001b[0m\n\u001b[1;32m 3168\u001b[0m \u001b[0mdecimal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdecimal\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3169\u001b[0m )\n\u001b[0;32m-> 3170\u001b[0;31m \u001b[0mformatter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3171\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3172\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpath_or_buf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/envs/test_ponyo/lib/python3.7/site-packages/pandas/io/formats/csvs.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 188\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 189\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 190\u001b[0;31m \u001b[0mcompression\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompression_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompression\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 191\u001b[0m )\n\u001b[1;32m 192\u001b[0m \u001b[0mclose\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/envs/test_ponyo/lib/python3.7/site-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors)\u001b[0m\n\u001b[1;32m 491\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 492\u001b[0m \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnewline\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 494\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mis_text\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[0;31m# No explicit encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'test_results/selected_simulated_data_SRP016140_0.tsv'"
]
}
],
"source": [
"# Run simulation\n",
"simulate_expression_data.embed_shift_template_experiment(\n",
Expand Down

0 comments on commit f49b51c

Please sign in to comment.