#!/bin/bash -v DATA_DIR=/raid/data/nmt_data python -m nmt.nmt \ --src=en --tgt=de \ --vocab_prefix=${DATA_DIR}/vocab.bpe.32000 \ --train_prefix=${DATA_DIR}/train.tok.clean.bpe.32000 \ --dev_prefix=${DATA_DIR}/newstest2014.tok.clean.bpe.32000 \ --test_prefix=${DATA_DIR}/newstest2015.tok.clean.bpe.32000 \ --out_dir=/raid/skyw/train_dir/nmt_model/test \ --num_train_steps=12000 \ --steps_per_stats=100 \ --num_layers=2 \ --num_units=128 \ --dropout=0.2 \ --metrics=bleu \ --bpe_delimiter=@@ \ --hparams_path nmt/standard_hparams/wmt16_en_de.json # Job id 0 # Loading hparams from /raid/skyw/train_dir/nmt_model/test/hparams # Loading standard hparams from nmt/standard_hparams/wmt16_en_de.json # hparams: src=en tgt=de train_prefix=/raid/data/nmt_data/train.tok.clean.bpe.32000 dev_prefix=/raid/data/nmt_data/newstest2014.tok.clean.bpe.32000 test_prefix=/raid/data/nmt_data/newstest2015.tok.clean.bpe.32000 out_dir=/raid/skyw/train_dir/nmt_model/test # Vocab file /raid/data/nmt_data/vocab.bpe.32000.en exists The first 3 vocab words [-e , , ] are not [, , ] # Vocab file /raid/data/nmt_data/vocab.bpe.32000.de exists The first 3 vocab words [-e , , ] are not [, , ] saving hparams to /raid/skyw/train_dir/nmt_model/test/hparams saving hparams to /raid/skyw/train_dir/nmt_model/test/best_bleu/hparams attention=normed_bahdanau attention_architecture=standard batch_size=128 beam_width=10 best_bleu=0 best_bleu_dir=/raid/skyw/train_dir/nmt_model/test/best_bleu bpe_delimiter=@@ colocate_gradients_with_ops=True decay_factor=0.5 decay_steps=17000 dev_prefix=/raid/data/nmt_data/newstest2014.tok.clean.bpe.32000 dropout=0.2 encoder_type=bi eos= epoch_step=0 forget_bias=1.0 infer_batch_size=32 init_weight=0.1 learning_rate=1.0 length_penalty_weight=0.0 log_device_placement=False max_gradient_norm=5.0 max_train=0 metrics=[u'bleu'] num_buckets=5 num_gpus=1 num_layers=4 num_residual_layers=0 num_train_steps=340000 num_units=1024 optimizer=sgd out_dir=/raid/skyw/train_dir/nmt_model/test pass_hidden_state=True random_seed=None residual=False share_vocab=False sos= source_reverse=False src=en src_max_len=50 src_max_len_infer=None src_vocab_file=/raid/skyw/train_dir/nmt_model/test/vocab.bpe.32000.en src_vocab_size=36551 start_decay_step=170000 steps_per_external_eval=None steps_per_stats=100 test_prefix=/raid/data/nmt_data/newstest2015.tok.clean.bpe.32000 tgt=de tgt_max_len=50 tgt_max_len_infer=None tgt_vocab_file=/raid/skyw/train_dir/nmt_model/test/vocab.bpe.32000.de tgt_vocab_size=36551 time_major=True train_prefix=/raid/data/nmt_data/train.tok.clean.bpe.32000 unit_type=lstm vocab_prefix=/raid/data/nmt_data/vocab.bpe.32000 # creating train graph ... num_bi_layers = 2, num_bi_residual_layers=0 cell 0 LSTM, forget_bias=1 DropoutWrapper, dropout=0.2 DeviceWrapper, device=/gpu:0 cell 1 LSTM, forget_bias=1 DropoutWrapper, dropout=0.2 DeviceWrapper, device=/gpu:0 cell 0 LSTM, forget_bias=1 DropoutWrapper, dropout=0.2 DeviceWrapper, device=/gpu:0 cell 1 LSTM, forget_bias=1 DropoutWrapper, dropout=0.2 DeviceWrapper, device=/gpu:0 cell 0 LSTM, forget_bias=1 DropoutWrapper, dropout=0.2 DeviceWrapper, device=/gpu:0 cell 1 LSTM, forget_bias=1 DropoutWrapper, dropout=0.2 DeviceWrapper, device=/gpu:0 cell 2 LSTM, forget_bias=1 DropoutWrapper, dropout=0.2 DeviceWrapper, device=/gpu:0 cell 3 LSTM, forget_bias=1 DropoutWrapper, dropout=0.2 DeviceWrapper, device=/gpu:0 start_decay_step=170000, learning_rate=1, decay_steps 17000,decay_factor 0.5 # Trainable variables embeddings/encoder/embedding_encoder:0, (36551, 1024), embeddings/decoder/embedding_decoder:0, (36551, 1024), dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/memory_layer/kernel:0, (2048, 1024), dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (3072, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_2/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_2/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_3/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_3/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/query_layer/kernel:0, (1024, 1024), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/attention_v:0, (1024,), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/attention_g:0, (), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/attention_b:0, (1024,), /device:GPU:0 dynamic_seq2seq/decoder/attention/attention_layer/kernel:0, (3072, 1024), /device:GPU:0 dynamic_seq2seq/decoder/output_projection/kernel:0, (1024, 36551), /device:GPU:0 # creating eval graph ... num_bi_layers = 2, num_bi_residual_layers=0 cell 0 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 1 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 0 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 1 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 0 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 1 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 2 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 3 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 start_decay_step=170000, learning_rate=1, decay_steps 17000,decay_factor 0.5 # Trainable variables embeddings/encoder/embedding_encoder:0, (36551, 1024), embeddings/decoder/embedding_decoder:0, (36551, 1024), dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/memory_layer/kernel:0, (2048, 1024), dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (3072, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_2/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_2/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_3/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_3/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/query_layer/kernel:0, (1024, 1024), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/attention_v:0, (1024,), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/attention_g:0, (), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/attention_b:0, (1024,), /device:GPU:0 dynamic_seq2seq/decoder/attention/attention_layer/kernel:0, (3072, 1024), /device:GPU:0 dynamic_seq2seq/decoder/output_projection/kernel:0, (1024, 36551), /device:GPU:0 # creating infer graph ... num_bi_layers = 2, num_bi_residual_layers=0 cell 0 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 1 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 0 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 1 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 0 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 1 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 2 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 cell 3 LSTM, forget_bias=1 DeviceWrapper, device=/gpu:0 start_decay_step=170000, learning_rate=1, decay_steps 17000,decay_factor 0.5 # Trainable variables embeddings/encoder/embedding_encoder:0, (36551, 1024), embeddings/decoder/embedding_decoder:0, (36551, 1024), dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/fw/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/encoder/bidirectional_rnn/bw/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/memory_layer/kernel:0, (2048, 1024), dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (3072, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_2/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_2/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_3/basic_lstm_cell/kernel:0, (2048, 4096), /device:GPU:0 dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_3/basic_lstm_cell/bias:0, (4096,), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/query_layer/kernel:0, (1024, 1024), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/attention_v:0, (1024,), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/attention_g:0, (), /device:GPU:0 dynamic_seq2seq/decoder/attention/bahdanau_attention/attention_b:0, (1024,), /device:GPU:0 dynamic_seq2seq/decoder/attention/attention_layer/kernel:0, (3072, 1024), /device:GPU:0 dynamic_seq2seq/decoder/output_projection/kernel:0, (1024, 36551), # log_file=/raid/skyw/train_dir/nmt_model/test/log_1500309031 2017-07-17 09:30:31.459101: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2017-07-17 09:30:31.459116: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 2017-07-17 09:30:31.459119: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 2017-07-17 09:30:31.459122: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 2017-07-17 09:30:31.459124: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 2017-07-17 09:30:31.923512: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: name: Tesla P100-DGXS-16GB major: 6 minor: 0 memoryClockRate (GHz) 1.4805 pciBusID 0000:07:00.0 Total memory: 15.89GiB Free memory: 15.60GiB 2017-07-17 09:30:31.923534: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 2017-07-17 09:30:31.923538: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 2017-07-17 09:30:31.923544: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla P100-DGXS-16GB, pci bus id: 0000:07:00.0) 2017-07-17 09:30:31.925513: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla P100-DGXS-16GB, pci bus id: 0000:07:00.0) 2017-07-17 09:30:31.926086: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla P100-DGXS-16GB, pci bus id: 0000:07:00.0) created train model with fresh parameters, time 0.00s. 2017-07-17 09:30:32.616890: W tensorflow/core/framework/op_kernel.cc:1158] Failed precondition: HashTable has different value for same key. Key has 1 and trying to add value 4 2017-07-17 09:30:32.619153: W tensorflow/core/framework/op_kernel.cc:1158] Failed precondition: HashTable has different value for same key. Key has 1 and trying to add value 4 Traceback (most recent call last): File "/usr/lib/python2.7/runpy.py", line 162, in _run_module_as_main "__main__", fname, loader, pkg_name) File "/usr/lib/python2.7/runpy.py", line 72, in _run_code exec code in run_globals File "/raid/skyw/google-git/nmt/nmt/nmt.py", line 479, in tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 48, in run _sys.exit(main(_sys.argv[:1] + flags_passthrough)) File "/raid/skyw/google-git/nmt/nmt/nmt.py", line 292, in main train.train(hparams) File "nmt/train.py", line 212, in train train_model, model_dir, train_sess, hparams.out_dir, "train") File "nmt/model_helper.py", line 218, in create_or_load_model session.run(tf.tables_initializer()) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 789, in run run_metadata_ptr) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 997, in _run feed_dict_string, options, run_metadata) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1132, in _do_run target_list, options, run_metadata) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1152, in _do_call raise type(e)(node_def, op, message) tensorflow.python.framework.errors_impl.FailedPreconditionError: HashTable has different value for same key. Key has 1 and trying to add value 4 [[Node: string_to_index_1/hash_table/table_init = InitializeTableFromTextFileV2[delimiter="\t", key_index=-2, value_index=-1, vocab_size=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](string_to_index_1/hash_table, string_to_index_1/hash_table/table_init/asset_filepath)]] [[Node: string_to_index_1/hash_table/table_init/_14 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/gpu:0", send_device="/job:localhost/replica:0/task:0/cpu:0", send_device_incarnation=1, tensor_name="edge_8_string_to_index_1/hash_table/table_init", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"]()]] Caused by op u'string_to_index_1/hash_table/table_init', defined at: File "/usr/lib/python2.7/runpy.py", line 162, in _run_module_as_main "__main__", fname, loader, pkg_name) File "/usr/lib/python2.7/runpy.py", line 72, in _run_code exec code in run_globals File "/raid/skyw/google-git/nmt/nmt/nmt.py", line 479, in tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 48, in run _sys.exit(main(_sys.argv[:1] + flags_passthrough)) File "/raid/skyw/google-git/nmt/nmt/nmt.py", line 292, in main train.train(hparams) File "nmt/train.py", line 159, in train create_train_model(model_creator, hparams, scope)) File "nmt/train.py", line 58, in create_train_model tgt_vocab_file, default_value=vocab_utils.UNK_ID) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/ops/lookup_ops.py", line 947, in index_table_from_file init, default_value, shared_name=shared_name, name=hash_table_scope) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/ops/lookup_ops.py", line 275, in __init__ super(HashTable, self).__init__(table_ref, default_value, initializer) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/ops/lookup_ops.py", line 161, in __init__ self._init = initializer.initialize(self) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/ops/lookup_ops.py", line 519, in initialize name=scope) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/ops/gen_lookup_ops.py", line 187, in _initialize_table_from_text_file_v2 name=name) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op op_def=op_def) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2506, in create_op original_op=self._default_original_op, op_def=op_def) File "/raid/skyw/venv/tensorflow-pip-py27/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1269, in __init__ self._traceback = _extract_stack() FailedPreconditionError (see above for traceback): HashTable has different value for same key. Key has 1 and trying to add value 4 [[Node: string_to_index_1/hash_table/table_init = InitializeTableFromTextFileV2[delimiter="\t", key_index=-2, value_index=-1, vocab_size=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](string_to_index_1/hash_table, string_to_index_1/hash_table/table_init/asset_filepath)]] [[Node: string_to_index_1/hash_table/table_init/_14 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/gpu:0", send_device="/job:localhost/replica:0/task:0/cpu:0", send_device_incarnation=1, tensor_name="edge_8_string_to_index_1/hash_table/table_init", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"]()]]