Completely wrong infer result in tf.keras and tf.js

[theangels]

To get help from the community, check out our Google group.

TensorFlow.js version

TensorFlow version: 1.10.1
tensorflowjs_converters version: 0.6.1
TensorFlow.js version: script(src='https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@latest')

Browser version

Chrome 69.0.3497.100

Describe the problem or feature request

I trained a Keras Sequential model in python, and transofrm it to tf.js model by using tensorflowjs_converters. But I found the infer result are completely wrong in python and tf.js. There is the infer code.

Python infer

from keras.models import load_model

model = load_model('keras.h5')

test = np.genfromtxt('voice.txt', delimiter=',', dtype=np.float32)

test = test.reshape(1, 100, 1024, 1)
test -= 128
test /= 128.0

pred = model.predict(test)
print(pred)

tf.js infer

async function start() {
    model = await tf.loadModel('model2/model.json')
    model.summary()
    const pred = model.predict(tf.zeros([1, 100, 1024, 1]))
}
start()
let input_tensor = tf.tensor(values=intArray);

var tensorValue = input_tensor.dataSync()

for(var i = 0; i < intArray; i++){
    if(intArray != tensorValue){
        console.log('Wrong!')
    }
}
const reshaped_tensor = input_tensor.reshape([1, 100, 1024, 1]).toFloat();
const divided_tensor = reshaped_tensor.sub(tf.scalar(128)).div(tf.scalar(128))
divided_tensor.print(true)
var pred = model.predict(divided_tensor).dataSync()
console.log(pred)

var "intArray" are the input data and it's shape is [100.1024], dtype is int32(I transformed it to float32 in data process)

And there is the model:

I use the same input data, and here are the infer result

Python infer

tf.js infer

Models

The model files are more than 10MB, so I use SFX. If you want to use it, just change the name, such as

“model2.002.zip” to "model2.zip.002"

tf.js model

model2.002.zip
model2.003.zip
model2.004.zip
model2.005.zip
model2.001.zip

keras model

keras.001.zip
keras.002.zip
keras.003.zip
keras.004.zip
keras.005.zip

input file

voice.txt

Code to reproduce the bug / link to feature request

N/A

[caisq]

@theangels Thanks for reporting this issue. I suspect that the difference may have more to do with the model (json) than with the weights. So can you simply share the model JSON with us? In keras, this can be obtained with model.to_json(). That way we won't have to worry about sending large files.

[theangels]

@theangels Thanks for reporting this issue. I suspect that the difference may have more to do with the model (json) than with the weights. So can you simply share the model JSON with us? In keras, this can be obtained with model.to_json(). That way we won't have to worry about sending large files.

OK, there are the model define JSON files:

Keras model

{
	"keras_version": "2.2.2",
	"config": [{
		"config": {
			"dilation_rate": [1, 1],
			"kernel_size": [3, 3],
			"activity_regularizer": null,
			"activation": "relu",
			"kernel_initializer": {
				"config": {
					"mode": "fan_avg",
					"distribution": "uniform",
					"scale": 1.0,
					"seed": null
				},
				"class_name": "VarianceScaling"
			},
			"bias_constraint": null,
			"bias_regularizer": null,
			"filters": 16,
			"dtype": "float32",
			"bias_initializer": {
				"config": {},
				"class_name": "Zeros"
			},
			"batch_input_shape": [null, 100, 1024, 1],
			"name": "conv2d",
			"kernel_constraint": null,
			"padding": "same",
			"data_format": "channels_last",
			"use_bias": true,
			"strides": [1, 1],
			"trainable": true,
			"kernel_regularizer": null
		},
		"class_name": "Conv2D"
	}, {
		"config": {
			"pool_size": [2, 2],
			"data_format": "channels_last",
			"name": "max_pooling2d",
			"padding": "valid",
			"strides": [2, 2],
			"trainable": true
		},
		"class_name": "MaxPooling2D"
	}, {
		"config": {
			"dilation_rate": [1, 1],
			"kernel_size": [3, 3],
			"activity_regularizer": null,
			"activation": "relu",
			"kernel_initializer": {
				"config": {
					"mode": "fan_avg",
					"distribution": "uniform",
					"scale": 1.0,
					"seed": null
				},
				"class_name": "VarianceScaling"
			},
			"bias_constraint": null,
			"filters": 32,
			"bias_initializer": {
				"config": {},
				"class_name": "Zeros"
			},
			"bias_regularizer": null,
			"name": "conv2d_1",
			"kernel_constraint": null,
			"padding": "same",
			"data_format": "channels_last",
			"use_bias": true,
			"strides": [1, 1],
			"trainable": true,
			"kernel_regularizer": null
		},
		"class_name": "Conv2D"
	}, {
		"config": {
			"pool_size": [2, 2],
			"data_format": "channels_last",
			"name": "max_pooling2d_1",
			"padding": "valid",
			"strides": [2, 2],
			"trainable": true
		},
		"class_name": "MaxPooling2D"
	}, {
		"config": {
			"dilation_rate": [1, 1],
			"kernel_size": [3, 3],
			"activity_regularizer": null,
			"activation": "relu",
			"kernel_initializer": {
				"config": {
					"mode": "fan_avg",
					"distribution": "uniform",
					"scale": 1.0,
					"seed": null
				},
				"class_name": "VarianceScaling"
			},
			"bias_constraint": null,
			"filters": 64,
			"bias_initializer": {
				"config": {},
				"class_name": "Zeros"
			},
			"bias_regularizer": null,
			"name": "conv2d_2",
			"kernel_constraint": null,
			"padding": "same",
			"data_format": "channels_last",
			"use_bias": true,
			"strides": [1, 1],
			"trainable": true,
			"kernel_regularizer": null
		},
		"class_name": "Conv2D"
	}, {
		"config": {
			"pool_size": [2, 2],
			"data_format": "channels_last",
			"name": "max_pooling2d_2",
			"padding": "valid",
			"strides": [2, 2],
			"trainable": true
		},
		"class_name": "MaxPooling2D"
	}, {
		"config": {
			"data_format": "channels_last",
			"trainable": true,
			"name": "flatten"
		},
		"class_name": "Flatten"
	}, {
		"config": {
			"activation": "relu",
			"activity_regularizer": null,
			"bias_initializer": {
				"config": {},
				"class_name": "Zeros"
			},
			"bias_constraint": null,
			"kernel_initializer": {
				"config": {
					"mode": "fan_avg",
					"distribution": "uniform",
					"scale": 1.0,
					"seed": null
				},
				"class_name": "VarianceScaling"
			},
			"kernel_constraint": null,
			"bias_regularizer": null,
			"name": "dense",
			"units": 128,
			"use_bias": true,
			"trainable": true,
			"kernel_regularizer": null
		},
		"class_name": "Dense"
	}, {
		"config": {
			"activation": "softmax",
			"activity_regularizer": null,
			"bias_initializer": {
				"config": {},
				"class_name": "Zeros"
			},
			"bias_constraint": null,
			"kernel_initializer": {
				"config": {
					"mode": "fan_avg",
					"distribution": "uniform",
					"scale": 1.0,
					"seed": null
				},
				"class_name": "VarianceScaling"
			},
			"kernel_constraint": null,
			"bias_regularizer": null,
			"name": "dense_1",
			"units": 7,
			"use_bias": true,
			"trainable": true,
			"kernel_regularizer": null
		},
		"class_name": "Dense"
	}],
	"class_name": "Sequential",
	"backend": "tensorflow"
}

tf.js model

{
	"modelTopology": {
		"backend": "tensorflow",
		"model_config": {
			"config": [{
				"config": {
					"bias_regularizer": null,
					"kernel_constraint": null,
					"data_format": "channels_last",
					"bias_initializer": {
						"config": {
							"dtype": "float32"
						},
						"class_name": "Zeros"
					},
					"padding": "same",
					"dilation_rate": [1, 1],
					"use_bias": true,
					"dtype": "float32",
					"filters": 16,
					"kernel_regularizer": null,
					"kernel_initializer": {
						"config": {
							"mode": "fan_avg",
							"seed": null,
							"scale": 1.0,
							"distribution": "uniform",
							"dtype": "float32"
						},
						"class_name": "VarianceScaling"
					},
					"trainable": true,
					"batch_input_shape": [null, 100, 1024, 1],
					"name": "conv2d",
					"kernel_size": [3, 3],
					"activation": "relu",
					"strides": [1, 1],
					"bias_constraint": null,
					"activity_regularizer": null
				},
				"class_name": "Conv2D"
			}, {
				"config": {
					"data_format": "channels_last",
					"pool_size": [2, 2],
					"padding": "valid",
					"name": "max_pooling2d",
					"trainable": true,
					"strides": [2, 2],
					"dtype": "float32"
				},
				"class_name": "MaxPooling2D"
			}, {
				"config": {
					"bias_regularizer": null,
					"kernel_constraint": null,
					"data_format": "channels_last",
					"activation": "relu",
					"bias_initializer": {
						"config": {
							"dtype": "float32"
						},
						"class_name": "Zeros"
					},
					"padding": "same",
					"dilation_rate": [1, 1],
					"use_bias": true,
					"dtype": "float32",
					"filters": 32,
					"kernel_regularizer": null,
					"kernel_initializer": {
						"config": {
							"mode": "fan_avg",
							"seed": null,
							"scale": 1.0,
							"distribution": "uniform",
							"dtype": "float32"
						},
						"class_name": "VarianceScaling"
					},
					"name": "conv2d_1",
					"kernel_size": [3, 3],
					"trainable": true,
					"strides": [1, 1],
					"bias_constraint": null,
					"activity_regularizer": null
				},
				"class_name": "Conv2D"
			}, {
				"config": {
					"data_format": "channels_last",
					"pool_size": [2, 2],
					"padding": "valid",
					"name": "max_pooling2d_1",
					"trainable": true,
					"strides": [2, 2],
					"dtype": "float32"
				},
				"class_name": "MaxPooling2D"
			}, {
				"config": {
					"bias_regularizer": null,
					"kernel_constraint": null,
					"data_format": "channels_last",
					"activation": "relu",
					"bias_initializer": {
						"config": {
							"dtype": "float32"
						},
						"class_name": "Zeros"
					},
					"padding": "same",
					"dilation_rate": [1, 1],
					"use_bias": true,
					"dtype": "float32",
					"filters": 64,
					"kernel_regularizer": null,
					"kernel_initializer": {
						"config": {
							"mode": "fan_avg",
							"seed": null,
							"scale": 1.0,
							"distribution": "uniform",
							"dtype": "float32"
						},
						"class_name": "VarianceScaling"
					},
					"name": "conv2d_2",
					"kernel_size": [3, 3],
					"trainable": true,
					"strides": [1, 1],
					"bias_constraint": null,
					"activity_regularizer": null
				},
				"class_name": "Conv2D"
			}, {
				"config": {
					"data_format": "channels_last",
					"pool_size": [2, 2],
					"padding": "valid",
					"name": "max_pooling2d_2",
					"trainable": true,
					"strides": [2, 2],
					"dtype": "float32"
				},
				"class_name": "MaxPooling2D"
			}, {
				"config": {
					"name": "flatten",
					"data_format": "channels_last",
					"trainable": true,
					"dtype": "float32"
				},
				"class_name": "Flatten"
			}, {
				"config": {
					"bias_regularizer": null,
					"trainable": true,
					"use_bias": true,
					"dtype": "float32",
					"kernel_regularizer": null,
					"kernel_initializer": {
						"config": {
							"mode": "fan_avg",
							"seed": null,
							"scale": 1.0,
							"distribution": "uniform",
							"dtype": "float32"
						},
						"class_name": "VarianceScaling"
					},
					"units": 128,
					"bias_initializer": {
						"config": {
							"dtype": "float32"
						},
						"class_name": "Zeros"
					},
					"kernel_constraint": null,
					"activation": "relu",
					"name": "dense",
					"bias_constraint": null,
					"activity_regularizer": null
				},
				"class_name": "Dense"
			}, {
				"config": {
					"bias_regularizer": null,
					"trainable": true,
					"use_bias": true,
					"dtype": "float32",
					"kernel_regularizer": null,
					"kernel_initializer": {
						"config": {
							"mode": "fan_avg",
							"seed": null,
							"scale": 1.0,
							"distribution": "uniform",
							"dtype": "float32"
						},
						"class_name": "VarianceScaling"
					},
					"units": 5,
					"bias_initializer": {
						"config": {
							"dtype": "float32"
						},
						"class_name": "Zeros"
					},
					"kernel_constraint": null,
					"activation": "softmax",
					"name": "dense_1",
					"bias_constraint": null,
					"activity_regularizer": null
				},
				"class_name": "Dense"
			}],
			"class_name": "Sequential"
		},
		"keras_version": "2.1.6-tf"
	},
	"weightsManifest": [{
		"paths": ["group1-shard1of13", "group1-shard2of13", "group1-shard3of13", "group1-shard4of13", "group1-shard5of13", "group1-shard6of13", "group1-shard7of13", "group1-shard8of13", "group1-shard9of13", "group1-shard10of13", "group1-shard11of13", "group1-shard12of13", "group1-shard13of13"],
		"weights": [{
			"name": "conv2d/kernel",
			"shape": [3, 3, 1, 16],
			"dtype": "float32"
		}, {
			"name": "conv2d/bias",
			"shape": [16],
			"dtype": "float32"
		}, {
			"name": "conv2d_1/kernel",
			"shape": [3, 3, 16, 32],
			"dtype": "float32"
		}, {
			"name": "conv2d_1/bias",
			"shape": [32],
			"dtype": "float32"
		}, {
			"name": "conv2d_2/kernel",
			"shape": [3, 3, 32, 64],
			"dtype": "float32"
		}, {
			"name": "conv2d_2/bias",
			"shape": [64],
			"dtype": "float32"
		}, {
			"name": "dense/kernel",
			"shape": [98304, 128],
			"dtype": "float32"
		}, {
			"name": "dense/bias",
			"shape": [128],
			"dtype": "float32"
		}, {
			"name": "dense_1/kernel",
			"shape": [128, 5],
			"dtype": "float32"
		}, {
			"name": "dense_1/bias",
			"shape": [5],
			"dtype": "float32"
		}]
	}]
}

[caisq]

Thanks @theangels , for sending over the model JSONs. This is what I did

1. I loaded your Python model JSON with keras.models.model_from_json(). The weights are randomly initialized because of the lack of weight files. So I saved the loaded model using Model.save().
2. The saved keras model is converted into tfjs format and loaded into tfjs. This ensures that the model loaded in tfjs has the same weights as that in python keras.
3. In both Python and JS, I loaded the data from your voice.txt. It has 102301 values. So in order to form a valid input, I need to pad 99 zeros.
4. I shape the padded data as [1, 100, 1024, 1] and give it to model.predict(), in both Python Keras (v2.2.2, using TensorFlow CPU backend) and tfjs running in the browser (Linux Chrome)

The output I got from Python is:
[[7.8444309e-08 2.1324146e-07 8.6068791e-01 1.3916655e-01 8.7477653e-10
1.4430878e-04 9.4690552e-07]]

The output I got in tfjs is:
[[2e-7, 2e-7, 0.8726324, 0.1272686, 0, 0.0000974, 0.000001],]

So the two results are not that far apart, especially if we focus on the largest (3rd) value, which is about 0.86 - 0.87. The difference I got is definitely not as big as the difference you showed above.

So I think the debugging steps for you is: print out the values of the weights in Python and JS and compare them.

In Python, you can do something like

print(model.get_weights()[0])

In JS, you can do

model.getWeights()[0].print();

I suspect that there may be something messed up during your weight serving / downloading process. The situation feels very similar to another issue: #776

[theangels]

Thanks @caisq .I'm so sorry for provide the wrong model json before, because I trained another model.
There is the pretrained model JSON File which define the model.
The difference is the final layer's unit.

And the 'voice.txt' file has 102400 values (include '0' value), you can load values by using it

test = np.genfromtxt('voice.txt', delimiter=',', dtype=np.float32)

voice.txt

keras model

{
	"keras_version": "2.2.2",
	"config": [{
		"config": {
			"dtype": "float32",
			"batch_input_shape": [null, 100, 1024, 1],
			"data_format": "channels_last",
			"bias_initializer": {
				"config": {},
				"class_name": "Zeros"
			},
			"filters": 16,
			"padding": "same",
			"kernel_size": [3, 3],
			"bias_constraint": null,
			"activation": "relu",
			"kernel_constraint": null,
			"bias_regularizer": null,
			"name": "conv2d",
			"strides": [1, 1],
			"trainable": true,
			"kernel_initializer": {
				"config": {
					"mode": "fan_avg",
					"scale": 1.0,
					"distribution": "uniform",
					"seed": null
				},
				"class_name": "VarianceScaling"
			},
			"activity_regularizer": null,
			"kernel_regularizer": null,
			"use_bias": true,
			"dilation_rate": [1, 1]
		},
		"class_name": "Conv2D"
	}, {
		"config": {
			"strides": [2, 2],
			"data_format": "channels_last",
			"name": "max_pooling2d",
			"padding": "valid",
			"pool_size": [2, 2],
			"trainable": true
		},
		"class_name": "MaxPooling2D"
	}, {
		"config": {
			"data_format": "channels_last",
			"activity_regularizer": null,
			"filters": 32,
			"padding": "same",
			"bias_constraint": null,
			"activation": "relu",
			"kernel_constraint": null,
			"bias_regularizer": null,
			"name": "conv2d_1",
			"strides": [1, 1],
			"kernel_initializer": {
				"config": {
					"mode": "fan_avg",
					"scale": 1.0,
					"distribution": "uniform",
					"seed": null
				},
				"class_name": "VarianceScaling"
			},
			"trainable": true,
			"kernel_size": [3, 3],
			"bias_initializer": {
				"config": {},
				"class_name": "Zeros"
			},
			"kernel_regularizer": null,
			"use_bias": true,
			"dilation_rate": [1, 1]
		},
		"class_name": "Conv2D"
	}, {
		"config": {
			"strides": [2, 2],
			"data_format": "channels_last",
			"name": "max_pooling2d_1",
			"padding": "valid",
			"pool_size": [2, 2],
			"trainable": true
		},
		"class_name": "MaxPooling2D"
	}, {
		"config": {
			"data_format": "channels_last",
			"activity_regularizer": null,
			"filters": 64,
			"padding": "same",
			"bias_constraint": null,
			"activation": "relu",
			"kernel_constraint": null,
			"bias_regularizer": null,
			"name": "conv2d_2",
			"strides": [1, 1],
			"kernel_initializer": {
				"config": {
					"mode": "fan_avg",
					"scale": 1.0,
					"distribution": "uniform",
					"seed": null
				},
				"class_name": "VarianceScaling"
			},
			"trainable": true,
			"kernel_size": [3, 3],
			"bias_initializer": {
				"config": {},
				"class_name": "Zeros"
			},
			"kernel_regularizer": null,
			"use_bias": true,
			"dilation_rate": [1, 1]
		},
		"class_name": "Conv2D"
	}, {
		"config": {
			"strides": [2, 2],
			"data_format": "channels_last",
			"name": "max_pooling2d_2",
			"padding": "valid",
			"pool_size": [2, 2],
			"trainable": true
		},
		"class_name": "MaxPooling2D"
	}, {
		"config": {
			"data_format": "channels_last",
			"trainable": true,
			"name": "flatten"
		},
		"class_name": "Flatten"
	}, {
		"config": {
			"bias_initializer": {
				"config": {},
				"class_name": "Zeros"
			},
			"bias_constraint": null,
			"activation": "relu",
			"kernel_constraint": null,
			"bias_regularizer": null,
			"name": "dense",
			"activity_regularizer": null,
			"trainable": true,
			"units": 128,
			"kernel_regularizer": null,
			"use_bias": true,
			"kernel_initializer": {
				"config": {
					"mode": "fan_avg",
					"scale": 1.0,
					"distribution": "uniform",
					"seed": null
				},
				"class_name": "VarianceScaling"
			}
		},
		"class_name": "Dense"
	}, {
		"config": {
			"bias_initializer": {
				"config": {},
				"class_name": "Zeros"
			},
			"bias_constraint": null,
			"activation": "softmax",
			"kernel_constraint": null,
			"bias_regularizer": null,
			"name": "dense_1",
			"activity_regularizer": null,
			"trainable": true,
			"units": 5,
			"kernel_regularizer": null,
			"use_bias": true,
			"kernel_initializer": {
				"config": {
					"mode": "fan_avg",
					"scale": 1.0,
					"distribution": "uniform",
					"seed": null
				},
				"class_name": "VarianceScaling"
			}
		},
		"class_name": "Dense"
	}],
	"backend": "tensorflow",
	"class_name": "Sequential"
}

And I try to follow you to check the weights, there are the results. If you need my weight file or model file or hole project, I also can provide it.

python

tf.js

[theangels]

Thanks @caisq , I upgrade tensorflow version to 1.11.0 version, tfjs-converter to 0.6.2. And I retrain the 7-voice model by using the same dataset as before, try to infer by using tensorflowjs in Web application. I found it works.