Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Locally connected 1d layer #5891

Merged
merged 6 commits into from Jul 13, 2018
Merged

Locally connected 1d layer #5891

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7928b2f
1D convolution utils
maxpumperla Jul 12, 2018
3252b8f
locally connected 1D layer (samediff)
maxpumperla Jul 12, 2018
116da43
refac conv utils
maxpumperla Jul 12, 2018
5dd53c5
compute 1d output type
maxpumperla Jul 12, 2018
7067b2f
ff test
maxpumperla Jul 12, 2018
511176c
fix grad check test
maxpumperla Jul 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
120 changes: 95 additions & 25 deletions ...earning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java
View file
Open in desktop
Expand Up @@ -34,6 +34,76 @@ public class CNN1DGradientCheckTest extends BaseDL4JTest {
Nd4j.setDataType(DataBuffer.Type.DOUBLE);
}

@Test
public void testCnn1DWithLocallyConnected1D() {
Nd4j.getRandom().setSeed(1337);

int[] minibatchSizes = {2, 3};
int length = 7;
int convNIn = 2;
int convNOut1 = 3;
int convNOut2 = 4;
int finalNOut = 4;

int[] kernels = {1};
int stride = 1;
int padding = 0;

Activation[] activations = {Activation.SIGMOID};

for (Activation afn : activations) {
for (int minibatchSize : minibatchSizes) {
for (int kernel : kernels) {
INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length});
INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, length);
for (int i = 0; i < minibatchSize; i++) {
for (int j = 0; j < length; j++) {
labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0);
}
}

MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.updater(new NoOp()).weightInit(WeightInit.DISTRIBUTION)
.dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
.layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
.build())
.layer(new LocallyConnected1D.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2).hasBias(false)
.build())
.layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(finalNOut).build())
.setInputType(InputType.recurrent(convNIn, length)).build();

String json = conf.toJson();
MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
assertEquals(conf, c2);

MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();

String msg = "Minibatch=" + minibatchSize + ", activationFn="
+ afn + ", kernel = " + kernel;

if (PRINT_RESULTS) {
System.out.println(msg);
for (int j = 0; j < net.getnLayers(); j++)
System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
}

boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

assertTrue(msg, gradOK);

TestUtils.testModelSerialization(net);
}

}
}
}


@Test
public void testCnn1DWithCropping1D() {
Nd4j.getRandom().setSeed(1337);
Expand All @@ -55,18 +125,18 @@ public void testCnn1DWithCropping1D() {

Activation[] activations = {Activation.SIGMOID};
SubsamplingLayer.PoolingType[] poolingTypes =
new SubsamplingLayer.PoolingType[] {SubsamplingLayer.PoolingType.MAX,
new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

for (Activation afn : activations) {
for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
for (int minibatchSize : minibatchSizes) {
for (int kernel : kernels) {
INDArray input = Nd4j.rand(new int[] {minibatchSize, convNIn, length});
INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length});
INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, croppedLength);
for (int i = 0; i < minibatchSize; i++) {
for (int j = 0; j < croppedLength; j++) {
labels.putScalar(new int[] {i, i % finalNOut, j}, 1.0);
labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0);
}
}

Expand Down Expand Up @@ -135,18 +205,18 @@ public void testCnn1DWithZeroPadding1D() {

Activation[] activations = {Activation.SIGMOID};
SubsamplingLayer.PoolingType[] poolingTypes =
new SubsamplingLayer.PoolingType[] {SubsamplingLayer.PoolingType.MAX,
new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

for (Activation afn : activations) {
for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
for (int minibatchSize : minibatchSizes) {
for (int kernel : kernels) {
INDArray input = Nd4j.rand(new int[] {minibatchSize, convNIn, length});
INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length});
INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, paddedLength);
for (int i = 0; i < minibatchSize; i++) {
for (int j = 0; j < paddedLength; j++) {
labels.putScalar(new int[] {i, i % finalNOut, j}, 1.0);
labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0);
}
}

Expand Down Expand Up @@ -213,35 +283,35 @@ public void testCnn1DWithSubsampling1D() {

Activation[] activations = {Activation.SIGMOID, Activation.TANH};
SubsamplingLayer.PoolingType[] poolingTypes =
new SubsamplingLayer.PoolingType[] {SubsamplingLayer.PoolingType.MAX,
SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};
new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

for (Activation afn : activations) {
for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
for (int minibatchSize : minibatchSizes) {
for (int kernel : kernels) {
INDArray input = Nd4j.rand(new int[] {minibatchSize, convNIn, length});
INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length});
INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, length);
for (int i = 0; i < minibatchSize; i++) {
for (int j = 0; j < length; j++) {
labels.putScalar(new int[] {i, i % finalNOut, j}, 1.0);
labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0);
}
}

MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.updater(new NoOp()).weightInit(WeightInit.DISTRIBUTION)
.dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
.layer(0, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
.build())
.layer(1, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2)
.build())
.layer(2, new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel)
.stride(stride).padding(padding).pnorm(pnorm).build())
.layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(finalNOut).build())
.setInputType(InputType.recurrent(convNIn, length)).build();
.updater(new NoOp()).weightInit(WeightInit.DISTRIBUTION)
.dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
.layer(0, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
.build())
.layer(1, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2)
.build())
.layer(2, new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel)
.stride(stride).padding(padding).pnorm(pnorm).build())
.layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(finalNOut).build())
.setInputType(InputType.recurrent(convNIn, length)).build();

String json = conf.toJson();
MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
Expand All @@ -251,7 +321,7 @@ public void testCnn1DWithSubsampling1D() {
net.init();

String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
+ afn + ", kernel = " + kernel;
+ afn + ", kernel = " + kernel;

if (PRINT_RESULTS) {
System.out.println(msg);
Expand All @@ -260,7 +330,7 @@ public void testCnn1DWithSubsampling1D() {
}

boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

assertTrue(msg, gradOK);
TestUtils.testModelSerialization(net);
Expand Down