Merge branch 'master' into patch-2

src/itest/docker-image/authorized_keys

@@ -3,6 +3,7 @@ ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHQiZm0w
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDAdJiRkkBM8yC8seTEoAn2PfwbLKrkcahZ0xxPoWICJ root@sshj
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ8ww4hJG/gHJYdkjTTBDF1GNz+228nuWprPV+NbQauA ajvanerp@Heimdall.local
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOaWrwt3drIOjeBq2LSHRavxAT7ja2f+5soOUJl/zKSI ajvanerp@Heimdall.xebialabs.com
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICYfPGSYFOHuSzTJ67H0ynvKJDfgDmwPOj7iJaLGbIBi sshjtest@TranceLove
 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAoZ9l6Tkm2aL1tSBy2yw4xU5s8BE9MfqS/4J7DzvsYJxF6oQmTIjmStuhH/CT7UjuDtKXdXZUsIhKtafiizxGO8kHSzKDeitpth2RSr8ddMzZKyD6RNs7MfsgjA3UTtrrSrCXEY6O43S2cnuJrWzkPxtwxaQ3zOvDbS2tiulzyq0VzYmuhA/a4CyuQtJBuu+P2oqmu6pU/VB6IzONpvBvYbNPsH1WDmP7zko5wHPihXPCliztspKxS4DRtOZ7BGXyvg44UmIy0Kf4jOkaBV/eCCA4qH7ZHz71/5ceMOpszPcNOEmLGGYhwI+P3OuGMpkrSAv1f8IY6R8spZNncP6UaQ== no-passphrase
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDKRyZAtOJJfAhPU6xE6ZXY564vwErAI3n3Yn4lTHL9bxev9Ily6eCqPLcV0WbSV04pztngFn9MjT7yb8mcXheHpIaWEH569sMpmpOtyfn4p68SceuXBGyyPGMIcfOTknkASd1JYSD4EPkd9rZmCzcx3vEnLu8ChnA/G221xSVQ5VC/jD/c/CgNUayhQ+xbn57qHKKtZwfTa21QmwIabGYJNwlVjlKTCdddeVnZfKqKrG7cxHQApsxd21rhM9IT/C/f4Y/Tx3WUUVeam0iZ265oiPHoPALqJIWSQIUheRYAxYAQqJwSQ0Or9MM8XXun2Iy3RUSGk6eIvrCsFbNURsHNs7Pu0UnpYv6FZ3vCkFep/1pAT6fQvY7pDOOWDHKXArD4watc9gIWaQBH73wDW/KgBcnMRSoGWgQjsYqIamP4oV1+HqUI3lRAsXZaX+eiBGt3+3A5KebP27UJ1YUwhwlzs7wzTKaCu0OaL+hOsP1F2AxAa995bgFksMd23645ux3YCJKXG4sGpJ1Z/Hs49K72gv+QjLZVxXqY623c8+3OUhlixqoEFd4iG7UMc5a552ch/VA+jaspmLZoFhPz99aBRVb1oCSPxSwLw+Q/wxv6pZmT+14rqTzY2farjU53hM+CsUPh7dnWXhGG7RuA5wCdeOXOYjuksfzAoHIZhPqTgQ== ajvanerp@Heimdall.local
 ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBMvfRYSe44VQGwxexOMibcM3+fWeUP1jrBofOxFDRRrzRF8dK/vll2svqTPXMRnITnT1UoemEcB5OHtvH4hzfh/HFeDxJ5S7UncYxoClTSa8MeMFG2Zj9CoUZs1SHbwSGg== root@sshj

src/itest/groovy/com/hierynomus/sshj/IntegrationSpec.groovy

@@ -80,6 +80,7 @@ class IntegrationSpec extends Specification {
         "id_ecdsa_opensshv1" | null
         "id_ed25519_opensshv1" | null
         "id_ed25519_opensshv1_aes256cbc.pem" | "foobar"
+        "id_ed25519_opensshv1_aes128cbc.pem" | "sshjtest"
         "id_ed25519_opensshv1_protected" | "sshjtest"
         "id_rsa" | null
         "id_rsa_opensshv1" | null

src/itest/resources/keyfiles/id_ed25519_opensshv1_aes128cbc.pem

@@ -0,0 +1,3 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAACmFlczEyOC1jYmMAAAAGYmNyeXB0AAAAGAAAABDfrL8SxDyrkNlsJdAmc7Z0AAAAEAAAAAEAAAAzAAAAC3NzaC1lZDI1NTE5AAAAICYfPGSYFOHuSzTJ67H0ynvKJDfgDmwPOj7iJaLGbIBiAAAAkLVqaDIfs+sPBNyy7ytdLnP/xH7Nt5FIXx3Upw6wKuMGdBzbFQLcvu60Le+SFP3uUfXE8TcHramXbH0n+UBMW6raCAKOkHUU1BtrKxPG1eKU/LBx3Bk5FxyKm7fo0XsCUmqSVK25EHOJfYq1QwIbWICkvQUNu+2Hg8/MQKoFJMentI+GqjdaG76f6Wf+aj9UwA==
+-----END OPENSSH PRIVATE KEY-----

src/main/java/com/hierynomus/sshj/userauth/keyprovider/OpenSSHKeyV1KeyFile.java

@@ -177,11 +177,11 @@ private void initializeCipher(String kdfName, byte[] kdfOptions, Cipher cipher)
                 Arrays.fill(charBuffer.array(), '\u0000');
                 Arrays.fill(byteBuffer.array(), (byte) 0);
             }
-            byte[] keyiv = new byte[48];
+            byte[] keyiv = new byte[cipher.getIVSize()+ cipher.getBlockSize()];
             new BCrypt().pbkdf(passphrase, opts.readBytes(), opts.readUInt32AsInt(), keyiv);
             Arrays.fill(passphrase, (byte) 0);
-            byte[] key = Arrays.copyOfRange(keyiv, 0, 32);
-            byte[] iv = Arrays.copyOfRange(keyiv, 32, 48);
+            byte[] key = Arrays.copyOfRange(keyiv, 0, cipher.getBlockSize());
+            byte[] iv = Arrays.copyOfRange(keyiv, cipher.getBlockSize(), cipher.getIVSize() + cipher.getBlockSize());
             cipher.init(Cipher.Mode.Decrypt, key, iv);
         } else {
             throw new IllegalStateException("No support for KDF '" + kdfName + "'.");
@@ -193,6 +193,8 @@ private Cipher createCipher(String cipherName) {
             return BlockCiphers.AES256CTR().create();
         } else if (cipherName.equals(BlockCiphers.AES256CBC().getName())) {
             return BlockCiphers.AES256CBC().create();
+        } else if (cipherName.equals(BlockCiphers.AES128CBC().getName())) {
+            return BlockCiphers.AES128CBC().create();
         }
         throw new IllegalStateException("Cipher '" + cipherName + "' not currently implemented for openssh-key-v1 format");
     }

src/main/java/net/schmizz/sshj/sftp/RemoteFile.java

@@ -220,16 +220,42 @@ public int read(byte[] into, int off, int len) throws IOException {
 
     public class ReadAheadRemoteFileInputStream
             extends InputStream {
+        private class UnconfirmedRead {
+            private final long offset;
+            private final Promise<Response, SFTPException> promise;
+            private final int length;
+
+            private UnconfirmedRead(long offset, int length, Promise<Response, SFTPException> promise) {
+                this.offset = offset;
+                this.length = length;
+                this.promise = promise;
+            }
+
+            UnconfirmedRead(long offset, int length) throws IOException {
+                this(offset, length, RemoteFile.this.asyncRead(offset, length));
+            }
+
+            public long getOffset() {
+                return offset;
+            }
+
+            public Promise<Response, SFTPException> getPromise() {
+                return promise;
+            }
+
+            public int getLength() {
+                return length;
+            }
+        }
 
         private final byte[] b = new byte[1];
 
         private final int maxUnconfirmedReads;
         private final long readAheadLimit;
-        private final Queue<Promise<Response, SFTPException>> unconfirmedReads = new LinkedList<Promise<Response, SFTPException>>();
-        private final Queue<Long> unconfirmedReadOffsets = new LinkedList<Long>();
+        private final Queue<UnconfirmedRead> unconfirmedReads = new LinkedList<>();
 
-        private long requestOffset;
-        private long responseOffset;
+        private long currentOffset;
+        private int maxReadLength = Integer.MAX_VALUE;
         private boolean eof;
 
         public ReadAheadRemoteFileInputStream(int maxUnconfirmedReads) {
@@ -247,28 +273,42 @@ public ReadAheadRemoteFileInputStream(int maxUnconfirmedReads, long fileOffset,
             assert 0 <= fileOffset;
 
             this.maxUnconfirmedReads = maxUnconfirmedReads;
-            this.requestOffset = this.responseOffset = fileOffset;
+            this.currentOffset = fileOffset;
             this.readAheadLimit = readAheadLimit > 0 ? fileOffset + readAheadLimit : Long.MAX_VALUE;
         }
 
         private ByteArrayInputStream pending = new ByteArrayInputStream(new byte[0]);
 
         private boolean retrieveUnconfirmedRead(boolean blocking) throws IOException {
-            if (unconfirmedReads.size() <= 0) {
+            final UnconfirmedRead unconfirmedRead = unconfirmedReads.peek();
+            if (unconfirmedRead == null || !blocking && !unconfirmedRead.getPromise().isDelivered()) {
                 return false;
             }
+            unconfirmedReads.remove(unconfirmedRead);
 
-            if (!blocking && !unconfirmedReads.peek().isDelivered()) {
-                return false;
-            }
-
-            unconfirmedReadOffsets.remove();
-            final Response res = unconfirmedReads.remove().retrieve(requester.getTimeoutMs(), TimeUnit.MILLISECONDS);
+            final Response res = unconfirmedRead.promise.retrieve(requester.getTimeoutMs(), TimeUnit.MILLISECONDS);
             switch (res.getType()) {
                 case DATA:
                     int recvLen = res.readUInt32AsInt();
-                    responseOffset += recvLen;
-                    pending = new ByteArrayInputStream(res.array(), res.rpos(), recvLen);
+                    if (unconfirmedRead.offset == currentOffset) {
+                        currentOffset += recvLen;
+                        pending = new ByteArrayInputStream(res.array(), res.rpos(), recvLen);
+
+                        if (recvLen < unconfirmedRead.length) {
+                            // The server returned a packet smaller than the client had requested.
+                            // It can be caused by at least one of the following:
+                            // * The file has been read fully. Then, few futile read requests can be sent during
+                            //   the next read(), but the file will be downloaded correctly anyway.
+                            // * The server shapes the request length. Then, the read window will be adjusted,
+                            //   and all further read-ahead requests won't be shaped.
+                            // * The file on the server is not a regular file, it is something like fifo.
+                            //   Then, the window will shrink, and the client will start reading the file slower than it
+                            //   hypothetically can. It must be a rare case, and it is not worth implementing a sort of
+                            //   congestion control algorithm here.
+                            maxReadLength = recvLen;
+                            unconfirmedReads.clear();
+                        }
+                    }
                     break;
 
                 case STATUS:
@@ -296,49 +336,24 @@ public int read(byte[] into, int off, int len) throws IOException {
                 // we also need to go here for len <= 0, because pending may be at
                 // EOF in which case it would return -1 instead of 0
 
+                long requestOffset = currentOffset;
                 while (unconfirmedReads.size() <= maxUnconfirmedReads) {
                     // Send read requests as long as there is no EOF and we have not reached the maximum parallelism
-                    int reqLen = Math.max(1024, len); // don't be shy!
+                    int reqLen = Math.min(Math.max(1024, len), maxReadLength);
                     if (readAheadLimit > requestOffset) {
                         long remaining = readAheadLimit - requestOffset;
                         if (reqLen > remaining) {
                             reqLen = (int) remaining;
                         }
                     }
-                    unconfirmedReads.add(RemoteFile.this.asyncRead(requestOffset, reqLen));
-                    unconfirmedReadOffsets.add(requestOffset);
+                    unconfirmedReads.add(new UnconfirmedRead(requestOffset, reqLen));
                     requestOffset += reqLen;
                     if (requestOffset >= readAheadLimit) {
                         break;
                     }
                 }
 
-                long nextOffset = unconfirmedReadOffsets.peek();
-                if (responseOffset != nextOffset) {
-
-                    // the server could not give us all the data we needed, so
-                    // we try to fill the gap synchronously
-
-                    assert responseOffset < nextOffset;
-                    assert 0 < (nextOffset - responseOffset);
-                    assert (nextOffset - responseOffset) <= Integer.MAX_VALUE;
-
-                    byte[] buf = new byte[(int) (nextOffset - responseOffset)];
-                    int recvLen = RemoteFile.this.read(responseOffset, buf, 0, buf.length);
-
-                    if (recvLen < 0) {
-                        eof = true;
-                        return -1;
-                    }
-
-                    if (0 == recvLen) {
-                        // avoid infinite loops
-                        throw new SFTPException("Unexpected response size (0), bailing out");
-                    }
-
-                    responseOffset += recvLen;
-                    pending = new ByteArrayInputStream(buf, 0, recvLen);
-                } else if (!retrieveUnconfirmedRead(true /*blocking*/)) {
+                if (!retrieveUnconfirmedRead(true /*blocking*/)) {
 
                     // this may happen if we change prefetch strategy
                     // currently, we should never get here...

src/test/java/com/hierynomus/sshj/sftp/RemoteFileTest.java

@@ -17,22 +17,24 @@
 
 import com.hierynomus.sshj.test.SshFixture;
 import net.schmizz.sshj.SSHClient;
+import net.schmizz.sshj.common.ByteArrayUtils;
 import net.schmizz.sshj.sftp.OpenMode;
 import net.schmizz.sshj.sftp.RemoteFile;
 import net.schmizz.sshj.sftp.SFTPEngine;
 import net.schmizz.sshj.sftp.SFTPException;
+import org.apache.sshd.common.util.io.IoUtils;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
+import java.security.SecureRandom;
 import java.util.EnumSet;
 import java.util.Random;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 public class RemoteFileTest {
@@ -174,4 +176,53 @@ public void limitedReadAheadInputStream() throws IOException {
 
         assertThat("The written and received data should match", data, equalTo(test2));
     }
+
+    @Test
+    public void shouldReadCorrectlyWhenWrappedInBufferedStream_FullSizeBuffer() throws IOException {
+        doTestShouldReadCorrectlyWhenWrappedInBufferedStream(1024 * 1024, 1024 * 1024);
+    }
+
+    @Test
+    public void shouldReadCorrectlyWhenWrappedInBufferedStream_HalfSizeBuffer() throws IOException {
+        doTestShouldReadCorrectlyWhenWrappedInBufferedStream(1024 * 1024, 512 * 1024);
+    }
+
+    @Test
+    public void shouldReadCorrectlyWhenWrappedInBufferedStream_QuarterSizeBuffer() throws IOException {
+        doTestShouldReadCorrectlyWhenWrappedInBufferedStream(1024 * 1024, 256 * 1024);
+    }
+
+    @Test
+    public void shouldReadCorrectlyWhenWrappedInBufferedStream_SmallSizeBuffer() throws IOException {
+        doTestShouldReadCorrectlyWhenWrappedInBufferedStream(1024 * 1024, 1024);
+    }
+
+    private void doTestShouldReadCorrectlyWhenWrappedInBufferedStream(int fileSize, int bufferSize) throws IOException {
+        SSHClient ssh = fixture.setupConnectedDefaultClient();
+        ssh.authPassword("test", "test");
+        SFTPEngine sftp = new SFTPEngine(ssh).init();
+
+        final byte[] expected = new byte[fileSize];
+        new SecureRandom(new byte[] { 31 }).nextBytes(expected);
+
+        File file = temp.newFile("shouldReadCorrectlyWhenWrappedInBufferedStream.bin");
+        try (OutputStream fStream = new FileOutputStream(file)) {
+            IoUtils.copy(new ByteArrayInputStream(expected), fStream);
+        }
+
+        RemoteFile rf = sftp.open(file.getPath());
+        final byte[] actual;
+        try (InputStream inputStream = new BufferedInputStream(
+                rf.new ReadAheadRemoteFileInputStream(10),
+                bufferSize)
+        ) {
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();
+            IoUtils.copy(inputStream, baos, expected.length);
+            actual = baos.toByteArray();
+        }
+
+        assertEquals("The file should be fully read", expected.length, actual.length);
+        assertThat("The file should be read correctly",
+                ByteArrayUtils.equals(expected, 0, actual, 0, expected.length));
+    }
 }

src/test/java/net/schmizz/sshj/keyprovider/OpenSSHKeyFileTest.java

@@ -247,6 +247,12 @@ public void shouldLoadProtectedED25519PrivateKeyAes256CBC() throws IOException {
         checkOpenSSHKeyV1("src/test/resources/keytypes/ed25519_aes256cbc.pem", "foobar", true);
     }
 
+    @Test
+    public void shouldLoadProtectedED25519PrivateKeyAes128CBC() throws IOException {
+        checkOpenSSHKeyV1("src/test/resources/keytypes/ed25519_aes128cbc.pem", "sshjtest", false);
+        checkOpenSSHKeyV1("src/test/resources/keytypes/ed25519_aes128cbc.pem", "sshjtest", true);
+    }
+
     @Test(expected = KeyDecryptionFailedException.class)
     public void shouldFailOnIncorrectPassphraseAfterRetries() throws IOException {
         OpenSSHKeyV1KeyFile keyFile = new OpenSSHKeyV1KeyFile();

src/test/resources/keytypes/ed25519_aes128cbc.pem

@@ -0,0 +1,3 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAACmFlczEyOC1jYmMAAAAGYmNyeXB0AAAAGAAAABDfrL8SxDyrkNlsJdAmc7Z0AAAAEAAAAAEAAAAzAAAAC3NzaC1lZDI1NTE5AAAAICYfPGSYFOHuSzTJ67H0ynvKJDfgDmwPOj7iJaLGbIBiAAAAkLVqaDIfs+sPBNyy7ytdLnP/xH7Nt5FIXx3Upw6wKuMGdBzbFQLcvu60Le+SFP3uUfXE8TcHramXbH0n+UBMW6raCAKOkHUU1BtrKxPG1eKU/LBx3Bk5FxyKm7fo0XsCUmqSVK25EHOJfYq1QwIbWICkvQUNu+2Hg8/MQKoFJMentI+GqjdaG76f6Wf+aj9UwA==
+-----END OPENSSH PRIVATE KEY-----

src/test/resources/keytypes/ed25519_aes128cbc.pem.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICYfPGSYFOHuSzTJ67H0ynvKJDfgDmwPOj7iJaLGbIBi sshjtest@TranceLove