GH-525: Fix sntrup761x25519-sha512

Because all other KEX algorithms treat the secret resulting from the
key agreement as "mpint", our key agreements all returned the "mpint"
representation of the result of the key agreement.

But sntrup761x25519-sha512 needs the raw 32 bytes of the key agreement
(curve25519-sha256).

Add a flag to XDH that determines whether it returns the raw bytes or
the "mpint" bytes.

Bug: #525

CHANGES.md

@@ -34,6 +34,8 @@
 
 ## Bug Fixes
 
+* [GH-525](https://github.com/apache/mina-sshd/issues/525) Fix sntrup761x25519-sha512 key exchange
+
 ## New Features
 
 ## Potential compatibility issues

sshd-core/src/main/java/org/apache/sshd/common/kex/BuiltinDHFactories.java

@@ -253,7 +253,7 @@ public XDH create(Object... params) throws Exception {
             if (!GenericUtils.isEmpty(params)) {
                 throw new IllegalArgumentException("No accepted parameters for " + getName());
             }
-            return new XDH(MontgomeryCurve.x25519) {
+            return new XDH(MontgomeryCurve.x25519, false) {
 
                 @Override
                 public Digest getHash() throws Exception {
@@ -274,7 +274,7 @@ public XDH create(Object... params) throws Exception {
             if (!GenericUtils.isEmpty(params)) {
                 throw new IllegalArgumentException("No accepted parameters for " + getName());
             }
-            return new XDH(MontgomeryCurve.x25519) {
+            return new XDH(MontgomeryCurve.x25519, false) {
 
                 @Override
                 public Digest getHash() throws Exception {
@@ -298,7 +298,7 @@ public XDH create(Object... params) throws Exception {
             if (!GenericUtils.isEmpty(params)) {
                 throw new IllegalArgumentException("No accepted parameters for " + getName());
             }
-            return new XDH(MontgomeryCurve.x448) {
+            return new XDH(MontgomeryCurve.x448, false) {
 
                 @Override
                 public Digest getHash() throws Exception {
@@ -322,7 +322,7 @@ public XDH create(Object... params) throws Exception {
             if (!GenericUtils.isEmpty(params)) {
                 throw new IllegalArgumentException("No accepted parameters for " + getName());
             }
-            return new XDH(MontgomeryCurve.x25519) {
+            return new XDH(MontgomeryCurve.x25519, true) {
 
                 @Override
                 public KeyEncapsulationMethod getKeyEncapsulation() {

sshd-core/src/main/java/org/apache/sshd/common/kex/XDH.java

@@ -31,11 +31,13 @@
  */
 public abstract class XDH extends AbstractDH {
 
-    protected MontgomeryCurve curve;
+    protected final MontgomeryCurve curve;
+    protected final boolean raw;
     protected byte[] f;
 
-    public XDH(MontgomeryCurve curve) throws Exception {
+    public XDH(MontgomeryCurve curve, boolean raw) throws Exception {
         this.curve = Objects.requireNonNull(curve, "No MontgomeryCurve provided");
+        this.raw = raw;
         myKeyAgree = curve.createKeyAgreement();
     }
 
@@ -77,6 +79,7 @@ public void putF(Buffer buffer, byte[] f) {
     protected byte[] calculateK() throws Exception {
         Objects.requireNonNull(f, "Missing 'f' value");
         myKeyAgree.doPhase(curve.decode(f), true);
-        return stripLeadingZeroes(myKeyAgree.generateSecret());
+        byte[] secret = myKeyAgree.generateSecret();
+        return raw ? secret : stripLeadingZeroes(secret);
     }
 }

sshd-core/src/main/java/org/apache/sshd/common/session/helpers/SessionHelper.java

@@ -758,7 +758,7 @@ protected byte[] resizeKey(
                 buffer = new ByteArrayBuffer();
             }
 
-            buffer.putMPInt(k);
+            buffer.putBytes(k);
             buffer.putRawBytes(h);
             buffer.putRawBytes(e);
             hash.update(buffer.array(), 0, buffer.available());