8226352: Improve Kerberos interop capabilities

Reviewed-by: ahgross, mullan, valeriep

src/jdk.security.jgss/share/classes/com/sun/security/sasl/gsskerb/GssKrb5Base.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -106,8 +106,12 @@ public Object getNegotiatedProperty(String propName) {
         }
 
         try {
-            MessageProp msgProp = new MessageProp(JGSS_QOP, privacy);
+            MessageProp msgProp = new MessageProp(JGSS_QOP, false);
             byte[] answer = secCtx.unwrap(incoming, start, len, msgProp);
+            if (privacy && !msgProp.getPrivacy()) {
+                throw new SaslException("Privacy not protected");
+            }
+            checkMessageProp("", msgProp);
             if (logger.isLoggable(Level.FINEST)) {
                 traceOutput(myClassName, "KRB501:Unwrap", "incoming: ",
                     incoming, start, len);
@@ -162,4 +166,20 @@ public void dispose() throws SaslException {
     protected void finalize() throws Throwable {
         dispose();
     }
+
+    void checkMessageProp(String label, MessageProp msgProp)
+            throws SaslException {
+        if (msgProp.isDuplicateToken()) {
+            throw new SaslException(label + "Duplicate token");
+        }
+        if (msgProp.isGapToken()) {
+            throw new SaslException(label + "Gap token");
+        }
+        if (msgProp.isOldToken()) {
+            throw new SaslException(label + "Old token");
+        }
+        if (msgProp.isUnseqToken()) {
+            throw new SaslException(label + "Token not in sequence");
+        }
+    }
 }

src/jdk.security.jgss/share/classes/com/sun/security/sasl/gsskerb/GssKrb5Client.java

@@ -228,8 +228,10 @@ public boolean hasInitialResponse() {
 
             // Received S1 (security layer, server max recv size)
 
+            MessageProp msgProp = new MessageProp(false);
             byte[] gssOutToken = secCtx.unwrap(challengeData, 0,
-                challengeData.length, new MessageProp(0, false));
+                challengeData.length, msgProp);
+            checkMessageProp("Handshake failure: ", msgProp);
 
             // First octet is a bit-mask specifying the protections
             // supported by the server

src/jdk.security.jgss/share/classes/com/sun/security/sasl/gsskerb/GssKrb5Server.java

@@ -252,8 +252,10 @@
         try {
             // Expecting 4 octets from client selected protection
             // and client's receive buffer size
+            MessageProp msgProp = new MessageProp(false);
             byte[] gssOutToken = secCtx.unwrap(responseData, 0,
-                responseData.length, new MessageProp(0, false));
+                responseData.length, msgProp);
+            checkMessageProp("Handshake failure: ", msgProp);
 
             if (logger.isLoggable(Level.FINER)) {
                 traceOutput(MY_CLASS_NAME, "doHandshake2",