Fix QAT_HW & QAT_SW init to fallback to OpenSSL_SW on failure

- Fallback to QAT_SW when QAT_HW init Failure and Fallback
  to OpenSSL_SW when both the initialization fails.

Signed-off-by: BinZhu <bin.zhu@intel.com>

e_qat.c

@@ -162,7 +162,8 @@ const char *engine_qat_name =
     "Reference implementation of QAT crypto engine(qat_sw) v1.1.0";
 #endif
 unsigned int engine_inited = 0;
-
+int fallback_to_openssl = 0;
+int fallback_to_qat_sw = 0; /* QAT HW initialize fail, offload to QAT SW. */
 int qat_hw_offload = 0;
 int qat_sw_offload = 0;
 int qat_hw_rsa_offload = 0;
@@ -562,17 +563,23 @@ int qat_engine_init(ENGINE *e)
 #ifdef QAT_HW
     if (qat_hw_offload) {
         if (!qat_hw_init(e)) {
-            fprintf(stderr, "QAT_HW initialization Failed\n");
+# ifdef QAT_SW /* Co-Existence mode: Don't return failure when QAT HW initialization Failed. */
+            fallback_to_qat_sw = 1;
+            WARN("QAT HW initialization Failed, switching to QAT SW.\n");
+# else
+            fprintf(stderr, "QAT HW initialization Failed.\n");
+            qat_pthread_mutex_unlock();
             return 0;
+# endif
         }
     }
 #endif
 
 #ifdef QAT_SW
     if (qat_sw_offload) {
         if (!qat_sw_init(e)) {
-            fprintf(stderr, "QAT_SW initialization Failed\n");
-            return 0;
+            WARN("QAT SW initialization Failed, switching to OpenSSL.\n");
+            fallback_to_openssl = 1;
         }
     }
 #endif
@@ -624,6 +631,8 @@ int qat_engine_finish_int(ENGINE *e, int reset_globals)
         enable_heuristic_polling = 0;
         qat_hw_offload = 0;
         qat_sw_offload = 0;
+        fallback_to_openssl = 0;
+        fallback_to_qat_sw = 0;
     }
     qat_pthread_mutex_unlock();
     CRYPTO_CLOSE_QAT_LOG();

e_qat.h

@@ -382,7 +382,8 @@ typedef struct {
 extern const char *engine_qat_id;
 extern const char *engine_qat_name;
 extern unsigned int engine_inited;
-
+extern int fallback_to_openssl;
+extern int fallback_to_qat_sw; /* QAT HW initialization fail, offload to QAT SW. */
 extern int qat_hw_offload;
 extern int qat_sw_offload;
 extern int qat_hw_rsa_offload;

qat_hw_ec.c

@@ -726,7 +726,8 @@ int qat_engine_ecdh_compute_key(unsigned char **out,
             ((curve_name == NID_X9_62_prime256v1) || 
              (curve_name == NID_secp384r1))) {
             DEBUG("- Switched to QAT_SW mode\n");
-            --qat_sw_ecdh_derive_req;
+            if (qat_sw_ecdh_derive_req > 0)
+                --qat_sw_ecdh_derive_req;
             return mb_ecdh_compute_key(out, outlen, pub_key, ecdh);
         }
 #endif
@@ -930,7 +931,8 @@ int qat_ecdh_generate_key(EC_KEY *ecdh)
             ((curve_name == NID_X9_62_prime256v1) || 
              (curve_name == NID_secp384r1))) {
             DEBUG("- Switched to QAT_SW mode\n");
-            --qat_sw_ecdh_keygen_req;
+            if (qat_sw_ecdh_keygen_req > 0)
+                --qat_sw_ecdh_keygen_req;
             return mb_ecdh_generate_key(ecdh);
         }
 #endif
@@ -1098,9 +1100,10 @@ int qat_ecdsa_sign(int type, const unsigned char *dgst, int dlen,
 
     curve_name = EC_GROUP_get_curve_name(group);
     if (qat_ecdsa_coexist) {
-        /* Use QAT_SW if the curve is P256 */
-        if (curve_name == NID_X9_62_prime256v1)
+        /* Use QAT SW if the curve is P256 or QAT device not enough.*/
+        if (curve_name == NID_X9_62_prime256v1 || qat_get_qat_offload_disabled()) {
             return mb_ecdsa_sign(type, dgst, dlen, sig, siglen, kinv, r, eckey);
+        }
 
         if ((qat_sw_ecdsa_sign_req > 0) && (curve_name == NID_secp384r1)) {
             --qat_sw_ecdsa_sign_req;
@@ -1188,6 +1191,12 @@ ECDSA_SIG *qat_ecdsa_do_sign(const unsigned char *dgst, int dgst_len,
         return ret;
     }
 
+    /* For the scenario of QAT HW initialization fail. */
+    if (fallback_to_qat_sw || fallback_to_openssl) {
+        WARN("- Fallback to software mode.\n");
+        return (*sign_sig_pfunc)(dgst, dgst_len, in_kinv, in_r, eckey);
+    }
+
     if (qat_get_qat_offload_disabled()) {
         fallback = 1;
         goto err;

qat_hw_ecx.c

@@ -562,7 +562,8 @@ static int qat_pkey_ecx_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey, int type)
 #ifdef ENABLE_QAT_SW_ECX
         if (qat_ecx_coexist && !is_ecx_448) {
             DEBUG("- Switched to QAT_SW mode\n");
-            --qat_sw_ecx_keygen_req;
+            if (qat_sw_ecx_keygen_req > 0)
+                --qat_sw_ecx_keygen_req;
             return multibuff_x25519_keygen(ctx, pkey);
         }
 #endif
@@ -984,7 +985,8 @@ int qat_pkey_ecx_derive25519(EVP_PKEY_CTX *ctx, unsigned char *key, size_t *keyl
 #ifdef ENABLE_QAT_SW_ECX
         if (qat_ecx_coexist) {
             DEBUG("- Switched to QAT_SW mode\n");
-            --qat_sw_ecx_derive_req;
+            if (qat_sw_ecx_derive_req > 0)
+                --qat_sw_ecx_derive_req;
             return multibuff_x25519_derive(ctx, key, keylen);
         }
 #endif

qat_hw_init.c

@@ -98,9 +98,13 @@
 #include "icp_sal_user.h"
 #include "icp_sal_poll.h"
 
+/* 1. QAT HW mode, Check if QAT Engine Offload disabled. If yes, fallback to OpenSSL.
+   2. Co-Existence mode, if QAT HW initialization fail, fallback to QAT SW.
+   3. Sw fallback on, if instance not available, fallback to OpenSSL.
+ */
 int qat_get_qat_offload_disabled(void)
 {
-    if (disable_qat_offload ||
+    if (disable_qat_offload || fallback_to_qat_sw ||
         (qat_get_sw_fallback_enabled() && !is_any_device_available()))
         return 1;
     else
@@ -604,7 +608,9 @@ int qat_hw_init(ENGINE *e)
     if (CPA_STATUS_SUCCESS !=
         icp_sal_userStart(ICPConfigSectionName_libcrypto)) {
         WARN("icp_sal_userStart failed\n");
+# ifndef QAT_SW /* Co-Existence mode: Don't print error message. */
         QATerr(QAT_F_QAT_HW_INIT, QAT_R_ICP_SAL_USERSTART_FAIL);
+# endif
         pthread_key_delete(thread_local_variables);
         qat_pthread_mutex_unlock();
         return 0;
@@ -615,15 +621,15 @@ int qat_hw_init(ENGINE *e)
     if (CPA_STATUS_SUCCESS != status) {
         WARN("cpaCyGetNumInstances failed, status=%d\n", status);
         QATerr(QAT_F_QAT_HW_INIT, QAT_R_GET_NUM_INSTANCE_FAILURE);
-        qat_pthread_mutex_unlock();
-        qat_engine_finish(e);
+        qat_hw_finish_int(e, QAT_RESET_GLOBALS);
         return 0;
     }
     if (!qat_num_instances) {
         WARN("No crypto instances found\n");
+# ifndef QAT_SW /* Co-Existence mode: Don't print error message. */
         QATerr(QAT_F_QAT_HW_INIT, QAT_R_INSTANCE_UNAVAILABLE);
-        qat_pthread_mutex_unlock();
-        qat_engine_finish(e);
+# endif
+        qat_hw_finish_int(e, QAT_RESET_GLOBALS);
         return 0;
     }
 
@@ -636,8 +642,7 @@ int qat_hw_init(ENGINE *e)
     if (NULL == qat_instance_handles) {
         WARN("OPENSSL_zalloc() failed for instance handles.\n");
         QATerr(QAT_F_QAT_HW_INIT, QAT_R_INSTANCE_HANDLE_MALLOC_FAILURE);
-        qat_pthread_mutex_unlock();
-        qat_engine_finish(e);
+        qat_hw_finish_int(e, QAT_RESET_GLOBALS);
         return 0;
     }
 
@@ -646,8 +651,7 @@ int qat_hw_init(ENGINE *e)
     if (CPA_STATUS_SUCCESS != status) {
         WARN("cpaCyGetInstances failed, status=%d\n", status);
         QATerr(QAT_F_QAT_HW_INIT, QAT_R_GET_INSTANCE_FAILURE);
-        qat_pthread_mutex_unlock();
-        qat_engine_finish(e);
+        qat_hw_finish_int(e, QAT_RESET_GLOBALS);
         return 0;
     }
 
@@ -663,8 +667,7 @@ int qat_hw_init(ENGINE *e)
             if (-1 == internal_efd) {
                 WARN("Error creating epoll fd\n");
                 QATerr(QAT_F_QAT_HW_INIT, QAT_R_EPOLL_CREATE_FAILURE);
-                qat_pthread_mutex_unlock();
-                qat_engine_finish(e);
+                qat_hw_finish_int(e, QAT_RESET_GLOBALS);
                 return 0;
             }
 
@@ -676,8 +679,7 @@ int qat_hw_init(ENGINE *e)
                 if (CPA_STATUS_FAIL == status) {
                     WARN("Error getting file descriptor for instance\n");
                     QATerr(QAT_F_QAT_HW_INIT, QAT_R_GET_FILE_DESCRIPTOR_FAILURE);
-                    qat_pthread_mutex_unlock();
-                    qat_engine_finish(e);
+                    qat_hw_finish_int(e, QAT_RESET_GLOBALS);
                     return 0;
                 }
                 /*   Make the file descriptor non-blocking */
@@ -689,8 +691,7 @@ int qat_hw_init(ENGINE *e)
                     WARN("Failed to set engine_fd as NON BLOCKING\n");
                     QATerr(QAT_F_QAT_HW_INIT,
                            QAT_R_SET_FILE_DESCRIPTOR_NONBLOCKING_FAILURE);
-                    qat_pthread_mutex_unlock();
-                    qat_engine_finish(e);
+                    qat_hw_finish_int(e, QAT_RESET_GLOBALS);
                     return 0;
                 }
 
@@ -701,8 +702,7 @@ int qat_hw_init(ENGINE *e)
                               &eng_epoll_events[instNum])) {
                     WARN("Error adding fd to epoll\n");
                     QATerr(QAT_F_QAT_HW_INIT, QAT_R_EPOLL_CTL_FAILURE);
-                    qat_pthread_mutex_unlock();
-                    qat_engine_finish(e);
+                    qat_hw_finish_int(e, QAT_RESET_GLOBALS);
                     return 0;
                 }
             }
@@ -718,8 +718,7 @@ int qat_hw_init(ENGINE *e)
         if (CPA_STATUS_SUCCESS != status ) {
             WARN("cpaCyInstanceGetInfo2 failed. status = %d\n", status);
             QATerr(QAT_F_QAT_HW_INIT, QAT_R_GET_INSTANCE_INFO_FAILURE);
-            qat_pthread_mutex_unlock();
-            qat_engine_finish(e);
+            qat_hw_finish_int(e, QAT_RESET_GLOBALS);
             return 0;
         }
 
@@ -735,8 +734,7 @@ int qat_hw_init(ENGINE *e)
         if (CPA_STATUS_SUCCESS != status) {
             WARN("cpaCySetAddressTranslation failed, status=%d\n", status);
             QATerr(QAT_F_QAT_HW_INIT, QAT_R_SET_ADDRESS_TRANSLATION_FAILURE);
-            qat_pthread_mutex_unlock();
-            qat_engine_finish(e);
+            qat_hw_finish_int(e, QAT_RESET_GLOBALS);
             return 0;
         }
 
@@ -745,8 +743,7 @@ int qat_hw_init(ENGINE *e)
         if (CPA_STATUS_SUCCESS != status) {
             WARN("cpaCyStartInstance failed, status=%d\n", status);
             QATerr(QAT_F_QAT_HW_INIT, QAT_R_START_INSTANCE_FAILURE);
-            qat_pthread_mutex_unlock();
-            qat_engine_finish(e);
+            qat_hw_finish_int(e, QAT_RESET_GLOBALS);
             return 0;
         }
 
@@ -762,17 +759,15 @@ int qat_hw_init(ENGINE *e)
             if (CPA_STATUS_SUCCESS != status) {
                 WARN("cpaCyInstanceSetNotificationCb failed, status=%d\n", status);
                 QATerr(QAT_F_QAT_HW_INIT, QAT_R_SET_NOTIFICATION_CALLBACK_FAILURE);
-                qat_pthread_mutex_unlock();
-                qat_engine_finish(e);
+                qat_hw_finish_int(e, QAT_RESET_GLOBALS);
                 return 0;
             }
         }
 #endif
     }
 
     if (!qat_remap_instances()) {
-        qat_pthread_mutex_unlock();
-        qat_engine_finish(e);
+        qat_hw_finish_int(e, QAT_RESET_GLOBALS);
         return 0;
     }
 
@@ -813,17 +808,15 @@ int qat_hw_init(ENGINE *e)
             if (ret_pthread_sigmask != 0) {
                 WARN("pthread_sigmask error\n");
                 QATerr(QAT_F_QAT_HW_INIT, QAT_R_POLLING_THREAD_SIGMASK_FAILURE);
-                qat_pthread_mutex_unlock();
-                qat_engine_finish(e);
+                qat_hw_finish_int(e, QAT_RESET_GLOBALS);
                 return 0;
             }
         }
 
         if (sem_init(&hw_polling_thread_sem, 0, 0) != 0) {
             WARN("hw sem_init failed!\n");
             QATerr(QAT_F_QAT_HW_INIT, QAT_R_POLLING_THREAD_SEM_INIT_FAILURE);
-            qat_pthread_mutex_unlock();
-            qat_engine_finish(e);
+            qat_hw_finish_int(e, QAT_RESET_GLOBALS);
             return 0;
         }
 
@@ -836,15 +829,13 @@ int qat_hw_init(ENGINE *e)
             WARN("Creation of polling thread failed\n");
             QATerr(QAT_F_QAT_HW_INIT, QAT_R_POLLING_THREAD_CREATE_FAILURE);
             qat_polling_thread = pthread_self();
-            qat_pthread_mutex_unlock();
-            qat_engine_finish(e);
+            qat_hw_finish_int(e, QAT_RESET_GLOBALS);
             return 0;
         }
         if (qat_adjust_thread_affinity(qat_polling_thread) == 0) {
             WARN("Setting polling thread affinity failed\n");
             QATerr(QAT_F_QAT_HW_INIT, QAT_R_SET_POLLING_THREAD_AFFINITY_FAILURE);
-            qat_pthread_mutex_unlock();
-            qat_engine_finish(e);
+            qat_hw_finish_int(e, QAT_RESET_GLOBALS);
             return 0;
         }
         if (!qat_is_event_driven()) {

qat_hw_rsa.c

@@ -1069,7 +1069,8 @@ int qat_rsa_priv_enc(int flen, const unsigned char *from, unsigned char *to,
 #ifdef ENABLE_QAT_SW_RSA
         if (qat_rsa_coexist) {
             DEBUG("- Switch to QAT SW mode.\n");
-            --qat_sw_rsa_priv_req;
+            if (qat_sw_rsa_priv_req > 0)
+                --qat_sw_rsa_priv_req;
             return multibuff_rsa_priv_enc(flen, from, to, rsa, padding);
         }
 #endif
@@ -1276,7 +1277,8 @@ int qat_rsa_priv_dec(int flen, const unsigned char *from,
 #ifdef ENABLE_QAT_SW_RSA
         if (qat_rsa_coexist) {
             DEBUG("- Switch to QAT_SW mode.\n");
-            --qat_sw_rsa_priv_req;
+            if (qat_sw_rsa_priv_req > 0)
+                --qat_sw_rsa_priv_req;
             return multibuff_rsa_priv_dec(flen, from, to, rsa, padding);
         }
 #endif
@@ -1383,7 +1385,8 @@ int qat_rsa_pub_enc(int flen, const unsigned char *from,
 #ifdef ENABLE_QAT_SW_RSA
         if (qat_rsa_coexist) {
             DEBUG("- Switch to QAT_SW mode.\n");
-            --qat_sw_rsa_pub_req;
+            if (qat_sw_rsa_pub_req > 0)
+                --qat_sw_rsa_pub_req;
             return multibuff_rsa_pub_enc(flen, from, to, rsa, padding);
         }
 #endif
@@ -1517,7 +1520,8 @@ int qat_rsa_pub_dec(int flen, const unsigned char *from, unsigned char *to,
 #ifdef ENABLE_QAT_SW_RSA
         if (qat_rsa_coexist) {
             DEBUG("- Switch to QAT_SW mode.\n");
-            --qat_sw_rsa_pub_req;
+            if (qat_sw_rsa_pub_req > 0)
+                --qat_sw_rsa_pub_req;
             return multibuff_rsa_pub_dec(flen, from, to, rsa, padding);
         }
 #endif