Implemented #6929: Add support of PKCS v.1.5 padding to RSA functions…

…, needed for backward compatibility with old systems.

doc/sql.extensions/README.builtin_functions.txt

@@ -1069,11 +1069,13 @@ Function:
     short symmetric keys which are then used in block ciphers to encrypt a message.
 
 Format:
-    RSA_ENCRYPT ( <string> KEY <public key> [LPARAM <string>] [HASH <hash>] )
+    RSA_ENCRYPT ( <string> KEY <public key> [LPARAM <string>] [HASH <hash>] [PKCS_1_5] )
         KEY should be a value, returhed by RSA_PUBLIC function.
         LPARAM is an additional system specific tag that can be applied to identify which
             system encoded the message. Default value is NULL.
         hash ::= { MD5 | SHA1 | SHA256 | SHA512 } Default is SHA256.
+        PKCS_1_5 makes engine use old padding, needed for backward compatibility.
+            Due to security reasons should NOT be used in new projects.
 
 Example:
     (tip - start running samples one by one from RSA_PRIVATE function)
@@ -1089,11 +1091,13 @@ Function:
     Decrypts using RSA private key and OAEP de-pads the resulting data.
 
 Format:
-    RSA_DECRYPT ( <string> KEY <private key> [LPARAM <string>] [HASH <hash>] )
+    RSA_DECRYPT ( <string> KEY <private key> [LPARAM <string>] [HASH <hash>] [PKCS_1_5] )
         KEY should be a value, returhed by RSA_PRIVATE function.
         LPARAM is the same variable passed to RSA_ENCRYPT. If it does not match
           what was used during encoding this function will not decrypt the packet.
         hash ::= { MD5 | SHA1 | SHA256 | SHA512 } Default is SHA256.
+        PKCS_1_5 makes engine use old padding, needed for backward compatibility.
+            Due to security reasons should NOT be used in new projects.
 
 Example:
     (tip - start running samples one by one from RSA_PRIVATE function)
@@ -1109,11 +1113,12 @@ Function:
     Performs PSS encoding of message digest to be signed and signs using RSA private key.
 
 Format:
-    RSA_SIGN_HASH ( <string> KEY <private key> [HASH <hash>] [SALT_LENGTH <smallint>] )
+    RSA_SIGN_HASH ( <string> KEY <private key> [HASH <hash>] [SALT_LENGTH <smallint>] [PKCS_1_5] )
         KEY should be a value, returhed by RSA_PRIVATE function.
         hash ::= { MD5 | SHA1 | SHA256 | SHA512 } Default is SHA256.
         SALT_LENGTH indicates the length of the desired salt, and should typically be small.
             A good value is between 8 and 16.
+        PKCS_1_5 makes engine use old padding, needed for backward compatibility.
 
 Example:
     (tip - start running samples one by one from RSA_PRIVATE function)
@@ -1130,12 +1135,13 @@ Function:
         using RSA public key.
 
 Format:
-    RSA_VERIFY_HASH ( <string> SIGNATURE <string> KEY <public key> [HASH <hash>] [SALT_LENGTH <smallint>] )
+    RSA_VERIFY_HASH ( <string> SIGNATURE <string> KEY <public key> [HASH <hash>] [SALT_LENGTH <smallint>] [PKCS_1_5] )
         SIGNATURE should be a value, returhed by RSA_SIGN function.
         KEY should be a value, returhed by RSA_PUBLIC function.
         hash ::= { MD5 | SHA1 | SHA256 | SHA512 } Default is SHA256.
         SALT_LENGTH indicates the length of the desired salt, and should typically be small.
             A good value is between 8 and 16.
+        PKCS_1_5 makes engine use old padding, needed for backward compatibility.
 
 Example:
     (tip - start running samples one by one from RSA_PRIVATE function)

src/common/keywords.cpp

@@ -358,6 +358,7 @@ static const TOK tokens[] =
 	{TOK_PASSWORD, "PASSWORD", true},
 	{TOK_PERCENT_RANK, "PERCENT_RANK", true},
 	{TOK_PI, "PI", true},
+	{TOK_PKCS_1_5, "PKCS_1_5", true},
 	{TOK_PLACING, "PLACING", true},
 	{TOK_PLAN, "PLAN", false},
 	{TOK_PLUGIN, "PLUGIN", true},

src/dsql/parse.y

@@ -680,6 +680,7 @@ using namespace Firebird;
 // tokens added for Firebird 4.0.1
 
 %token <metaNamePtr> DEBUG
+%token <metaNamePtr> PKCS_1_5
 
 // tokens added for Firebird 5.0
 
@@ -8260,25 +8261,25 @@ system_function_special_syntax
 		}
 	| POSITION '(' value_list_opt  ')'
 		{ $$ = newNode<SysFuncCallNode>(*$1, $3); }
-	| rsa_encrypt_decrypt '(' value KEY value crypt_opt_lparam crypt_opt_hash ')'
+	| rsa_encrypt_decrypt '(' value KEY value crypt_opt_lparam crypt_opt_hash crypt_opt_pkcs')'
 		{
 			$$ = newNode<SysFuncCallNode>(*$1,
 				newNode<ValueListNode>($3)->add($5)->add($6)->
-					add(MAKE_str_constant(newIntlString($7->c_str()), CS_ASCII)));
+					add(MAKE_str_constant(newIntlString($7->c_str()), CS_ASCII))->add($8));
 			$$->dsqlSpecialSyntax = true;
 		}
-	| RSA_SIGN_HASH '(' value KEY value crypt_opt_hash crypt_opt_saltlen ')'
+	| RSA_SIGN_HASH '(' value KEY value crypt_opt_hash crypt_opt_saltlen crypt_opt_pkcs ')'
 		{
 			$$ = newNode<SysFuncCallNode>(*$1,
 				newNode<ValueListNode>($3)->add($5)->
-					add(MAKE_str_constant(newIntlString($6->c_str()), CS_ASCII))->add($7));
+					add(MAKE_str_constant(newIntlString($6->c_str()), CS_ASCII))->add($7)->add($8));
 			$$->dsqlSpecialSyntax = true;
 		}
-	| RSA_VERIFY_HASH '(' value SIGNATURE value KEY value crypt_opt_hash crypt_opt_saltlen ')'
+	| RSA_VERIFY_HASH '(' value SIGNATURE value KEY value crypt_opt_hash crypt_opt_saltlen crypt_opt_pkcs ')'
 		{
 			$$ = newNode<SysFuncCallNode>(*$1,
 				newNode<ValueListNode>($3)->add($5)->add($7)->
-					add(MAKE_str_constant(newIntlString($8->c_str()), CS_ASCII))->add($9));
+					add(MAKE_str_constant(newIntlString($8->c_str()), CS_ASCII))->add($9)->add($10));
 			$$->dsqlSpecialSyntax = true;
 		}
 	| RDB_SYSTEM_PRIVILEGE '(' valid_symbol_name ')'
@@ -8306,6 +8307,14 @@ crypt_opt_lparam
 		{ $$ = $2; }
 	;
 
+%type <valueExprNode> crypt_opt_pkcs
+crypt_opt_pkcs
+	: // nothing
+		{ $$ = MAKE_const_slong(0); }
+	| PKCS_1_5
+		{ $$ = MAKE_const_slong(1); }
+	;
+
 %type <metaNamePtr> crypt_opt_hash
 crypt_opt_hash
 	: // nothing
@@ -9088,6 +9097,7 @@ non_reserved_word
 	| TRAPS
 	| ZONE
 	| DEBUG				// added in FB 4.0.1
+	| PKCS_1_5
 	| TIMEZONE_NAME		// added in FB 5.0
 	| UNICODE_CHAR
 	| UNICODE_VAL

src/jrd/SysFunction.cpp

@@ -705,11 +705,12 @@ const unsigned RSA_CRYPT_ARG_VALUE = 0;
 const unsigned RSA_CRYPT_ARG_KEY = 1;
 const unsigned RSA_CRYPT_ARG_LPARAM = 2;
 const unsigned RSA_CRYPT_ARG_HASH = 3;
-const unsigned RSA_CRYPT_ARG_MAX = 4;
+const unsigned RSA_CRYPT_ARG_PKCS_1_5 = 4;
+const unsigned RSA_CRYPT_ARG_MAX = 5;
 
 void setParamsRsaEncrypt(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc** args)
 {
-	fb_assert(argsCount == RSA_CRYPT_ARG_MAX);
+	fb_assert(argsCount == RSA_CRYPT_ARG_MAX || argsCount == RSA_CRYPT_ARG_MAX - 1);
 
 	setParamVarying(args[RSA_CRYPT_ARG_VALUE], ttype_binary);
 	setParamVarying(args[RSA_CRYPT_ARG_KEY], ttype_binary);
@@ -719,18 +720,22 @@ void setParamsRsaEncrypt(DataTypeUtilBase*, const SysFunction*, int argsCount, d
 
 	if (args[RSA_CRYPT_ARG_HASH]->dsc_length)
 		args[RSA_CRYPT_ARG_HASH]->makeVarying(args[RSA_CRYPT_ARG_HASH]->getStringLength(), ttype_binary);
+
+	if (argsCount == RSA_CRYPT_ARG_MAX)
+		args[RSA_CRYPT_ARG_PKCS_1_5]->makeShort(0);
 }
 
 
 const unsigned RSA_SIGN_ARG_VALUE = 0;
 const unsigned RSA_SIGN_ARG_KEY = 1;
 const unsigned RSA_SIGN_ARG_HASH = 2;
 const unsigned RSA_SIGN_ARG_SALTLEN = 3;
-const unsigned RSA_SIGN_ARG_MAX = 4;
+const unsigned RSA_SIGN_ARG_PKCS_1_5 = 4;
+const unsigned RSA_SIGN_ARG_MAX = 5;
 
 void setParamsRsaSign(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc** args)
 {
-	fb_assert(argsCount == RSA_SIGN_ARG_MAX);
+	fb_assert(argsCount == RSA_SIGN_ARG_MAX || argsCount == RSA_SIGN_ARG_MAX - 1);
 
 	setParamVarying(args[RSA_SIGN_ARG_VALUE], ttype_binary);
 	setParamVarying(args[RSA_SIGN_ARG_KEY], ttype_binary);
@@ -740,6 +745,9 @@ void setParamsRsaSign(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc*
 
 	if (args[RSA_SIGN_ARG_SALTLEN]->dsc_length)
 		args[RSA_SIGN_ARG_SALTLEN]->makeShort(0);
+
+	if (argsCount == RSA_SIGN_ARG_MAX)
+		args[RSA_SIGN_ARG_PKCS_1_5]->makeShort(0);
 }
 
 
@@ -748,11 +756,12 @@ const unsigned RSA_VERIFY_ARG_SIGNATURE = 1;
 const unsigned RSA_VERIFY_ARG_KEY = 2;
 const unsigned RSA_VERIFY_ARG_HASH = 3;
 const unsigned RSA_VERIFY_ARG_SALTLEN = 4;
-const unsigned RSA_VERIFY_ARG_MAX = 5;
+const unsigned RSA_VERIFY_ARG_PKCS_1_5 = 5;
+const unsigned RSA_VERIFY_ARG_MAX = 6;
 
 void setParamsRsaVerify(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc** args)
 {
-	fb_assert(argsCount == RSA_VERIFY_ARG_MAX);
+	fb_assert(argsCount == RSA_VERIFY_ARG_MAX || argsCount == RSA_VERIFY_ARG_MAX - 1);
 
 	setParamVarying(args[RSA_VERIFY_ARG_VALUE], ttype_binary);
 	setParamVarying(args[RSA_VERIFY_ARG_KEY], ttype_binary);
@@ -763,6 +772,9 @@ void setParamsRsaVerify(DataTypeUtilBase*, const SysFunction*, int argsCount, ds
 
 	if (args[RSA_VERIFY_ARG_SALTLEN]->dsc_length)
 		args[RSA_VERIFY_ARG_SALTLEN]->makeShort(0);
+
+	if (argsCount == RSA_VERIFY_ARG_MAX)
+		args[RSA_VERIFY_ARG_PKCS_1_5]->makeShort(0);
 }
 
 
@@ -1465,7 +1477,7 @@ void makeCrypt(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc*
 void makeRsaCrypt(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result,
 	int argsCount, const dsc** args)
 {
-	fb_assert(argsCount == RSA_CRYPT_ARG_MAX);
+	fb_assert(argsCount == RSA_CRYPT_ARG_MAX || argsCount == RSA_CRYPT_ARG_MAX - 1);
 
 	result->makeVarying(256, ttype_binary);
 	result->setNullable(args[0]->isNullable());
@@ -3466,14 +3478,15 @@ dsc* evlRsaEncryptDecrypt(thread_db* tdbb, const SysFunction* function, const Ne
 {
 	tomcryptInitializer();
 
-	fb_assert(args.getCount() == RSA_CRYPT_ARG_MAX);
+	fb_assert(args.getCount() == RSA_CRYPT_ARG_MAX || args.getCount() == RSA_CRYPT_ARG_MAX - 1);
 
 	jrd_req* request = tdbb->getRequest();
 
 	// parse args and check correctness
 	const dsc* dscs[RSA_CRYPT_ARG_MAX];
-	for (unsigned i = 0; i < RSA_CRYPT_ARG_MAX; ++i)
+	for (unsigned i = 0; i < args.getCount(); ++i)
 		dscs[i] = EVL_expr(tdbb, request, args[i]);
+	SSHORT pkcs15 = args.getCount() < RSA_CRYPT_ARG_MAX ? 0 : *(SSHORT*)(dscs[RSA_CRYPT_ARG_PKCS_1_5]->dsc_address);
 
 	MetaName hashName;
 	if (dscs[RSA_CRYPT_ARG_HASH])
@@ -3504,10 +3517,12 @@ dsc* evlRsaEncryptDecrypt(thread_db* tdbb, const SysFunction* function, const Ne
 	UCharBuffer outBuf;
 	int stat = 0;
 	int cryptRc = encryptFlag ?
-		rsa_encrypt_key(data.getBytes(), data.getLength(), outBuf.getBuffer(outlen), &outlen,
-			lParam.getBytes(), lParam.getLength(), prng().getState(), prng().getIndex(), hash, &rsaKey) :
-		rsa_decrypt_key(data.getBytes(), data.getLength(), outBuf.getBuffer(outlen), &outlen,
-			lParam.getBytes(), lParam.getLength(), hash, &stat, &rsaKey);
+		rsa_encrypt_key_ex(data.getBytes(), data.getLength(), outBuf.getBuffer(outlen), &outlen,
+			lParam.getBytes(), lParam.getLength(), prng().getState(), prng().getIndex(), hash,
+			pkcs15 ? LTC_PKCS_1_V1_5 : LTC_PKCS_1_OAEP, &rsaKey) :
+		rsa_decrypt_key_ex(data.getBytes(), data.getLength(), outBuf.getBuffer(outlen), &outlen,
+			lParam.getBytes(), lParam.getLength(), hash,
+			pkcs15 ? LTC_PKCS_1_V1_5 : LTC_PKCS_1_OAEP, &stat, &rsaKey);
 	rsa_free(&rsaKey);
 	tomCheck(cryptRc, Arg::Gds(encryptFlag ? isc_tom_crypt_cip : isc_tom_decrypt_cip) << "RSA");
 	if ((!encryptFlag) && (!stat))
@@ -3603,15 +3618,17 @@ dsc* evlRsaSign(thread_db* tdbb, const SysFunction* function, const NestValueArr
 {
 	tomcryptInitializer();
 
-	fb_assert(args.getCount() == RSA_SIGN_ARG_MAX);
+	fb_assert(args.getCount() == RSA_SIGN_ARG_MAX || args.getCount() == RSA_SIGN_ARG_MAX - 1);
 
 	jrd_req* request = tdbb->getRequest();
 
 	// parse args and check correctness
 	const dsc* dscs[RSA_SIGN_ARG_MAX];
-	for (unsigned i = 0; i < RSA_SIGN_ARG_MAX; ++i)
+	for (unsigned i = 0; i < args.getCount(); ++i)
 		dscs[i] = EVL_expr(tdbb, request, args[i]);
 
+	SSHORT pkcs15 = args.getCount() < RSA_SIGN_ARG_MAX ? 0 : *(SSHORT*)(dscs[RSA_SIGN_ARG_PKCS_1_5]->dsc_address);
+
 	MetaName hashName;
 	if (dscs[RSA_SIGN_ARG_HASH])
 		MOV_get_metaname(tdbb, dscs[RSA_SIGN_ARG_HASH], hashName);
@@ -3643,8 +3660,8 @@ dsc* evlRsaSign(thread_db* tdbb, const SysFunction* function, const NestValueArr
 
 	unsigned long signLen = 1024;
 	UCharBuffer sign;
-	int cryptRc = rsa_sign_hash(data.getBytes(), data.getLength(), sign.getBuffer(signLen), &signLen,
-		prng().getState(), prng().getIndex(), hash, saltLength, &rsaKey);
+	int cryptRc = rsa_sign_hash_ex(data.getBytes(), data.getLength(), sign.getBuffer(signLen), &signLen,
+		pkcs15 ? LTC_PKCS_1_V1_5 : LTC_PKCS_1_PSS, prng().getState(), prng().getIndex(), hash, saltLength, &rsaKey);
 	rsa_free(&rsaKey);
 	tomCheck(cryptRc, Arg::Gds(isc_tom_rsa_sign));
 
@@ -3670,14 +3687,15 @@ dsc* evlRsaVerify(thread_db* tdbb, const SysFunction* function, const NestValueA
 {
 	tomcryptInitializer();
 
-	fb_assert(args.getCount() == RSA_VERIFY_ARG_MAX);
+	fb_assert(args.getCount() == RSA_VERIFY_ARG_MAX || args.getCount() == RSA_VERIFY_ARG_MAX - 1);
 
 	jrd_req* request = tdbb->getRequest();
 
 	// parse args and check correctness
 	const dsc* dscs[RSA_VERIFY_ARG_MAX];
-	for (unsigned i = 0; i < RSA_VERIFY_ARG_MAX; ++i)
+	for (unsigned i = 0; i < args.getCount(); ++i)
 		dscs[i] = EVL_expr(tdbb, request, args[i]);
+	SSHORT pkcs15 = args.getCount() < RSA_VERIFY_ARG_MAX ? 0 : *(SSHORT*)(dscs[RSA_VERIFY_ARG_PKCS_1_5]->dsc_address);
 
 	MetaName hashName;
 	if (dscs[RSA_VERIFY_ARG_HASH])
@@ -3713,8 +3731,8 @@ dsc* evlRsaVerify(thread_db* tdbb, const SysFunction* function, const NestValueA
 	}
 
 	int state = 0;
-	int cryptRc = rsa_verify_hash(sign.getBytes(), sign.getLength(), data.getBytes(), data.getLength(),
-		hash, saltLength, &state, &rsaKey);
+	int cryptRc = rsa_verify_hash_ex(sign.getBytes(), sign.getLength(), data.getBytes(), data.getLength(),
+		pkcs15 ? LTC_PKCS_1_V1_5 : LTC_PKCS_1_PSS, hash, saltLength, &state, &rsaKey);
 	rsa_free(&rsaKey);
 	if (cryptRc != CRYPT_INVALID_PACKET)
 		tomCheck(cryptRc, Arg::Gds(isc_tom_rsa_verify));
@@ -6553,8 +6571,8 @@ const SysFunction SysFunction::functions[] =
 		{"CRYPT_HASH", 2, 2, setParamsHash, makeHash, evlHash, NULL},
 		{"DATEADD", 3, 3, setParamsDateAdd, makeDateAdd, evlDateAdd, NULL},
 		{"DATEDIFF", 3, 3, setParamsDateDiff, makeInt64Result, evlDateDiff, NULL},
-		{"DECRYPT", 7, 7, setParamsEncrypt, makeCrypt, evlDecrypt, NULL},
-		{"ENCRYPT", 7, 7, setParamsEncrypt, makeCrypt, evlEncrypt, NULL},
+		{"DECRYPT", CRYPT_ARG_MAX, CRYPT_ARG_MAX, setParamsEncrypt, makeCrypt, evlDecrypt, NULL},
+		{"ENCRYPT", CRYPT_ARG_MAX, CRYPT_ARG_MAX, setParamsEncrypt, makeCrypt, evlEncrypt, NULL},
 		{"EXP", 1, 1, setParamsDblDec, makeDblDecResult, evlExp, NULL},
 		{"FIRST_DAY", 2, 2, setParamsFirstLastDay, makeFirstLastDayResult, evlFirstLastDay, (void*) funFirstDay},
 		{"FLOOR", 1, 1, setParamsDblDec, makeCeilFloor, evlFloor, NULL},
@@ -6589,12 +6607,12 @@ const SysFunction SysFunction::functions[] =
 		{"RIGHT", 2, 2, setParamsSecondInteger, makeLeftRight, evlRight, NULL},
 		{"ROUND", 1, 2, setParamsRoundTrunc, makeRound, evlRound, NULL},
 		{"RPAD", 2, 3, setParamsSecondInteger, makePad, evlPad, (void*) funRPad},
-		{"RSA_DECRYPT", 4, 4, setParamsRsaEncrypt, makeRsaCrypt, evlRsaDecrypt, NULL},
-		{"RSA_ENCRYPT", 4, 4, setParamsRsaEncrypt, makeRsaCrypt, evlRsaEncrypt, NULL},
+		{"RSA_DECRYPT", RSA_CRYPT_ARG_MAX, RSA_CRYPT_ARG_MAX, setParamsRsaEncrypt, makeRsaCrypt, evlRsaDecrypt, NULL},
+		{"RSA_ENCRYPT", RSA_CRYPT_ARG_MAX, RSA_CRYPT_ARG_MAX, setParamsRsaEncrypt, makeRsaCrypt, evlRsaEncrypt, NULL},
 		{"RSA_PRIVATE", 1, 1, setParamsInteger, makeRsaPrivate, evlRsaPrivate, NULL},
 		{"RSA_PUBLIC", 1, 1, setParamsRsaPublic, makeRsaPublic, evlRsaPublic, NULL},
-		{"RSA_SIGN_HASH", 4, 4, setParamsRsaSign, makeRsaSign, evlRsaSign, NULL},
-		{"RSA_VERIFY_HASH", 5, 5, setParamsRsaVerify, makeBoolResult, evlRsaVerify, NULL},
+		{"RSA_SIGN_HASH", RSA_SIGN_ARG_MAX, RSA_SIGN_ARG_MAX, setParamsRsaSign, makeRsaSign, evlRsaSign, NULL},
+		{"RSA_VERIFY_HASH", RSA_VERIFY_ARG_MAX, RSA_VERIFY_ARG_MAX, setParamsRsaVerify, makeBoolResult, evlRsaVerify, NULL},
 		{"SIGN", 1, 1, setParamsDblDec, makeShortResult, evlSign, NULL},
 		{"SIN", 1, 1, setParamsDouble, makeDoubleResult, evlStdMath, (void*) trfSin},
 		{"SINH", 1, 1, setParamsDouble, makeDoubleResult, evlStdMath, (void*) trfSinh},