Permalink
Browse files

Post-improvement for CORE-5970: added 3 useful related helper functio…

…ns: base64_encode/decode and crc32
  • Loading branch information...
AlexPeshkoff committed Jan 5, 2019
1 parent 61a905e commit 38823d01843c84100a4499b0762af82d8199ad23
Showing with 235 additions and 13 deletions.
  1. +29 −0 doc/sql.extensions/README.builtin_functions.txt
  2. +9 −0 src/dsql/parse.y
  3. +179 −13 src/jrd/SysFunction.cpp
  4. +5 −0 src/jrd/blb.cpp
  5. +10 −0 src/jrd/blb.h
  6. +3 −0 src/yvalve/keywords.cpp
@@ -177,6 +177,21 @@ Example:
select atanh(x) from y;


-----------------------------
BASE64_ENCODE / BASE64_DECODE
-----------------------------

Function:
Encodes / decodes input data to / from BASE64 representation. Works with character strings and blobs.

Format:
BASE64_ENCODE( <binary data> )
BASE64_DECODE( <base64 data> )

Example:
select base64_encode(public_key) from clients;


-------
BIN_AND
-------
@@ -346,6 +361,20 @@ Example:
select cot(x) from y;


-----
CRC32
-----

Function:
Returns CRC-32 with the polynomial 0x04C11DB7. Accepts argument of any type.

Format:
CRC32( <any value> )

Example:
select crc32(job_title) from job;


-------
DATEADD
-------
@@ -448,6 +448,8 @@ using namespace Firebird;
%token <metaNamePtr> ASIN
%token <metaNamePtr> ATAN
%token <metaNamePtr> ATAN2
%token <metaNamePtr> BASE64_DECODE
%token <metaNamePtr> BASE64_ENCODE
%token <metaNamePtr> BIN_AND
%token <metaNamePtr> BIN_OR
%token <metaNamePtr> BIN_SHL
@@ -458,6 +460,7 @@ using namespace Firebird;
%token <metaNamePtr> COS
%token <metaNamePtr> COSH
%token <metaNamePtr> COT
%token <metaNamePtr> CRC32
%token <metaNamePtr> DATEADD
%token <metaNamePtr> DATEDIFF
%token <metaNamePtr> DECODE
@@ -7881,6 +7884,8 @@ system_function_std_syntax
| ATAN
| ATAN2
| ATANH
| BASE64_DECODE
| BASE64_ENCODE
| BIN_AND
| BIN_NOT
| BIN_OR
@@ -7892,6 +7897,7 @@ system_function_std_syntax
| COS
| COSH
| COT
| CRC32
| EXP
| FLOOR
| GEN_UUID
@@ -8813,6 +8819,9 @@ non_reserved_word
| RSA_VERIFY
| SALT_LENGTH
| SIGNATURE
| BASE64_DECODE

This comment has been minimized.

@asfernandes

asfernandes Jan 5, 2019

Member

We used to have per-version alphabetically ordered list of keywords, but now we have mixed per-version, per-group, and no order at all.

| BASE64_ENCODE
| CRC32
;
%%
@@ -218,10 +218,10 @@ void makeBin(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* r
void makeBinShift(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeCeilFloor(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeDateAdd(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeEncrypt(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeDecode64(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeEncode64(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeDecrypt(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeRsaEncrypt(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeRsaDecrypt(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeEncrypt(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeFirstLastDayResult(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeGetSetContext(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeGetTranCN(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
@@ -234,6 +234,8 @@ void makePi(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* re
void makeReplace(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeReverse(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeRound(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeRsaDecrypt(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeRsaEncrypt(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeRsaPrivate(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeRsaPublic(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
void makeRsaSign(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
@@ -253,8 +255,11 @@ dsc* evlBin(thread_db* tdbb, const SysFunction* function, const NestValueArray&
dsc* evlBinShift(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
dsc* evlCeil(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
dsc* evlCharToUuid(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
dsc* evlCrc32(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
dsc* evlDateDiff(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
dsc* evlDecode64(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
dsc* evlEncode64(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
dsc* evlDecrypt(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
dsc* evlEncrypt(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
dsc* evlRsaDecrypt(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
@@ -1228,6 +1233,56 @@ void makeHash(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc*
}


void raise(const char* x)
{
(Arg::Gds(isc_random) << x).raise();
}


unsigned decodeLen(unsigned len)
{
if (len % 4 || !len)
raise("Invalid parameter length");
len = len / 4 * 3;
return len;
}


void makeDecode64(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args)
{
fb_assert(argsCount == 1);
if (args[0]->isBlob())
result->makeBlob(isc_blob_untyped, ttype_binary);
else if (args[0]->isText())
result->makeVarying(decodeLen(args[0]->getStringLength()), ttype_binary);
else
raise("Invalid parameter datatype - need string or blob");

result->setNullable(args[0]->isNullable());
}


unsigned encodeLen(unsigned len)
{
len = (len + 2) / 3 * 4;
return len;
}


void makeEncode64(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args)
{
fb_assert(argsCount == 1);
if (args[0]->isBlob())
result->makeBlob(isc_blob_text, ttype_ascii);
else if (args[0]->isText())
result->makeVarying(encodeLen(args[0]->dsc_length), ttype_ascii);
else
raise("Invalid parameter datatype - need string or blob");

result->setNullable(args[0]->isNullable());
}


void makeEncrypt(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result,
int argsCount, const dsc** args)
{
@@ -2361,12 +2416,6 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr
return &impure->vlu_desc;
}

void raise(const char* x)
{
(Arg::Gds(isc_random) << x).raise();
}


// Prepare tomcrypt library

class TomcryptInitializer
@@ -2449,7 +2498,9 @@ class PseudoRandom
InitInstance<PseudoRandom> prng;


// Data exchanger between tommath and firebird
// Data exchange between tommath and firebird

const UCHAR streamBpb[] = {isc_bpb_version1, isc_bpb_type, 1, isc_bpb_type_stream};

class DataPipe
{
@@ -2477,11 +2528,11 @@ class DataPipe

try
{
const UCHAR bpb[] = {isc_bpb_version1, isc_bpb_type, 1, isc_bpb_type_stream};
const UCHAR streamBpb[] = {isc_bpb_version1, isc_bpb_type, 1, isc_bpb_type_stream};
newBlob = blb::create2(tdbb, tdbb->getRequest()->req_transaction, &impure->vlu_misc.vlu_bid,
sizeof(bpb), bpb);
sizeof(streamBpb), streamBpb);
blob = blb::open2(tdbb, tdbb->getRequest()->req_transaction, reinterpret_cast<bid*>(desc->dsc_address),
sizeof(bpb), bpb);
sizeof(streamBpb), streamBpb);

ptr = inBuf.getBuffer(BLOB_STEP);
len = blob->BLB_get_data(tdbb, inBuf.begin(), inBuf.getCount(), false);
@@ -2910,6 +2961,118 @@ dsc* evlDecrypt(thread_db* tdbb, const SysFunction* function, const NestValueArr
}


dsc* evlEncodeDecode64(thread_db* tdbb, bool encodeFlag, const SysFunction* function, const NestValueArray& args, impure_value* impure)
{
const dsc* arg = EVL_expr(tdbb, tdbb->getRequest(), args[0]);
if (!arg) // return NULL if value is NULL
return NULL;

UCharBuffer in;
if (arg->isBlob())
{
AutoPtr<blb> blob(blb::open2(tdbb, tdbb->getRequest()->req_transaction, reinterpret_cast<const bid*>(arg->dsc_address),
sizeof(streamBpb), streamBpb));

UCHAR buf[4096];
in.clear();
for(;;)
{
const unsigned l = blob->BLB_get_data(tdbb, buf, sizeof buf, false);
if (!l)
break;
in.append(buf, l);
}

blob->BLB_close(tdbb);
blob.release();
}
else
{
unsigned len;
const UCHAR* ptr = CVT_get_bytes(arg, len);
in.assign(ptr, len);
}

UCharBuffer out;
unsigned long outLen = encodeFlag ? encodeLen(in.getCount()) + 1 : decodeLen(in.getCount());
auto* func = encodeFlag ? base64_encode : base64_decode;
auto* msg = encodeFlag ? "encoding BASE64" : "decoding BASE64";
tomCheck(func(in.begin(), in.getCount(), out.getBuffer(outLen), &outLen), msg);
out.resize(outLen);

dsc result;
if (arg->isBlob())
{
AutoPtr<blb> blob(blb::create2(tdbb, tdbb->getRequest()->req_transaction, &impure->vlu_misc.vlu_bid,
sizeof(streamBpb), streamBpb));
blob->BLB_put_data(tdbb, out.begin(), out.getCount());
blob->BLB_close(tdbb);
blob.release();

result.makeBlob(encodeFlag ? isc_blob_text : isc_blob_untyped, encodeFlag ? ttype_ascii : ttype_binary,
(ISC_QUAD*)&impure->vlu_misc.vlu_bid);
}
else
result.makeText(out.getCount(), encodeFlag ? ttype_ascii : ttype_binary, const_cast<UCHAR*>(out.begin()));

EVL_make_value(tdbb, &result, impure);
return &impure->vlu_desc;
}

dsc* evlDecode64(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure)
{
return evlEncodeDecode64(tdbb, false, function, args, impure);
}

dsc* evlEncode64(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure)
{
return evlEncodeDecode64(tdbb, true, function, args, impure);
}


dsc* evlCrc32(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure)
{
crc32_state ctx;
crc32_init(&ctx);

const dsc* arg = EVL_expr(tdbb, tdbb->getRequest(), args[0]);
if (!arg) // return NULL if value is NULL
return NULL;

if (arg->isBlob())
{
blb* blob = blb::open2(tdbb, tdbb->getRequest()->req_transaction, reinterpret_cast<const bid*>(arg->dsc_address),
sizeof(streamBpb), streamBpb);

UCHAR buf[4096];
for(;;)
{
const unsigned l = blob->BLB_get_data(tdbb, buf, sizeof buf, false);
if (!l)
break;
crc32_update(&ctx, buf, l);
}

blob->BLB_close(tdbb);
}
else
{
unsigned len;
const UCHAR* ptr = CVT_get_bytes(arg, len);
crc32_update(&ctx, ptr, len);
}

SLONG hash;
crc32_finish(&ctx, &hash, sizeof hash);

dsc result;
result.makeLong(0, &hash);
EVL_make_value(tdbb, &result, impure);

return &impure->vlu_desc;
}


dsc* evlRsaEncryptDecrypt(thread_db* tdbb, const SysFunction* function, const NestValueArray& args,
impure_value* impure, bool encryptFlag)
{
@@ -5687,6 +5850,9 @@ const SysFunction SysFunction::functions[] =
{"ATAN", 1, 1, setParamsDouble, makeDoubleResult, evlStdMath, (void*) trfAtan},
{"ATANH", 1, 1, setParamsDouble, makeDoubleResult, evlStdMath, (void*) trfAtanh},
{"ATAN2", 2, 2, setParamsDouble, makeDoubleResult, evlAtan2, NULL},
{"BASE64_DECODE", 1, 1, NULL, makeDecode64, evlDecode64, NULL},
{"BASE64_ENCODE", 1, 1, NULL, makeEncode64, evlEncode64, NULL},
{"CRC32", 1, 1, NULL, makeLongResult, evlCrc32, NULL},
{"BIN_AND", 2, -1, setParamsInteger, makeBin, evlBin, (void*) funBinAnd},
{"BIN_NOT", 1, 1, setParamsInteger, makeBin, evlBin, (void*) funBinNot},
{"BIN_OR", 2, -1, setParamsInteger, makeBin, evlBin, (void*) funBinOr},
@@ -2969,3 +2969,8 @@ void blb::storeToPage(USHORT* length, Firebird::Array<UCHAR>& buffer, const UCHA
}
}
}

void blb::BLB_cancel()
{
BLB_cancel(JRD_get_thread_data());
}
@@ -33,6 +33,7 @@
#include "../jrd/EngineInterface.h"
#include "../common/classes/array.h"
#include "../common/classes/File.h"
#include "../common/classes/auto.h"

#include "firebird/Interface.h"
#include "../common/classes/ImplementHelper.h"
@@ -97,6 +98,7 @@ class blb : public pool_alloc<type_blb>
// end inline

void BLB_cancel(thread_db* tdbb);
void BLB_cancel();
void BLB_check_well_formed(thread_db*, const dsc* desc);
bool BLB_close(thread_db*);
static blb* create(thread_db*, jrd_tra*, bid*);
@@ -245,4 +247,12 @@ inline USHORT blb::getMaxSegment() const

} //namespace Jrd


template <>
inline void Firebird::SimpleDelete<Jrd::blb>::clear(Jrd::blb* b)
{
if (b)
b->BLB_cancel();
}

#endif // JRD_BLB_H

This comment has been minimized.

@hvlad

hvlad Jan 8, 2019

Member

In case you missed it - this broke the Travis build:

/home/travis/build/FirebirdSQL/firebird/src/jrd/../jrd/../jrd/blb.h:252:64: error: specialization of ‘template static void Firebird::SimpleDelete::clear(What*)’ in different namespace [-fpermissive]
inline void Firebird::SimpleDeleteJrd::blb::clear(Jrd::blb* b)

MSVC builds are OK

This comment has been minimized.

@asfernandes

asfernandes Jan 8, 2019

Member

Not sure it's due to very old gcc from Ubuntu trusty, but travis seems to support xenial version.

This comment has been minimized.

@AlexPeshkoff

AlexPeshkoff Jan 8, 2019

Member

Ubuntu always used to have 'strange' gcc (remember a version where asm did not work correctly) but let's better try to support it too. I've committed trivial fix which should help.

This comment has been minimized.

@hvlad

hvlad Jan 8, 2019

Member

Thanks, seems it helps ;)

Oops, something went wrong.

0 comments on commit 38823d0

Please sign in to comment.