Skip to content

Commit

Permalink
Use Linux fd syscalls instead of PG's File Vfd
Browse files Browse the repository at this point in the history 
During the server start, when pg_tde module is loading and it needs to
read *.map, *.dat file, InitFileAccess is yet to be called, hence Vfd
isn't ready to use. The same gonna happen during recovery. So use raw
pread/pwrite calls istead.
  • Loading branch information
@dAdAbird
dAdAbird committed Jun 14, 2024
1 parent e9805ad commit 6d4e944
Show file tree
Hide file tree
Showing 7 changed files with 261 additions and 202 deletions.
276 changes: 162 additions & 114 deletions src/access/pg_tde_tdemap.c
View file

Large diffs are not rendered by default.

16 changes: 0 additions & 16 deletions src/access/pg_tde_xlog.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,22 +40,6 @@ static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count,
static void SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char* iv_prefix);
static int XLOGChooseNumBuffers(void);

typedef enum
{
TDE_GCAT_KEY_XLOG,

/* must be last */
TDE_GCAT_KEYS_COUNT
} GlobalCatalogKeyTypes;

/* TODO: move TDEXLogEncryptBuf here*/
typedef struct EncryptionStateData
{
GenericKeyring *keyring;
/* TODO: locking */
TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT];

} EncryptionStateData;

/*
* TDE fork XLog
Expand Down
83 changes: 46 additions & 37 deletions src/catalog/tde_global_catalog.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,31 +24,34 @@
#include <openssl/err.h>
#include <sys/time.h>

#define MASTER_KEY_DEFAULT_NAME "tde-global-catalog-key"

/* TODO: not sure if we need an option of multiple master keys for the global catalog */
typedef enum
{
TDE_GCAT_KEY_XLOG,
TDE_GCAT_XLOG_KEY,

/* must be last */
TDE_GCAT_KEYS_COUNT
} GlobalCatalogKeyTypes;
} GlobalCatalogKeyTypes;

typedef struct EncryptionStateData
{
GenericKeyring *keyring;
TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT];
} EncryptionStateData;
TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT];
} EncryptionStateData;

static EncryptionStateData *EncryptionState = NULL;
static EncryptionStateData * EncryptionState = NULL;

/* GUC */
static char *KRingProviderType = NULL;
static char *KRingProviderFilePath = NULL;

static void init_gl_catalog_keys(void);
static void init_keyring(void);
static TDEMasterKey *create_master_key(const char *key_name,
GenericKeyring *keyring, Oid dbOid, Oid spcOid,
bool ensure_new_key);
static TDEMasterKey * create_master_key(const char *key_name,
GenericKeyring * keyring, Oid dbOid, Oid spcOid,
bool ensure_new_key);

void
TDEGlCatInitGUC(void)
Expand Down Expand Up @@ -81,7 +84,7 @@ TDEGlCatInitGUC(void)
Size
TDEGlCatEncStateSize(void)
{
Size size;
Size size;

size = sizeof(EncryptionStateData);
size = add_size(size, sizeof(KeyringProviders));
Expand All @@ -92,12 +95,12 @@ TDEGlCatEncStateSize(void)
void
TDEGlCatShmemInit(void)
{
bool foundBuf;
char *allocptr;
bool foundBuf;
char *allocptr;

EncryptionState = (EncryptionStateData *)
ShmemInitStruct("TDE XLog Encryption State",
TDEGlCatEncStateSize(), &foundBuf);
ShmemInitStruct("TDE XLog Encryption State",
TDEGlCatEncStateSize(), &foundBuf);

allocptr = ((char *) EncryptionState) + MAXALIGN(sizeof(EncryptionStateData));
EncryptionState->keyring = (GenericKeyring *) allocptr;
Expand All @@ -108,70 +111,76 @@ TDEGlCatShmemInit(void)
void
TDEGlCatKeyInit(void)
{
char db_map_path[MAXPGPATH] = {0};
char db_map_path[MAXPGPATH] = {0};

init_keyring();

pg_tde_set_db_file_paths(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID),
db_map_path, NULL);
db_map_path, NULL);
if (access(db_map_path, F_OK) == -1)
{
init_gl_catalog_keys();
}
else
{
/* put an internal key into the cache */
GetGlCatInternalKey(XLOG_TDE_OID);
}
}

TDEMasterKey *
TDEGetGlCatKeyFromCache(void)
{
TDEMasterKey *mkey;
mkey = &EncryptionState->master_keys[TDE_GCAT_KEY_XLOG];

mkey = &EncryptionState->master_keys[TDE_GCAT_XLOG_KEY];
if (mkey->keyLength == 0)
return NULL;

return mkey;
}

void
TDEPutGlCatKeyInCache(TDEMasterKey *mkey)
TDEPutGlCatKeyInCache(TDEMasterKey * mkey)
{
memcpy(EncryptionState->master_keys + TDE_GCAT_KEY_XLOG, mkey, sizeof(TDEMasterKey));
memcpy(EncryptionState->master_keys + TDE_GCAT_XLOG_KEY, mkey, sizeof(TDEMasterKey));
}

RelKeyData *
GetGlCatInternalKey(Oid obj_id)
{
return GetInternalKey(GLOBAL_SPACE_RLOCATOR(obj_id), EncryptionState->keyring);
return GetRelationKeyWithKeyring(GLOBAL_SPACE_RLOCATOR(obj_id), EncryptionState->keyring);
}

/* TODO: add Vault */
static void
init_keyring(void)
{
EncryptionState->keyring->type = get_keyring_provider_from_typename(KRingProviderType);
switch (EncryptionState->keyring->type)
{
case FILE_KEY_PROVIDER:
FileKeyring *kring = (FileKeyring *) EncryptionState->keyring;
FileKeyring * kring = (FileKeyring *) EncryptionState->keyring;
strncpy(kring->file_name, KRingProviderFilePath, sizeof(kring->file_name));
break;
}
}

/*
/*
* Keys are created during the cluster start only, so no locks needed here.
*/
static void
init_gl_catalog_keys(void)
{
InternalKey int_key;
RelKeyData *rel_key_data;
RelKeyData *enc_rel_key_data;
RelFileLocator *rlocator;
TDEMasterKey *mkey;
InternalKey int_key;
RelKeyData *rel_key_data;
RelKeyData *enc_rel_key_data;
RelFileLocator *rlocator;
TDEMasterKey *mkey;

mkey = create_master_key("global-catalog-master-key",
EncryptionState->keyring,
GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, false);
mkey = create_master_key(MASTER_KEY_DEFAULT_NAME,
EncryptionState->keyring,
GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, false);

memset(&int_key, 0, sizeof(InternalKey));

Expand All @@ -180,8 +189,8 @@ init_gl_catalog_keys(void)
{
ereport(FATAL,
(errcode(ERRCODE_INTERNAL_ERROR),
errmsg("could not generate internal key for \"WAL\": %s",
ERR_error_string(ERR_get_error(), NULL))));
errmsg("could not generate internal key for \"WAL\": %s",
ERR_error_string(ERR_get_error(), NULL))));
}

rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID);
Expand All @@ -193,11 +202,11 @@ init_gl_catalog_keys(void)
}

static TDEMasterKey *
create_master_key(const char *key_name, GenericKeyring *keyring,
Oid dbOid, Oid spcOid, bool ensure_new_key)
create_master_key(const char *key_name, GenericKeyring * keyring,
Oid dbOid, Oid spcOid, bool ensure_new_key)
{
TDEMasterKey *masterKey;
keyInfo *keyInfo = NULL;
TDEMasterKey *masterKey;
keyInfo *keyInfo = NULL;

masterKey = palloc(sizeof(TDEMasterKey));
masterKey->keyInfo.databaseId = dbOid;
Expand All @@ -222,4 +231,4 @@ create_master_key(const char *key_name, GenericKeyring *keyring,
memcpy(masterKey->keyData, keyInfo->data.data, keyInfo->data.len);

return masterKey;
}
}
11 changes: 8 additions & 3 deletions src/catalog/tde_master_key.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,11 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring)
LWLock *lock_cache = tde_lwlock_mk_cache();

LWLockAcquire(lock_cache, LW_SHARED);
masterKey = get_master_key_from_cache(dbOid);
/* Global catalog has its own cache */
if (spcOid == GLOBALTABLESPACE_OID)
masterKey = TDEGetGlCatKeyFromCache();
else
masterKey = get_master_key_from_cache(dbOid);
LWLockRelease(lock_cache);

if (masterKey)
Expand Down Expand Up @@ -260,7 +264,8 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring)
return NULL;
}

if (keyring == NULL) {
if (keyring == NULL)
{
keyring = GetKeyProviderByID(masterKeyInfo->keyringId);
if (keyring == NULL)
{
Expand Down Expand Up @@ -288,7 +293,7 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring)
memcpy(masterKey->keyData, keyInfo->data.data, keyInfo->data.len);
masterKey->keyLength = keyInfo->data.len;

Assert(MyDatabaseId == masterKey->keyInfo.databaseId);
Assert(dbOid == masterKey->keyInfo.databaseId);
if (spcOid == GLOBALTABLESPACE_OID)
TDEPutGlCatKeyInCache(masterKey);
else
Expand Down
2 changes: 1 addition & 1 deletion src/include/access/pg_tde_tdemap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator);
extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset);

extern RelKeyData *GetRelationKey(RelFileLocator rel);
extern RelKeyData *GetInternalKey(RelFileLocator rel, GenericKeyring *keyring);
extern RelKeyData *GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring);

extern void pg_tde_delete_tde_files(Oid dbOid, Oid spcOid);

Expand Down
6 changes: 5 additions & 1 deletion src/include/catalog/tde_global_catalog.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@
#define GLOBAL_DATA_TDE_OID 607 /* Global objects fake "db" */
#define XLOG_TDE_OID 608

#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) {GLOBALTABLESPACE_OID, 0, _obj_oid}
#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) { \
GLOBALTABLESPACE_OID, \
GLOBAL_DATA_TDE_OID, \
_obj_oid \
}

extern void TDEGlCatInitGUC(void);
extern Size TDEGlCatEncStateSize(void);
Expand Down
Loading

0 comments on commit 6d4e944

Please sign in to comment.