Skip to content
Permalink
Browse files

Add Perl interface to C PgQuery object.

This validates that all current queries work with the new interface and removes the dependency on DBD::Pg.
  • Loading branch information...
dwsteele committed Jul 25, 2019
1 parent 415542b commit d8ca0e5c5bc2af6b85b20f48b40b83762d415b7a
@@ -458,24 +458,24 @@
</execute-list>

<!-- Perl installation -->
<p><backrest/> contains embedded Perl which requires some additional modules.</p>
<p><backrest/> contains embedded Perl which requires some additional packages.</p>

<execute-list host="{[br-install-host]}">
<title>Install required Perl packages</title>

<execute if="{[os-type-is-debian]}" user="root" pre="y">
<exe-cmd>apt-get install libdbd-pg-perl</exe-cmd>
<exe-cmd>apt-get install perl</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>

<execute if="{[os-type-is-centos6]}" user="root" pre="y">
<exe-cmd>yum install perl perl-Time-HiRes perl-parent perl-JSON
perl-Digest-SHA perl-DBD-Pg</exe-cmd>
perl-Digest-SHA</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>

<execute if="{[os-type-is-centos7]}" user="root" pre="y">
<exe-cmd>yum install perl perl-Time-HiRes perl-Digest-SHA perl-DBD-Pg perl-JSON-PP</exe-cmd>
<exe-cmd>yum install perl perl-Time-HiRes perl-Digest-SHA perl-JSON-PP</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>
</execute-list>
@@ -6,13 +6,13 @@ package pgBackRest::Db;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';

use DBD::Pg ':async';
use DBI;
use Exporter qw(import);
our @EXPORT = qw();
use Fcntl qw(O_RDONLY);
use File::Basename qw(dirname);
use JSON::PP;

use pgBackRest::DbVersion;
use pgBackRest::Common::Exception;
@@ -126,10 +126,10 @@ sub DESTROY
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->DESTROY');

if (defined($self->{hDb}))
if (defined($self->{oDb}))
{
$self->{hDb}->disconnect();
undef($self->{hDb});
$self->{oDb}->close();
undef($self->{oDb});
}

# Return from function and log return values if any
@@ -167,63 +167,46 @@ sub connect
# Else run locally
else
{
if (!defined($self->{hDb}))
if (!defined($self->{oDb}))
{
# Connect to the db
my $strDbName = 'postgres';
my $strDbSocketPath = cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $self->{iRemoteIdx}), false);
$self->{oDb} = new pgBackRest::LibC::PgClient(
cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $self->{iRemoteIdx}), false),
cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $self->{iRemoteIdx})), 'postgres',
cfgOption(CFGOPT_DB_TIMEOUT) * 1000);

# Make sure the socket path is absolute
if (defined($strDbSocketPath) && $strDbSocketPath !~ /^\//)
if ($bWarnOnError)
{
confess &log(ERROR, "'${strDbSocketPath}' is not valid for '" . cfgOptionName(CFGOPT_PG_SOCKET_PATH) . "' option:" .
" path must be absolute", ERROR_OPTION_INVALID_VALUE);
}

# Construct the URI
my $strDbUri =
"dbi:Pg:dbname=${strDbName};port=" . cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $self->{iRemoteIdx})) .
(defined($strDbSocketPath) ? ";host=${strDbSocketPath}" : '');

logDebugMisc
(
$strOperation, undef,
{name => 'strDbUri', value => $strDbUri},
);

$self->{hDb} = DBI->connect($strDbUri, undef, undef,
{AutoCommit => 1, RaiseError => 0, PrintError => 0, Warn => 0});

# If db handle is not valid then check error
if (!$self->{hDb})
{
# Throw an error unless a warning was requested
if (!$bWarnOnError)
eval
{
confess &log(ERROR, $DBI::errstr, ERROR_DB_CONNECT);
$self->{oDb}->open();
return true;
}
or do
{
&log(WARN, exceptionMessage($EVAL_ERROR));
$bResult = false;

# Log a warning
&log(WARN, $DBI::errstr);

$bResult = false;
undef($self->{hDb});
undef($self->{oDb});
}
}
else
{
$self->{oDb}->open();
}

if (defined($self->{oDb}))
{
my ($fDbVersion) = $self->versionGet();

if ($fDbVersion >= PG_VERSION_APPLICATION_NAME)
{
# Set application name for monitoring and debugging
$self->{hDb}->do(
$self->{oDb}->query(
"set application_name = '" . PROJECT_NAME . ' [' .
(cfgOptionValid(CFGOPT_COMMAND) ? cfgOption(CFGOPT_COMMAND) : cfgCommandName(cfgCommandGet())) . "]'")
or confess &log(ERROR, $self->{hDb}->errstr, ERROR_DB_QUERY);
(cfgOptionValid(CFGOPT_COMMAND) ? cfgOption(CFGOPT_COMMAND) : cfgCommandName(cfgCommandGet())) . "]'");

# Clear search path to prevent possible function overrides
$self->{hDb}->do("set search_path = 'pg_catalog'")
or confess &log(ERROR, $self->{hDb}->errstr, ERROR_DB_QUERY);
$self->{oDb}->query("set search_path = 'pg_catalog'");
}
}
}
@@ -273,73 +256,11 @@ sub executeSql
else
{
$self->connect();
my $strResult = $self->{oDb}->query($strSql);

# Prepare the query
my $hStatement = $self->{hDb}->prepare($strSql, {pg_async => PG_ASYNC})
or confess &log(ERROR, $DBI::errstr . ":\n${strSql}", ERROR_DB_QUERY);

# Execute the query
$hStatement->execute()
or confess &log(ERROR, $DBI::errstr. ":\n${strSql}", ERROR_DB_QUERY);

# Wait for the query to return
my $oWait = waitInit(cfgOption(CFGOPT_DB_TIMEOUT));
my $bTimeout = true;

do
{
# Is the statement done?
if ($hStatement->pg_ready())
{
# return now if there is no result expected
if (!$bResult)
{
return \@stryResult;
}

if (!$hStatement->pg_result())
{
# Return if the error should be ignored
if ($bIgnoreError)
{
return \@stryResult;
}

# Else report it
confess &log(ERROR, $DBI::errstr . ":\n${strSql}", ERROR_DB_QUERY);
}

# Get rows and return them
my @stryRow;

do
{
# Get next row
@stryRow = $hStatement->fetchrow_array;

# If the row has data then add it to the result
if (@stryRow)
{
push(@{$stryResult[@stryResult]}, @stryRow);
}
# Else check for error
elsif ($hStatement->err)
{
confess &log(ERROR, $DBI::errstr . ":\n${strSql}", ERROR_DB_QUERY);
}
}
while (@stryRow);

$bTimeout = false;
}
} while ($bTimeout && waitMore($oWait));

# If timeout then cancel the query and confess
if ($bTimeout)
if (defined($strResult))
{
$hStatement->pg_cancel();
confess &log(ERROR, 'statement timed out after ' . waitInterval($oWait) .
" second(s):\n${strSql}", ERROR_DB_TIMEOUT);
@stryResult = @{JSON::PP->new()->allow_nonref()->decode($strResult)};
}
}

@@ -864,7 +785,7 @@ sub walSwitch
# the user if there have been no writes since the last WAL switch.
if ($self->{strDbVersion} >= PG_VERSION_91)
{
$self->executeSql("select pg_create_restore_point('" . PROJECT_NAME . " Archive Check');");
$self->executeSql("select pg_create_restore_point('" . PROJECT_NAME . " Archive Check')::text;");
}

my $strWalFileName = $self->executeSqlOne(
@@ -1005,7 +926,7 @@ sub replayWait

if ($self->{strDbVersion} >= PG_VERSION_96)
{
$strCheckpointLSN = $self->executeSqlOne('select checkpoint_' . $self->lsnId() .' from pg_control_checkpoint()');
$strCheckpointLSN = $self->executeSqlOne('select checkpoint_' . $self->lsnId() .'::text from pg_control_checkpoint()');

if (lsnNormalize($strCheckpointLSN) le lsnNormalize($strTargetLSN))
{
@@ -72,6 +72,7 @@ These includes define data structures that are required for the C to Perl interf
***********************************************************************************************************************************/
#include "xs/crypto/hash.xsh"
#include "xs/common/encode.xsh"
#include "xs/postgres/client.xsh"
#include "xs/storage/storage.xsh"
#include "xs/storage/storageRead.xsh"
#include "xs/storage/storageWrite.xsh"
@@ -102,6 +103,7 @@ INCLUDE: xs/config/configTest.xs
INCLUDE: xs/config/define.xs
INCLUDE: xs/crypto/hash.xs
INCLUDE: xs/crypto/random.xs
INCLUDE: xs/postgres/client.xs
INCLUDE: xs/postgres/pageChecksum.xs
INCLUDE: xs/storage/storage.xs
INCLUDE: xs/storage/storageRead.xs
@@ -99,6 +99,7 @@ my @stryCFile =
'protocol/parallel.c',
'protocol/parallelJob.c',
'protocol/server.c',
'postgres/client.c',
'postgres/pageChecksum.c',
'storage/posix/read.c',
'storage/posix/storage.c',
@@ -132,6 +133,7 @@ WriteMakefile
-D_POSIX_C_SOURCE=200112L
-D_FILE_OFFSET_BITS=64
`xml2-config --cflags`
-I`pg_config --includedir`
)),

INC => join(' ', qw(
@@ -141,7 +143,7 @@ WriteMakefile

C => \@stryCFile,

LIBS => '-lcrypto -lssl -lxml2',
LIBS => '-lcrypto -lpq -lssl -lxml2',

OBJECT => '$(O_FILES)',
);
@@ -1,3 +1,4 @@
pgBackRest::LibC::PgClient T_PTROBJ
pgBackRest::LibC::Storage T_PTROBJ
pgBackRest::LibC::StorageRead T_PTROBJ
pgBackRest::LibC::StorageWrite T_PTROBJ
@@ -0,0 +1,89 @@
# ----------------------------------------------------------------------------------------------------------------------------------
# PostgreSQL Query Client
# ----------------------------------------------------------------------------------------------------------------------------------

MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::PgClient

####################################################################################################################################
pgBackRest::LibC::PgClient
new(class, host, port, database, queryTimeout)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
const String *class = STR_NEW_SV($arg);
const String *host = STR_NEW_SV($arg);
U32 port
const String *database = STR_NEW_SV($arg);
UV queryTimeout
CODE:
CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::PgClient"));

memContextSwitch(MEM_CONTEXT_XS_OLD());
RETVAL = pgClientNew(host, port, database, NULL, queryTimeout);
memContextSwitch(MEM_CONTEXT_XS_TEMP());
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();

####################################################################################################################################
void
open(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::PgClient self
CODE:
pgClientOpen(self);
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();

####################################################################################################################################
const char *
query(self, query)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::PgClient self
const String *query = STR_NEW_SV($arg);
CODE:
VariantList *result = pgClientQuery(self, query);
RETVAL = result ? strPtr(jsonFromVar(varNewVarLst(result), 0)) : NULL;
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();

####################################################################################################################################
void
close(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::PgClient self
CODE:
pgClientClose(self);
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();

####################################################################################################################################
void
DESTROY(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::PgClient self
CODE:
pgClientFree(self);
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
@@ -0,0 +1,6 @@
/***********************************************************************************************************************************
PostgreSQL Query Client Header
***********************************************************************************************************************************/
#include "postgres/client.h"

typedef PgClient *pgBackRest__LibC__PgClient;
@@ -447,7 +447,7 @@ main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.
perl/config.o: perl/config.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/config.c -o perl/config.o

perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/interface.h postgres/pageChecksum.h storage/helper.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/s3/storage.h storage/s3/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/hash.xsh ../libc/xs/storage/storage.xsh ../libc/xs/storage/storageRead.xsh ../libc/xs/storage/storageWrite.xsh
perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/client.h postgres/interface.h postgres/pageChecksum.h storage/helper.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/s3/storage.h storage/s3/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/hash.xsh ../libc/xs/postgres/client.xsh ../libc/xs/storage/storage.xsh ../libc/xs/storage/storageRead.xsh ../libc/xs/storage/storageWrite.xsh
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/exec.c -o perl/exec.o

postgres/client.o: postgres/client.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/client.h

0 comments on commit d8ca0e5

Please sign in to comment.
You can’t perform that action at this time.