From c5ae047e763f01a520286f9f4de7a85afba53942 Mon Sep 17 00:00:00 2001 From: David Steele Date: Sun, 18 Jul 2021 19:02:01 -0400 Subject: [PATCH] Partial migration of config code generation to C. Parse enough of config.yaml to auto-generate config.auto.h and config.auto.c. This commit implements most of the infrastructure needed to migrate the rest of the build code to C, but each set of auto-generated files will present its own challenges. The build is now dependent on libyaml. At this point there is no need for a hard requirement, but that will come soon so it seems better to add the dependency now. --- .cirrus.yml | 8 +- .github/workflows/test.yml | 3 + CONTRIBUTING.md | 2 +- build/lib/pgBackRestBuild/Config/Build.pm | 320 ----------- build/lib/pgBackRestBuild/Config/BuildHelp.pm | 2 +- .../lib/pgBackRestBuild/Config/BuildParse.pm | 23 +- doc/xml/contributing.xml | 2 +- doc/xml/user-guide.xml | 11 +- src/.gitignore | 1 + src/Makefile.in | 106 ++-- src/build/common/render.h | 43 ++ src/build/common/yaml.c | 200 +++++++ src/build/common/yaml.h | 88 +++ src/build/config/main.c | 39 ++ src/build/config/parse.c | 504 ++++++++++++++++++ src/build/config/parse.h | 53 ++ src/build/config/render.c | 330 ++++++++++++ src/build/config/render.h | 15 + src/build/configure.ac | 6 + src/config/config.auto.c | 2 +- src/config/config.auto.h | 2 +- src/configure | 57 +- test/Vagrantfile | 5 +- test/ci.pl | 2 +- test/define.yaml | 19 + test/lib/pgBackRestTest/Common/JobTest.pm | 2 +- test/src/module/build/commonTest.c | 60 +++ test/src/module/build/configTest.c | 300 +++++++++++ test/test.pl | 346 ++++++------ 29 files changed, 1989 insertions(+), 562 deletions(-) delete mode 100644 build/lib/pgBackRestBuild/Config/Build.pm create mode 100644 src/build/common/render.h create mode 100644 src/build/common/yaml.c create mode 100644 src/build/common/yaml.h create mode 100644 src/build/config/main.c create mode 100644 src/build/config/parse.c create mode 100644 src/build/config/parse.h create mode 100644 src/build/config/render.c create mode 100644 src/build/config/render.h create mode 100644 test/src/module/build/commonTest.c create mode 100644 test/src/module/build/configTest.c diff --git a/.cirrus.yml b/.cirrus.yml index f1c626be0c..6661a2895d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -15,7 +15,7 @@ freebsd_12_task: cpu: 4 memory: 4G - install_script: pkg install -y bash git postgresql-libpqxx pkgconf libxml2 gmake perl5 p5-YAML-LibYAML rsync + install_script: pkg install -y bash git postgresql-libpqxx pkgconf libxml2 gmake perl5 libyaml p5-YAML-LibYAML rsync script: - perl ${CIRRUS_WORKING_DIR}/test/test.pl --no-gen --make-cmd=gmake --vm=none --vm-max=2 --no-coverage --no-valgrind --module=command --test=backup @@ -30,12 +30,12 @@ macos_catalina_task: image: catalina-xcode environment: - LDFLAGS: -L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/libxml2/lib - CPPFLAGS: -I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/libpq/include -I/usr/local/opt/libxml2/include/libxml2 + LDFLAGS: -L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/libxml2/lib -L/usr/local/opt/libyaml/lib + CPPFLAGS: -I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/libpq/include -I/usr/local/opt/libxml2/include/libxml2 -I/usr/local/opt/libyaml/include PERL5LIB: /usr/local/opt/perl5/lib/perl5 install_script: - - brew install -q openssl@1.1 libpq libxml2 cpanm + - brew install -q openssl@1.1 libpq libxml2 libyaml cpanm - cpanm --local-lib=/usr/local/opt/perl5 install YAML::XS script: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5f1a3df3c0..c2e02d0b6e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -73,6 +73,9 @@ jobs: - name: Checkout Code uses: actions/checkout@v2 + - name: Install Packages + run: sudo apt-get install -y --no-install-recommends libyaml-dev + - name: Initialize CodeQL uses: github/codeql-action/init@v1 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6ed8a86079..59c5287851 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,7 @@ pgbackrest-dev => Install development tools sudo apt-get install rsync git devscripts build-essential valgrind lcov autoconf \ autoconf-archive libssl-dev zlib1g-dev libxml2-dev libpq-dev pkg-config \ libxml-checker-perl libyaml-perl libdbd-pg-perl liblz4-dev liblz4-tool \ - zstd libzstd-dev bzip2 libbz2-dev + zstd libzstd-dev bzip2 libbz2-dev libyaml-dev ``` Some unit tests and all the integration tests require Docker. Running in containers allows us to simulate multiple hosts, test on different distributions and versions of PostgreSQL, and use sudo without affecting the host system. diff --git a/build/lib/pgBackRestBuild/Config/Build.pm b/build/lib/pgBackRestBuild/Config/Build.pm deleted file mode 100644 index dff240c09f..0000000000 --- a/build/lib/pgBackRestBuild/Config/Build.pm +++ /dev/null @@ -1,320 +0,0 @@ -#################################################################################################################################### -# Auto-Generate Command and Option Configuration Enums, Constants and Data -#################################################################################################################################### -package pgBackRestBuild::Config::Build; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestBuild::Build::Common; -use pgBackRestBuild::Config::Data; - -#################################################################################################################################### -# Constants -#################################################################################################################################### -use constant BLDLCL_FILE_CONFIG => 'config'; - -use constant BLDLCL_CONSTANT_COMMAND => '01-constantCommand'; -use constant BLDLCL_CONSTANT_COMMAND_TOTAL => 'CFG_COMMAND_TOTAL'; -use constant BLDLCL_CONSTANT_OPTION_GROUP => '02-constantOptionGroup'; -use constant BLDLCL_CONSTANT_OPTION_GROUP_TOTAL => 'CFG_OPTION_GROUP_TOTAL'; -use constant BLDLCL_CONSTANT_OPTION => '03-constantOption'; -use constant BLDLCL_CONSTANT_OPTION_TOTAL => 'CFG_OPTION_TOTAL'; -use constant BLDLCL_CONSTANT_OPTION_VALUE => '04-constantOptionValue'; - -use constant BLDLCL_DATA_COMMAND => '01-command'; - -use constant BLDLCL_ENUM_COMMAND => '01-enumCommand'; -use constant BLDLCL_ENUM_OPTION_GROUP => '02-enumOptionGroup'; -use constant BLDLCL_ENUM_OPTION => '03-enumOption'; - -#################################################################################################################################### -# Definitions for constants and data to build -#################################################################################################################################### -my $rhBuild = -{ - &BLD_FILE => - { - #--------------------------------------------------------------------------------------------------------------------------- - &BLDLCL_FILE_CONFIG => - { - &BLD_SUMMARY => 'Command and Option Configuration', - - &BLD_CONSTANT_GROUP => - { - &BLDLCL_CONSTANT_COMMAND => - { - &BLD_SUMMARY => 'Command', - }, - &BLDLCL_CONSTANT_OPTION_GROUP => - { - &BLD_SUMMARY => 'Option group', - }, - &BLDLCL_CONSTANT_OPTION => - { - &BLD_SUMMARY => 'Option', - }, - &BLDLCL_CONSTANT_OPTION_VALUE => - { - &BLD_SUMMARY => 'Option value', - }, - }, - - &BLD_ENUM => - { - &BLDLCL_ENUM_COMMAND => - { - &BLD_SUMMARY => 'Command', - &BLD_NAME => 'ConfigCommand', - &BLD_LIST => [], - }, - - &BLDLCL_ENUM_OPTION_GROUP => - { - &BLD_SUMMARY => 'Option group', - &BLD_NAME => 'ConfigOptionGroup', - &BLD_LIST => [], - }, - - &BLDLCL_ENUM_OPTION => - { - &BLD_SUMMARY => 'Option', - &BLD_NAME => 'ConfigOption', - &BLD_LIST => [], - }, - }, - - &BLD_DATA => - { - &BLDLCL_DATA_COMMAND => - { - &BLD_SUMMARY => 'Command data', - }, - }, - }, - }, -}; - -#################################################################################################################################### -# Generate enum names -#################################################################################################################################### -sub buildConfigCommandEnum -{ - return bldEnum('cfgCmd', shift) -} - -push @EXPORT, qw(buildConfigCommandEnum); - -sub buildConfigOptionEnum -{ - return bldEnum('cfgOpt', shift) -} - -push @EXPORT, qw(buildConfigOptionEnum); - -sub buildConfigOptionGroupEnum -{ - return bldEnum('cfgOptGrp', shift) -} - -push @EXPORT, qw(buildConfigOptionGroupEnum); - -#################################################################################################################################### -# Build constants and data -#################################################################################################################################### -sub buildConfig -{ - # Build command constants and data - #------------------------------------------------------------------------------------------------------------------------------- - my $strCommandConst; - my $rhCommandDefine = cfgDefineCommand(); - my $rhEnum = $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_ENUM}{&BLDLCL_ENUM_COMMAND}; - my $iCommandTotal = 0; - - my $strBuildSource = - 'static const ConfigCommandData configCommandData[' . BLDLCL_CONSTANT_COMMAND_TOTAL . "] = CONFIG_COMMAND_LIST\n" . - "("; - - foreach my $strCommand (sort(keys(%{$rhCommandDefine}))) - { - my $rhCommand = $rhCommandDefine->{$strCommand}; - - # Build command constant name - $strCommandConst = "CFGCMD_" . uc($strCommand); - $strCommandConst =~ s/\-/_/g; - - # Build C enum - my $strCommandEnum = buildConfigCommandEnum($strCommand); - push(@{$rhEnum->{&BLD_LIST}}, $strCommandEnum); - - # Build command data - $strBuildSource .= - "\n" . - " CONFIG_COMMAND\n" . - " (\n" . - " CONFIG_COMMAND_NAME(${strCommandConst})\n" . - "\n" . - " CONFIG_COMMAND_LOG_FILE(" . ($rhCommand->{&CFGDEF_LOG_FILE} ? 'true' : 'false') . ")\n" . - " CONFIG_COMMAND_LOG_LEVEL_DEFAULT(logLevel" . ucfirst(lc($rhCommand->{&CFGDEF_LOG_LEVEL_DEFAULT})) . ")\n" . - " CONFIG_COMMAND_LOCK_REQUIRED(" . ($rhCommand->{&CFGDEF_LOCK_REQUIRED} ? 'true' : 'false') . ")\n" . - " CONFIG_COMMAND_LOCK_REMOTE_REQUIRED(" . - ($rhCommand->{&CFGDEF_LOCK_REMOTE_REQUIRED} ? 'true' : 'false') . ")\n" . - " CONFIG_COMMAND_LOCK_TYPE(lockType" . ucfirst(lc($rhCommand->{&CFGDEF_LOCK_TYPE})) . ")\n" . - " )\n"; - - $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_CONSTANT_GROUP}{&BLDLCL_CONSTANT_COMMAND}{&BLD_CONSTANT} - {$strCommandConst}{&BLD_CONSTANT_VALUE} = "\"${strCommand}\""; - - $iCommandTotal++; - } - - # Add "none" command that is used to initialize the current command before anything is parsed - push(@{$rhEnum->{&BLD_LIST}}, buildConfigCommandEnum('none')); - - $strBuildSource .= - ")\n"; - - $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_DATA}{&BLDLCL_DATA_COMMAND}{&BLD_SOURCE} = $strBuildSource; - - # Add an LF to the last command constant so there's whitespace before the total - $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_CONSTANT_GROUP}{&BLDLCL_CONSTANT_COMMAND}{&BLD_CONSTANT} - {$strCommandConst}{&BLD_CONSTANT_VALUE} .= "\n"; - - # Set option total constant - $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_CONSTANT_GROUP}{&BLDLCL_CONSTANT_COMMAND}{&BLD_CONSTANT} - {&BLDLCL_CONSTANT_COMMAND_TOTAL}{&BLD_CONSTANT_VALUE} = $iCommandTotal; - - # Build option group constants and data - #------------------------------------------------------------------------------------------------------------------------------- - my $rhOptionGroupDefine = cfgDefineOptionGroup(); - $rhEnum = $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_ENUM}{&BLDLCL_ENUM_OPTION_GROUP}; - my $iGroupTotal = 0; - - foreach my $strGroup (sort(keys(%{$rhOptionGroupDefine}))) - { - my $strGroupEnum = buildConfigOptionGroupEnum($strGroup); - push(@{$rhEnum->{&BLD_LIST}}, $strGroupEnum); - - $iGroupTotal++; - } - - $strBuildSource .= - "};\n"; - - # Set option total constant - $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_CONSTANT_GROUP}{&BLDLCL_CONSTANT_OPTION_GROUP}{&BLD_CONSTANT} - {&BLDLCL_CONSTANT_OPTION_GROUP_TOTAL}{&BLD_CONSTANT_VALUE} = $iGroupTotal; - - # Build option constants and data - #------------------------------------------------------------------------------------------------------------------------------- - my $strOptionConst; - my $rhConfigDefine = cfgDefine(); - $rhEnum = $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_ENUM}{&BLDLCL_ENUM_OPTION}; - my $iOptionTotal = 0; - - foreach my $strOption (sort(keys(%{$rhConfigDefine}))) - { - # Build C enum - my $strOptionEnum = buildConfigOptionEnum($strOption); - push(@{$rhEnum->{&BLD_LIST}}, $strOptionEnum); - $rhEnum->{&BLD_VALUE}{$strOptionEnum} = $iOptionTotal; - - # Build option constant name - $strOptionConst = "CFGOPT_" . uc($strOption); - $strOptionConst =~ s/\-/_/g; - - if (!$rhConfigDefine->{$strOption}{&CFGDEF_GROUP}) - { - $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_CONSTANT_GROUP}{&BLDLCL_CONSTANT_OPTION}{&BLD_CONSTANT} - {$strOptionConst}{&BLD_CONSTANT_VALUE} = "\"${strOption}\""; - } - - $iOptionTotal += 1; - } - - # Add an LF to the last option constant so there's whitespace before the total - $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_CONSTANT_GROUP}{&BLDLCL_CONSTANT_OPTION}{&BLD_CONSTANT} - {$strOptionConst}{&BLD_CONSTANT_VALUE} .= "\n"; - - # Set option total constant - $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_CONSTANT_GROUP}{&BLDLCL_CONSTANT_OPTION}{&BLD_CONSTANT} - {&BLDLCL_CONSTANT_OPTION_TOTAL}{&BLD_CONSTANT_VALUE} = $iOptionTotal; - - # Build option value constants - #------------------------------------------------------------------------------------------------------------------------------- - my $rhLastConstant = undef; - - foreach my $strOption (sort(keys(%{$rhConfigDefine}))) - { - my $rhOption = $rhConfigDefine->{$strOption}; - - # Only output allowed values for string options - if ($rhOption->{&CFGDEF_TYPE} eq CFGDEF_TYPE_STRING) - { - # Add LF to last option value list so they are not all jumbled together - if (defined($rhLastConstant)) - { - $rhLastConstant->{&BLD_CONSTANT_VALUE} .= "\n"; - $rhLastConstant = undef; - } - - # Add allowed values for the option, if any - my $rhValueHash = {}; - - if (defined($rhOption->{&CFGDEF_ALLOW_LIST})) - { - foreach my $strValue (sort(@{$rhOption->{&CFGDEF_ALLOW_LIST}})) - { - $rhValueHash->{$strValue} = true; - } - } - - # Add allowed values for the option commands, if any - foreach my $strCommand (sort(keys(%{$rhOption->{&CFGDEF_COMMAND}}))) - { - my $rhOptionCommand = $rhOption->{&CFGDEF_COMMAND}{$strCommand}; - - if (defined($rhOptionCommand->{&CFGDEF_ALLOW_LIST})) - { - foreach my $strValue (sort(@{$rhOptionCommand->{&CFGDEF_ALLOW_LIST}})) - { - $rhValueHash->{$strValue} = true; - } - } - } - - # Output list of allowed values - foreach my $strValue (sort(keys(%{$rhValueHash}))) - { - my $strOptionValueConst = 'CFGOPTVAL_' . uc($strOption) . '_' . uc($strValue) . '_Z'; - $strOptionValueConst =~ s/\-/_/g; - - $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_CONSTANT_GROUP}{&BLDLCL_CONSTANT_OPTION_VALUE}{&BLD_CONSTANT} - {$strOptionValueConst}{&BLD_CONSTANT_VALUE} = "\"${strValue}\""; - - # Save last constant so an LF can be added later, if needed - $rhLastConstant = - $rhBuild->{&BLD_FILE}{&BLDLCL_FILE_CONFIG}{&BLD_CONSTANT_GROUP}{&BLDLCL_CONSTANT_OPTION_VALUE}{&BLD_CONSTANT} - {$strOptionValueConst}; - } - } - } - - return $rhBuild; -} - -push @EXPORT, qw(buildConfig); - -1; diff --git a/build/lib/pgBackRestBuild/Config/BuildHelp.pm b/build/lib/pgBackRestBuild/Config/BuildHelp.pm index 693abb4294..2c3b74662e 100644 --- a/build/lib/pgBackRestBuild/Config/BuildHelp.pm +++ b/build/lib/pgBackRestBuild/Config/BuildHelp.pm @@ -20,7 +20,7 @@ use pgBackRestDoc::Common::String; use pgBackRestDoc::ProjectInfo; use pgBackRestBuild::Build::Common; -use pgBackRestBuild::Config::Build; +use pgBackRestBuild::Config::BuildParse; use pgBackRestBuild::Config::Data; #################################################################################################################################### diff --git a/build/lib/pgBackRestBuild/Config/BuildParse.pm b/build/lib/pgBackRestBuild/Config/BuildParse.pm index d4379a3ae6..e716866c1d 100644 --- a/build/lib/pgBackRestBuild/Config/BuildParse.pm +++ b/build/lib/pgBackRestBuild/Config/BuildParse.pm @@ -19,7 +19,6 @@ use pgBackRestDoc::Common::String; use pgBackRestDoc::ProjectInfo; use pgBackRestBuild::Build::Common; -use pgBackRestBuild::Config::Build; use pgBackRestBuild::Config::Data; #################################################################################################################################### @@ -80,6 +79,28 @@ my $rhBuild = #################################################################################################################################### # Generate enum names #################################################################################################################################### +sub buildConfigCommandEnum +{ + return bldEnum('cfgCmd', shift) +} + +push @EXPORT, qw(buildConfigCommandEnum); + +sub buildConfigOptionEnum +{ + return bldEnum('cfgOpt', shift) +} + +push @EXPORT, qw(buildConfigOptionEnum); + +sub buildConfigOptionGroupEnum +{ + return bldEnum('cfgOptGrp', shift) +} + +push @EXPORT, qw(buildConfigOptionGroupEnum); + + sub buildConfigDefineOptionTypeEnum { return bldEnum('cfgOptType', shift); diff --git a/doc/xml/contributing.xml b/doc/xml/contributing.xml index 782b8e0b3c..0d77348d18 100644 --- a/doc/xml/contributing.xml +++ b/doc/xml/contributing.xml @@ -83,7 +83,7 @@ apt-get install rsync git devscripts build-essential valgrind lcov autoconf autoconf-archive libssl-dev zlib1g-dev libxml2-dev libpq-dev pkg-config libxml-checker-perl libyaml-perl libdbd-pg-perl liblz4-dev liblz4-tool - zstd libzstd-dev bzip2 libbz2-dev + zstd libzstd-dev bzip2 libbz2-dev libyaml-dev -y 2>&1 diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index a4a549f542..ca56489fc0 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -36,7 +36,7 @@ '{[os-type]}' eq '{[os-rhel]}' - ubuntu:16.04 + ubuntu:18.04 centos:8 {[os-debian-title]} @@ -355,7 +355,10 @@ VOLUME [ "/sys/fs/cgroup" ] # Install packages - RUN yum install -y openssh-server openssh-clients sudo wget vim 2>&1 + RUN yum install -y openssh-server openssh-clients sudo wget vim dnf-plugins-core 2>&1 + + # Enable PowerTools repository (only available on RHEL8) + RUN dnf config-manager --set-enabled powertools || true # Install CA certificate RUN update-ca-trust extract @@ -836,7 +839,7 @@ apt-get install make gcc libpq-dev libssl-dev libxml2-dev pkg-config - liblz4-dev libzstd-dev libbz2-dev libz-dev + liblz4-dev libzstd-dev libbz2-dev libz-dev libyaml-dev -y 2>&1 @@ -844,7 +847,7 @@ yum install make gcc postgresql{[pg-version-nodot]}-devel - openssl-devel libxml2-devel lz4-devel libzstd-devel bzip2-devel + openssl-devel libxml2-devel lz4-devel libzstd-devel bzip2-devel libyaml-devel -y 2>&1 diff --git a/src/.gitignore b/src/.gitignore index a5ef16f4f4..1ac33f9aab 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -5,3 +5,4 @@ autom4te.cache /Makefile /build.auto.h /pgbackrest +/build-* diff --git a/src/Makefile.in b/src/Makefile.in index 1658e756fd..2bf54b5992 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -5,6 +5,42 @@ #################################################################################################################################### # List of required source files. main.c should always be listed last and the rest in alpha order. #################################################################################################################################### +SRCS_BUILD = \ + common/debug.c \ + common/encode.c \ + common/error.c \ + common/io/filter/buffer.c \ + common/io/filter/filter.c \ + common/io/filter/group.c \ + common/io/filter/sink.c \ + common/io/io.c \ + common/io/read.c \ + common/io/write.c \ + common/log.c \ + common/memContext.c \ + common/regExp.c \ + common/stackTrace.c \ + common/time.c \ + common/type/buffer.c \ + common/type/convert.c \ + common/type/keyValue.c \ + common/type/list.c \ + common/type/object.c \ + common/type/pack.c \ + common/type/string.c \ + common/type/stringId.c \ + common/type/stringList.c \ + common/type/variant.c \ + common/type/variantList.c \ + common/user.c \ + common/wait.c \ + storage/posix/read.c \ + storage/posix/storage.c \ + storage/posix/write.c \ + storage/read.c \ + storage/storage.c \ + storage/write.c + SRCS = \ command/archive/common.c \ command/archive/get/file.c \ @@ -15,11 +51,11 @@ SRCS = \ command/archive/push/push.c \ command/backup/backup.c \ command/backup/common.c \ - command/backup/file.c \ command/backup/pageChecksum.c \ + command/backup/protocol.c \ + command/backup/file.c \ command/check/check.c \ command/check/common.c \ - command/backup/protocol.c \ command/expire/expire.c \ command/help/help.c \ command/info/info.c \ @@ -60,22 +96,16 @@ SRCS = \ common/crypto/cipherBlock.c \ common/crypto/common.c \ common/crypto/hash.c \ - common/debug.c \ - common/encode.c \ - common/error.c \ common/exec.c \ common/exit.c \ common/fork.c \ + common/ini.c \ common/io/bufferRead.c \ common/io/bufferWrite.c \ common/io/client.c \ common/io/fd.c \ common/io/fdRead.c \ common/io/fdWrite.c \ - common/io/filter/buffer.c \ - common/io/filter/filter.c \ - common/io/filter/group.c \ - common/io/filter/sink.c \ common/io/filter/size.c \ common/io/http/client.c \ common/io/http/common.c \ @@ -85,39 +115,17 @@ SRCS = \ common/io/http/response.c \ common/io/http/session.c \ common/io/http/url.c \ - common/io/io.c \ - common/io/read.c \ common/io/session.c \ common/io/socket/client.c \ common/io/socket/common.c \ common/io/socket/session.c \ common/io/tls/client.c \ common/io/tls/session.c \ - common/io/write.c \ - common/ini.c \ common/lock.c \ - common/log.c \ - common/memContext.c \ - common/regExp.c \ - common/stackTrace.c \ common/stat.c \ - common/time.c \ - common/type/buffer.c \ - common/type/convert.c \ common/type/json.c \ - common/type/keyValue.c \ - common/type/list.c \ common/type/mcv.c \ - common/type/object.c \ - common/type/pack.c \ - common/type/string.c \ - common/type/stringId.c \ - common/type/stringList.c \ - common/type/variant.c \ - common/type/variantList.c \ common/type/xml.c \ - common/user.c \ - common/wait.c \ config/config.c \ config/exec.c \ config/load.c \ @@ -161,21 +169,14 @@ SRCS = \ storage/gcs/read.c \ storage/gcs/storage.c \ storage/gcs/write.c \ - storage/posix/read.c \ - storage/posix/storage.c \ - storage/posix/write.c \ + storage/helper.c \ storage/remote/read.c \ storage/remote/protocol.c \ storage/remote/storage.c \ storage/remote/write.c \ storage/s3/read.c \ storage/s3/storage.c \ - storage/s3/write.c \ - storage/helper.c \ - storage/read.c \ - storage/storage.c \ - storage/write.c \ - main.c + storage/s3/write.c #################################################################################################################################### # Compiler options @@ -185,6 +186,7 @@ CFLAGS = $(CFLAGS_EXTRA) @CFLAGS@ CPPFLAGS = @CPPFLAGS@ -I@srcdir@ LDFLAGS = $(LDFLAGS_EXTRA) @LDFLAGS@ LIBS = @LIBS@ +LIBS_BUILD = @LIBS_BUILD@ #################################################################################################################################### # Directory options @@ -196,15 +198,27 @@ bindir = @bindir@ BUILDDIR=.build #################################################################################################################################### -# Create object list from source list +# Compile and link pgbackrest #################################################################################################################################### -OBJS = $(patsubst %.c,$(BUILDDIR)/%.o,$(SRCS)) +OBJS_PGBACKREST = $(patsubst %.c,$(BUILDDIR)/%.o,$(SRCS_BUILD) $(SRCS) main.c) + +pgbackrest: $(OBJS_PGBACKREST) + $(CC) -o pgbackrest $(OBJS_PGBACKREST) $(LDFLAGS) $(LIBS) #################################################################################################################################### -# Compile and link +# Compile and link config generator #################################################################################################################################### -pgbackrest: $(OBJS) - $(CC) -o pgbackrest $(OBJS) $(LDFLAGS) $(LIBS) +SRCS_BUILD_CONFIG = \ + build/common/yaml.c \ + build/config/main.c \ + build/config/parse.c \ + build/config/render.c + +OBJS_BUILD_CONFIG = $(patsubst %.c,$(BUILDDIR)/%.o,$(SRCS_BUILD) $(SRCS_BUILD_CONFIG)) + +build-config: $(OBJS_BUILD_CONFIG) build/config/config.yaml config/config.auto.h + $(CC) -o build-config $(OBJS_BUILD_CONFIG) $(LDFLAGS) $(LIBS) $(LIBS_BUILD) + ./build-config $(VPATH) #################################################################################################################################### # Installation. DESTDIR can be used to modify the install location. @@ -227,7 +241,7 @@ uninstall: # Clean build files and executable created by make clean: rm -rf $(BUILDDIR) - rm -f pgbackrest + rm -f pgbackrest build-config .PHONY = clean-all diff --git a/src/build/common/render.h b/src/build/common/render.h new file mode 100644 index 0000000000..8de665a99c --- /dev/null +++ b/src/build/common/render.h @@ -0,0 +1,43 @@ +/*********************************************************************************************************************************** +Build Common +***********************************************************************************************************************************/ +#ifndef BUILD_COMMON_COMMON_H +#define BUILD_COMMON_COMMON_H + +#include "common/type/string.h" + +/*********************************************************************************************************************************** +Block comments +***********************************************************************************************************************************/ +#define COMMENT_BLOCK_BEGIN \ + "/***************************************************************************************************************************" \ + "********" + +#define COMMENT_BLOCK_END \ + "****************************************************************************************************************************" \ + "*******/" + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Format a #define with the value aligned at column 69 +__attribute__((always_inline)) static inline String * +bldDefineRender(const String *const define, const String *const value) +{ + return strNewFmt("#define %s%*s%s", strZ(define), (int)(60 - strSize(define)), "", strZ(value)); +} + +// Format file header +__attribute__((always_inline)) static inline String * +bldHeader(const char *const module, const char *const description) +{ + return strNewFmt( + COMMENT_BLOCK_BEGIN "\n" + "%s\n" + "\n" + "Automatically generated by 'make build-%s' -- do not modify directly.\n" + COMMENT_BLOCK_END "\n", + description, module); +} + +#endif diff --git a/src/build/common/yaml.c b/src/build/common/yaml.c new file mode 100644 index 0000000000..4a106ae532 --- /dev/null +++ b/src/build/common/yaml.c @@ -0,0 +1,200 @@ +/*********************************************************************************************************************************** +Yaml Handler +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include + +#include "common/debug.h" +#include "common/log.h" +#include "common/memContext.h" + +#include "build/common/yaml.h" + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +struct Yaml +{ + MemContext *memContext; // Mem context + yaml_parser_t parser; // Parse context +}; + +/*********************************************************************************************************************************** +Free parser context +***********************************************************************************************************************************/ +static void +yamlFreeResource(THIS_VOID) +{ + THIS(Yaml); + + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(YAML, this); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + + yaml_parser_delete(&this->parser); + + FUNCTION_LOG_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +Yaml * +yamlNew(const Buffer *const buffer) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(BUFFER, buffer); + FUNCTION_TEST_END(); + + Yaml *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("Yaml") + { + // Create object + this = memNew(sizeof(Yaml)); + + *this = (Yaml) + { + .memContext = MEM_CONTEXT_NEW(), + }; + + // Initialize parser context + CHECK(yaml_parser_initialize(&this->parser)); + memContextCallbackSet(this->memContext, yamlFreeResource, this); + + // Set yaml string + yaml_parser_set_input_string(&this->parser, bufPtrConst(buffer), bufUsed(buffer)); + + // Start document + CHECK(yamlEventNext(this).type == yamlEventTypeStreamBegin); + CHECK(yamlEventNext(this).type == yamlEventTypeDocBegin); + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +// Helper to map event type +static YamlEventType +yamlEventType(yaml_event_type_t type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_END(); + + switch (type) + { + case YAML_STREAM_START_EVENT: + FUNCTION_TEST_RETURN(yamlEventTypeStreamBegin); + + case YAML_STREAM_END_EVENT: + FUNCTION_TEST_RETURN(yamlEventTypeStreamEnd); + + case YAML_DOCUMENT_START_EVENT: + FUNCTION_TEST_RETURN(yamlEventTypeDocBegin); + + case YAML_DOCUMENT_END_EVENT: + FUNCTION_TEST_RETURN(yamlEventTypeDocEnd); + + case YAML_ALIAS_EVENT: + FUNCTION_TEST_RETURN(yamlEventTypeAlias); + + case YAML_SCALAR_EVENT: + FUNCTION_TEST_RETURN(yamlEventTypeScalar); + + case YAML_SEQUENCE_START_EVENT: + FUNCTION_TEST_RETURN(yamlEventTypeSeqBegin); + + case YAML_SEQUENCE_END_EVENT: + FUNCTION_TEST_RETURN(yamlEventTypeSeqEnd); + + case YAML_MAPPING_START_EVENT: + FUNCTION_TEST_RETURN(yamlEventTypeMapBegin); + + case YAML_MAPPING_END_EVENT: + FUNCTION_TEST_RETURN(yamlEventTypeMapEnd); + + default: + CHECK(type == YAML_NO_EVENT); + FUNCTION_TEST_RETURN(yamlEventTypeNone); + } +} + +YamlEvent +yamlEventNext(Yaml *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(YAML, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + yaml_event_t event; + + if (!yaml_parser_parse(&this->parser, &event)) + { + // These should always be set + CHECK(this->parser.problem_mark.line && this->parser.problem_mark.column); + + THROW_FMT( + FormatError, "yaml parse error: %s at line: %lu column: %lu", this->parser.problem, + (unsigned long)this->parser.problem_mark.line + 1, (unsigned long)this->parser.problem_mark.column + 1); + } + + YamlEvent result = {.type = yamlEventType(event.type)}; + + if (result.type == yamlEventTypeScalar) + result.value = strNewZ((const char *)event.data.scalar.value); + + yaml_event_delete(&event); + + FUNCTION_TEST_RETURN(result); +} + +/**********************************************************************************************************************************/ +YamlEvent +yamlEventNextCheck(Yaml *this, YamlEventType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(YAML, this); + FUNCTION_TEST_PARAM(STRING_ID, type); + FUNCTION_TEST_END(); + + YamlEvent result = yamlEventNext(this); + yamlEventCheck(result, type); + + FUNCTION_TEST_RETURN(result); +} + +/**********************************************************************************************************************************/ +void +yamlEventCheck(YamlEvent event, YamlEventType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(YAML_EVENT, event); + FUNCTION_TEST_PARAM(STRING_ID, type); + FUNCTION_TEST_END(); + + if (event.type != type) + THROW_FMT(FormatError, "expected event type '%s' but got '%s'", strZ(strIdToStr(type)), strZ(strIdToStr(event.type))); + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +bool +yamlBoolParse(YamlEvent event) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(YAML_EVENT, event); + FUNCTION_TEST_END(); + + if (strEq(event.value, FALSE_STR)) + FUNCTION_TEST_RETURN(false); + else if (strEq(event.value, TRUE_STR)) + FUNCTION_TEST_RETURN(true); + + THROW_FMT(FormatError, "invalid boolean '%s'", strZ(event.value)); +} diff --git a/src/build/common/yaml.h b/src/build/common/yaml.h new file mode 100644 index 0000000000..7a9ecd4cbb --- /dev/null +++ b/src/build/common/yaml.h @@ -0,0 +1,88 @@ +/*********************************************************************************************************************************** +Yaml Handler +***********************************************************************************************************************************/ +#ifndef COMMON_TYPE_YAML_H +#define COMMON_TYPE_YAML_H + +#include + +/*********************************************************************************************************************************** +Yaml object +***********************************************************************************************************************************/ +typedef struct Yaml Yaml; + +#include "common/memContext.h" +#include "common/type/object.h" +#include "common/type/buffer.h" +#include "common/type/stringId.h" + +/*********************************************************************************************************************************** +Yaml event type +***********************************************************************************************************************************/ +typedef enum +{ + yamlEventTypeNone = STRID5("none", 0x2b9ee0), + yamlEventTypeStreamBegin = STRID5("stream-begin", 0x724e516da12ca930), + yamlEventTypeStreamEnd = STRID5("stream-end", 0x8e2eda12ca930), + yamlEventTypeDocBegin = STRID5("doc-begin", 0xe49ca2d8de40), + yamlEventTypeDocEnd = STRID5("doc-end", 0x11c5d8de40), + yamlEventTypeAlias = STRID5("alias", 0x130a5810), + yamlEventTypeScalar = STRID5("scalar", 0x241604730), + yamlEventTypeSeqBegin = STRID5("seq-begin", 0xe49ca2dc4b30), + yamlEventTypeSeqEnd = STRID5("seq-end", 0x11c5dc4b30), + yamlEventTypeMapBegin = STRID5("map-begin", 0xe49ca2dc02d0), + yamlEventTypeMapEnd = STRID5("map-end", 0x11c5dc02d0), +} YamlEventType; + +/*********************************************************************************************************************************** +Yaml event +***********************************************************************************************************************************/ +typedef struct YamlEvent +{ + YamlEventType type; // Type (e.g. scalar) + const String *value; // Value, when type is scalar +} YamlEvent; + +/*********************************************************************************************************************************** +Constructors +***********************************************************************************************************************************/ +Yaml *yamlNew(const Buffer *const buffer); + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Get next event from doc +YamlEvent yamlEventNext(Yaml *this); + +// Get next event from doc and check the type +YamlEvent yamlEventNextCheck(Yaml *this, YamlEventType type); + +// Check the event type +void yamlEventCheck(YamlEvent event, YamlEventType type); + +// Convert an event to a boolean (or error) +bool yamlBoolParse(YamlEvent event); + +/*********************************************************************************************************************************** +Destructor +***********************************************************************************************************************************/ +__attribute__((always_inline)) static inline void +yamlFree(Yaml *const this) +{ + objFree(this); +} + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +#define FUNCTION_LOG_YAML_TYPE \ + Yaml * +#define FUNCTION_LOG_YAML_FORMAT(value, buffer, bufferSize) \ + objToLog(value, "Yaml", buffer, bufferSize) + +#define FUNCTION_LOG_YAML_EVENT_TYPE \ + YamlEvent +#define FUNCTION_LOG_YAML_EVENT_FORMAT(value, buffer, bufferSize) \ + objToLog(&value, "YamlEvent", buffer, bufferSize) + +#endif diff --git a/src/build/config/main.c b/src/build/config/main.c new file mode 100644 index 0000000000..3c880d4ec0 --- /dev/null +++ b/src/build/config/main.c @@ -0,0 +1,39 @@ +/*********************************************************************************************************************************** +Auto-Generate Command and Option Configuration Enums, Constants and Data +***********************************************************************************************************************************/ +#include + +#include "common/log.h" +#include "storage/posix/storage.h" + +#include "build/config/parse.h" +#include "build/config/render.h" + +int +main(int argListSize, const char *argList[]) +{ + // Check parameters + CHECK(argListSize <= 2); + + // If the path was specified + const String *pathRepo; + + if (argListSize >= 2) + { + pathRepo = strPath(STR(argList[1])); + } + // Else use current working directory + else + { + char currentWorkDir[1024]; + THROW_ON_SYS_ERROR(getcwd(currentWorkDir, sizeof(currentWorkDir)) == NULL, FormatError, "unable to get cwd"); + + pathRepo = strPath(STR(currentWorkDir)); + } + + // Render config + const Storage *const storageRepo = storagePosixNewP(pathRepo, .write = true); + bldCfgRender(storageRepo, bldCfgParse(storageRepo)); + + return 0; +} diff --git a/src/build/config/parse.c b/src/build/config/parse.c new file mode 100644 index 0000000000..b309d35778 --- /dev/null +++ b/src/build/config/parse.c @@ -0,0 +1,504 @@ +/*********************************************************************************************************************************** +Parse Configuration Yaml +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include + +#include "common/log.h" +#include "storage/posix/storage.h" + +#include "build/common/yaml.h" +#include "build/config/parse.h" + +/**********************************************************************************************************************************/ +// Helper to parse allow list +static const StringList * +bldCfgParseAllowList(Yaml *const yaml, const List *const optList) +{ + YamlEvent allowListVal = yamlEventNext(yaml); + + // If allow list is defined + if (allowListVal.type == yamlEventTypeSeqBegin) + { + YamlEvent allowListVal = yamlEventNext(yaml); + StringList *result = strLstNew(); + + do + { + yamlEventCheck(allowListVal, yamlEventTypeScalar); + strLstAdd(result, allowListVal.value); + + allowListVal = yamlEventNext(yaml); + } + while (allowListVal.type != yamlEventTypeSeqEnd); + + strLstSort(result, sortOrderAsc); + + return result; + } + + // Else allow list is inherited + CHECK(optList != NULL); + yamlEventCheck(allowListVal, yamlEventTypeScalar); + + const BldCfgOption *const optInherit = lstFind(optList, &allowListVal.value); + CHECK(optInherit != NULL); + + return optInherit->allowList; +} + +// Helper to parse allow range +static void +bldCfgParseAllowRange(Yaml *const yaml) +{ + yamlEventNextCheck(yaml, yamlEventTypeSeqBegin); + + YamlEvent allowRangeVal = yamlEventNext(yaml); + + do + { + yamlEventCheck(allowRangeVal, yamlEventTypeScalar); + + allowRangeVal = yamlEventNext(yaml); + } + while (allowRangeVal.type != yamlEventTypeSeqEnd); +} + +// Helper to parse command roles +static void +bldCfgParseCommandRole(Yaml *const yaml) +{ + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + + YamlEvent commandRoleVal = yamlEventNext(yaml); + + if (commandRoleVal.type != yamlEventTypeMapEnd) + { + do + { + yamlEventCheck(commandRoleVal, yamlEventTypeScalar); + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + yamlEventNextCheck(yaml, yamlEventTypeMapEnd); + + commandRoleVal = yamlEventNext(yaml); + } + while (commandRoleVal.type != yamlEventTypeMapEnd); + } +} + +// Helper to parse depend +static void +bldCfgParseDepend(Yaml *const yaml) +{ + YamlEvent dependVal = yamlEventNext(yaml); + + if (dependVal.type == yamlEventTypeMapBegin) + { + YamlEvent dependDef = yamlEventNext(yaml); + + do + { + yamlEventCheck(dependDef, yamlEventTypeScalar); + + if (strEqZ(dependDef.value, "list")) + { + yamlEventNextCheck(yaml, yamlEventTypeSeqBegin); + + YamlEvent dependDefVal = yamlEventNext(yaml); + + do + { + yamlEventCheck(dependDefVal, yamlEventTypeScalar); + dependDefVal = yamlEventNext(yaml); + } + while (dependDefVal.type != yamlEventTypeSeqEnd); + } + else + { + YamlEvent dependDefVal = yamlEventNext(yaml); + yamlEventCheck(dependDefVal, yamlEventTypeScalar); + + if (strEqZ(dependDef.value, "option")) + { + } + else + THROW_FMT(FormatError, "unknown depend definition '%s'", strZ(dependDef.value)); + } + + dependDef = yamlEventNext(yaml); + } + while (dependDef.type != yamlEventTypeMapEnd); + } + else + yamlEventCheck(dependVal, yamlEventTypeScalar); +} + +// Helper to parse deprecate +static void +bldCfgParseOptionDeprecate(Yaml *const yaml) +{ + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + + YamlEvent optDeprecate = yamlEventNext(yaml); + + do + { + yamlEventCheck(optDeprecate, yamlEventTypeScalar); + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + + YamlEvent optDeprecateDef = yamlEventNext(yaml); + + if (optDeprecateDef.type == yamlEventTypeScalar) + { + do + { + yamlEventCheck(optDeprecateDef, yamlEventTypeScalar); + yamlEventNextCheck(yaml, yamlEventTypeScalar); + + if (strEqZ(optDeprecateDef.value, "index")) + { + } + else if (strEqZ(optDeprecateDef.value, "reset")) + { + } + else + THROW_FMT(FormatError, "unknown deprecate definition '%s'", strZ(optDeprecateDef.value)); + + optDeprecateDef = yamlEventNext(yaml); + } + while (optDeprecateDef.type != yamlEventTypeMapEnd); + } + else + yamlEventCheck(optDeprecateDef, yamlEventTypeMapEnd); + + optDeprecate = yamlEventNext(yaml); + } + while (optDeprecate.type != yamlEventTypeMapEnd); +} + +// Helper to parse commands +static const List * +bldCfgParseOptionCommand(Yaml *const yaml, const List *const optList) +{ + YamlEvent optCmdVal = yamlEventNext(yaml); + + // If command list is defined + if (optCmdVal.type == yamlEventTypeMapBegin) + { + List *result = lstNewP(sizeof(BldCfgOptionCommand), .comparator = lstComparatorStr); + + YamlEvent optCmd = yamlEventNext(yaml); + + do + { + yamlEventCheck(optCmd, yamlEventTypeScalar); + BldCfgOptionCommand optCmdData = {.name = optCmd.value}; + + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + YamlEvent optCmdDef = yamlEventNext(yaml); + + if (optCmdDef.type == yamlEventTypeScalar) + { + do + { + yamlEventCheck(optCmdDef, yamlEventTypeScalar); + + if (strEqZ(optCmdDef.value, "allow-list")) + { + optCmdData.allowList = bldCfgParseAllowList(yaml, NULL); + } + else if (strEqZ(optCmdDef.value, "command-role")) + { + bldCfgParseCommandRole(yaml); + } + else if (strEqZ(optCmdDef.value, "depend")) + { + bldCfgParseDepend(yaml); + } + else + { + yamlEventNextCheck(yaml, yamlEventTypeScalar); + + if (strEqZ(optCmdDef.value, "default")) + { + } + else if (strEqZ(optCmdDef.value, "internal")) + { + } + else if (strEqZ(optCmdDef.value, "required")) + { + } + else + THROW_FMT(FormatError, "unknown option command definition '%s'", strZ(optCmdDef.value)); + } + + optCmdDef = yamlEventNext(yaml); + } + while (optCmdDef.type != yamlEventTypeMapEnd); + } + else + yamlEventCheck(optCmdDef, yamlEventTypeMapEnd); + + lstAdd(result, &optCmdData); + + optCmd = yamlEventNext(yaml); + } + while (optCmd.type != yamlEventTypeMapEnd); + + lstSort(result, sortOrderAsc); + + return result; + } + + // Else command list is inherited + CHECK(optList != NULL); + yamlEventCheck(optCmdVal, yamlEventTypeScalar); + + const BldCfgOption *const optInherit = lstFind(optList, &optCmdVal.value); + CHECK(optInherit != NULL); + + return optInherit->cmdList; +} + +BldCfg +bldCfgParse(const Storage *const storageRepo) +{ + BldCfg result = + { + .commandList = lstNewP(sizeof(BldCfgCommand), .comparator = lstComparatorStr), + .optGrpList = lstNewP(sizeof(BldCfgOptionGroup), .comparator = lstComparatorStr), + .optList = lstNewP(sizeof(BldCfgOption), .comparator = lstComparatorStr), + }; + + // Initialize yaml + Yaml *const yaml = yamlNew(storageGetP(storageNewReadP(storageRepo, STRDEF("src/build/config/config.yaml")))); + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + + // Parse commands + // ----------------------------------------------------------------------------------------------------------------------------- + yamlEventNextCheck(yaml, yamlEventTypeScalar); + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + + YamlEvent cmd = yamlEventNext(yaml); + + do + { + yamlEventCheck(cmd, yamlEventTypeScalar); + + BldCfgCommand cmdData = + { + .name = cmd.value, + .logFile = true, + .logLevelDefault = strNewZ("info"), + .lockType = strNewZ("none"), + }; + + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + + YamlEvent cmdDef = yamlEventNext(yaml); + + if (cmdDef.type == yamlEventTypeScalar) + { + do + { + yamlEventCheck(cmdDef, yamlEventTypeScalar); + + if (strEqZ(cmdDef.value, "command-role")) + { + bldCfgParseCommandRole(yaml); + } + else + { + YamlEvent cmdDefVal = yamlEventNextCheck(yaml, yamlEventTypeScalar); + + if (strEqZ(cmdDef.value, "internal")) + { + } + else if (strEqZ(cmdDef.value, "lock-type")) + { + cmdData.lockType = cmdDefVal.value; + } + else if (strEqZ(cmdDef.value, "lock-remote-required")) + { + cmdData.lockRemoteRequired = yamlBoolParse(cmdDefVal); + } + else if (strEqZ(cmdDef.value, "lock-required")) + { + cmdData.lockRequired = yamlBoolParse(cmdDefVal); + } + else if (strEqZ(cmdDef.value, "log-file")) + { + cmdData.logFile = yamlBoolParse(cmdDefVal); + } + else if (strEqZ(cmdDef.value, "log-level-default")) + { + cmdData.logLevelDefault = strLower(strDup(cmdDefVal.value)); + } + else if (strEqZ(cmdDef.value, "parameter-allowed")) + { + } + else + THROW_FMT(FormatError, "unknown command definition '%s'", strZ(cmdDef.value)); + } + + cmdDef = yamlEventNext(yaml); + } + while (cmdDef.type != yamlEventTypeMapEnd); + } + else + yamlEventCheck(cmdDef, yamlEventTypeMapEnd); + + lstAdd(result.commandList, &cmdData); + + cmd = yamlEventNext(yaml); + } + while (cmd.type != yamlEventTypeMapEnd); + + lstSort(result.commandList, sortOrderAsc); + + // Parse option groups + // ----------------------------------------------------------------------------------------------------------------------------- + yamlEventNextCheck(yaml, yamlEventTypeScalar); + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + + YamlEvent optGrp = yamlEventNext(yaml); + + do + { + yamlEventCheck(optGrp, yamlEventTypeScalar); + BldCfgOptionGroup optGrpData = {.name = optGrp.value}; + + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + + YamlEvent optGrpDef = yamlEventNext(yaml); + + do + { + yamlEventCheck(optGrpDef, yamlEventTypeScalar); + yamlEventNextCheck(yaml, yamlEventTypeScalar); + + if (strEqZ(optGrpDef.value, "indexTotal")) + { + } + else if (strEqZ(optGrpDef.value, "prefix")) + { + } + else + THROW_FMT(FormatError, "unknown option group definition '%s'", strZ(optGrpDef.value)); + + optGrpDef = yamlEventNext(yaml); + } + while (optGrpDef.type != yamlEventTypeMapEnd); + + lstAdd(result.optGrpList, &optGrpData); + + optGrp = yamlEventNext(yaml); + } + while (optGrp.type != yamlEventTypeMapEnd); + + lstSort(result.optGrpList, sortOrderAsc); + + // Parse options + // ----------------------------------------------------------------------------------------------------------------------------- + yamlEventNextCheck(yaml, yamlEventTypeScalar); + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + + YamlEvent opt = yamlEventNext(yaml); + + do + { + yamlEventCheck(opt, yamlEventTypeScalar); + BldCfgOption optData = {.name = opt.value}; + + yamlEventNextCheck(yaml, yamlEventTypeMapBegin); + + YamlEvent optDef = yamlEventNext(yaml); + + do + { + yamlEventCheck(optDef, yamlEventTypeScalar); + + if (strEqZ(optDef.value, "allow-list")) + { + optData.allowList = bldCfgParseAllowList(yaml, result.optList); + } + else if (strEqZ(optDef.value, "allow-range")) + { + bldCfgParseAllowRange(yaml); + } + else if (strEqZ(optDef.value, "command")) + { + optData.cmdList = bldCfgParseOptionCommand(yaml, result.optList); + } + else if (strEqZ(optDef.value, "command-role")) + { + bldCfgParseCommandRole(yaml); + } + else if (strEqZ(optDef.value, "depend")) + { + bldCfgParseDepend(yaml); + } + else if (strEqZ(optDef.value, "deprecate")) + { + bldCfgParseOptionDeprecate(yaml); + } + else + { + YamlEvent optDefVal = yamlEventNextCheck(yaml, yamlEventTypeScalar); + + if (strEqZ(optDef.value, "default")) + { + } + else if (strEqZ(optDef.value, "default-literal")) + { + } + else if (strEqZ(optDef.value, "group")) + { + optData.group = optDefVal.value; + } + else if (strEqZ(optDef.value, "inherit")) + { + const BldCfgOption *const optInherit = lstFind(result.optList, &optDefVal.value); + CHECK(optInherit != NULL); + + optData = *optInherit; + optData.name = opt.value; + } + else if (strEqZ(optDef.value, "internal")) + { + } + else if (strEqZ(optDef.value, "negate")) + { + } + else if (strEqZ(optDef.value, "required")) + { + } + else if (strEqZ(optDef.value, "section")) + { + } + else if (strEqZ(optDef.value, "secure")) + { + } + else if (strEqZ(optDef.value, "type")) + { + optData.type = optDefVal.value; + } + else + THROW_FMT(FormatError, "unknown option definition '%s'", strZ(optDef.value)); + } + + optDef = yamlEventNext(yaml); + } + while (optDef.type != yamlEventTypeMapEnd); + + lstAdd(result.optList, &optData); + + opt = yamlEventNext(yaml); + } + while (opt.type != yamlEventTypeMapEnd); + + lstSort(result.optList, sortOrderAsc); + + return result; +} diff --git a/src/build/config/parse.h b/src/build/config/parse.h new file mode 100644 index 0000000000..2968f8d5d4 --- /dev/null +++ b/src/build/config/parse.h @@ -0,0 +1,53 @@ +/*********************************************************************************************************************************** +Parse Configuration Yaml +***********************************************************************************************************************************/ +#ifndef BUILD_CONFIG_PARSE_H +#define BUILD_CONFIG_PARSE_H + +/*********************************************************************************************************************************** +Types +***********************************************************************************************************************************/ +typedef struct BldCfgCommand +{ + const String *name; // Name + bool logFile; // Does the command write automatically to a log file? + const String *logLevelDefault; // Default log level + bool lockRequired; // Is a lock required + bool lockRemoteRequired; // Is a remote lock required? + const String *lockType; // Lock type +} BldCfgCommand; + +typedef struct BldCfgOptionGroup +{ + const String *name; // Name +} BldCfgOptionGroup; + +typedef struct BldCfgOptionCommand +{ + const String *name; // Name + const StringList *allowList; // Allowed value list +} BldCfgOptionCommand; + +typedef struct BldCfgOption +{ + const String *name; // Name + const String *type; // Option type, e.g. integer + const String *group; // Option group, if any + const StringList *allowList; // Allowed value list + const List *cmdList; // Command override list +} BldCfgOption; + +typedef struct BldCfg +{ + List *commandList; // Command list + List *optGrpList; // Option group list + List *optList; // Option list +} BldCfg; + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Parse config.yaml +BldCfg bldCfgParse(const Storage *const storageRepo); + +#endif diff --git a/src/build/config/render.c b/src/build/config/render.c new file mode 100644 index 0000000000..3506f6dc2e --- /dev/null +++ b/src/build/config/render.c @@ -0,0 +1,330 @@ +/*********************************************************************************************************************************** +Render Configuration Yaml +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include + +#include "common/log.h" +#include "common/type/convert.h" +#include "storage/posix/storage.h" + +#include "build/common/render.h" +#include "build/config/render.h" + +/*********************************************************************************************************************************** +Option type constants +***********************************************************************************************************************************/ +#define CFGDEF_TYPE_STRING "string" + +/*********************************************************************************************************************************** +Build constant from a string +***********************************************************************************************************************************/ +static String * +bldConst(const char *const prefix, const String *const value) +{ + return strUpper(strReplaceChr(strNewFmt("%s_%s", prefix, strZ(value)), '-', '_')); +} + +/*********************************************************************************************************************************** +Build enum from a string +***********************************************************************************************************************************/ +static String * +bldEnum(const char *const prefix, const String *const value) +{ + String *const result = strNewZ(prefix); + const char *const valuePtr = strZ(value); + + bool upper = true; + + for (unsigned int valueIdx = 0; valueIdx < strSize(value); valueIdx++) + { + strCatChr(result, upper ? (char)toupper(valuePtr[valueIdx]) : valuePtr[valueIdx]); + upper = false; + + if (valuePtr[valueIdx + 1] == '-') + { + upper = true; + valueIdx++; + } + } + + return result; +} + +// Build command enum from a string +static String * +bldEnumCmd(const String *const value) +{ + return bldEnum("cfgCmd", value); +} + +// Build option group enum from a string +static String * +bldEnumOptGrp(const String *const value) +{ + return bldEnum("cfgOptGrp", value); +} + +// Build option enum from a string +static String * +bldEnumOpt(const String *const value) +{ + return bldEnum("cfgOpt", value); +} + +/**********************************************************************************************************************************/ +void +bldCfgRender(const Storage *const storageRepo, const BldCfg bldCfg) +{ + // Build Header + // ----------------------------------------------------------------------------------------------------------------------------- + const String *const header = bldHeader("config", "Command and Option Configuration"); + String *config = strNewFmt( + "%s" + "#ifndef CONFIG_CONFIG_AUTO_H\n" + "#define CONFIG_CONFIG_AUTO_H\n", + strZ(header)); + + // Command constants + // ----------------------------------------------------------------------------------------------------------------------------- + strCatZ( + config, + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Command constants\n" + COMMENT_BLOCK_END "\n"); + + unsigned int cmdTotal = 0; + + for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldCfg.commandList); cmdIdx++) + { + const BldCfgCommand *const cmd = lstGet(bldCfg.commandList, cmdIdx); + + strCatFmt(config, "%s\n", strZ(bldDefineRender(bldConst("CFGCMD", cmd->name), strNewFmt("\"%s\"", strZ(cmd->name))))); + + cmdTotal++; + } + + strCatFmt(config, "\n%s\n", strZ(bldDefineRender(STRDEF("CFG_COMMAND_TOTAL"), strNewFmt("%u", cmdTotal)))); + + // Option group constants + // ----------------------------------------------------------------------------------------------------------------------------- + strCatZ( + config, + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Option group constants\n" + COMMENT_BLOCK_END "\n"); + + unsigned int optGrpTotal = 0; + + for (unsigned int optGrpIdx = 0; optGrpIdx < lstSize(bldCfg.optGrpList); optGrpIdx++) + optGrpTotal++; + + strCatFmt(config, "%s\n", strZ(bldDefineRender(STRDEF("CFG_OPTION_GROUP_TOTAL"), strNewFmt("%u", optGrpTotal)))); + + // Option constants + // ----------------------------------------------------------------------------------------------------------------------------- + strCatZ( + config, + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Option constants\n" + COMMENT_BLOCK_END "\n"); + + unsigned int optTotal = 0; + + for (unsigned int optIdx = 0; optIdx < lstSize(bldCfg.optList); optIdx++) + { + const BldCfgOption *const opt = lstGet(bldCfg.optList, optIdx); + + if (opt->group == NULL) + strCatFmt(config, "%s\n", strZ(bldDefineRender(bldConst("CFGOPT", opt->name), strNewFmt("\"%s\"", strZ(opt->name))))); + + optTotal++; + } + + strCatFmt(config, "\n%s\n", strZ(bldDefineRender(STRDEF("CFG_OPTION_TOTAL"), strNewFmt("%u", optTotal)))); + + // Option value constants + // ----------------------------------------------------------------------------------------------------------------------------- + bool lf = false; + + strCatZ( + config, + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Option value constants\n" + COMMENT_BLOCK_END "\n"); + + for (unsigned int optIdx = 0; optIdx < lstSize(bldCfg.optList); optIdx++) + { + const BldCfgOption *const opt = lstGet(bldCfg.optList, optIdx); + + if (strEqZ(opt->type, CFGDEF_TYPE_STRING)) + { + StringList *const allowList = strLstNew(); + + if (opt->allowList != NULL) + { + for (unsigned int allowListIdx = 0; allowListIdx < strLstSize(opt->allowList); allowListIdx++) + strLstAddIfMissing(allowList, strLstGet(opt->allowList, allowListIdx)); + } + + if (opt->cmdList != NULL) + { + for (unsigned int optCmdListIdx = 0; optCmdListIdx < lstSize(opt->cmdList); optCmdListIdx++) + { + BldCfgOptionCommand *optCmd = lstGet(opt->cmdList, optCmdListIdx); + + if (optCmd->allowList != NULL) + { + for (unsigned int allowListIdx = 0; allowListIdx < strLstSize(optCmd->allowList); allowListIdx++) + strLstAddIfMissing(allowList, strLstGet(optCmd->allowList, allowListIdx)); + } + } + } + + strLstSort(allowList, sortOrderAsc); + + if (!strLstEmpty(allowList)) + { + if (lf) + strCatChr(config, '\n'); + + for (unsigned int allowListIdx = 0; allowListIdx < strLstSize(allowList); allowListIdx++) + { + const String *const allowListItem = strLstGet(allowList, allowListIdx); + + strCatFmt( + config, "%s\n", + strZ( + bldDefineRender( + strUpper( + strReplaceChr(strNewFmt("CFGOPTVAL_%s_%s_Z", strZ(opt->name), strZ(allowListItem)), '-', '_')), + strNewFmt("\"%s\"", strZ(allowListItem))))); + } + + lf = true; + } + } + } + + // Command enum + // ----------------------------------------------------------------------------------------------------------------------------- + strCatZ( + config, + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Command enum\n" + COMMENT_BLOCK_END "\n" + "typedef enum\n" + "{\n"); + + for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldCfg.commandList); cmdIdx++) + { + const BldCfgCommand *const cmd = lstGet(bldCfg.commandList, cmdIdx); + + strCatFmt(config, " %s,\n", strZ(bldEnumCmd(cmd->name))); + } + + strCatFmt(config, " %s,\n", strZ(bldEnumCmd(STRDEF("none")))); + + strCatZ( + config, + "} ConfigCommand;\n"); + + // Option group enum + // ----------------------------------------------------------------------------------------------------------------------------- + strCatZ( + config, + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Option group enum\n" + COMMENT_BLOCK_END "\n" + "typedef enum\n" + "{\n"); + + for (unsigned int optGrpIdx = 0; optGrpIdx < lstSize(bldCfg.optGrpList); optGrpIdx++) + { + const BldCfgOptionGroup *const optGrp = lstGet(bldCfg.optGrpList, optGrpIdx); + + strCatFmt(config, " %s,\n", strZ(bldEnumOptGrp(optGrp->name))); + } + + strCatZ( + config, + "} ConfigOptionGroup;\n"); + + // Option enum + // ----------------------------------------------------------------------------------------------------------------------------- + strCatZ( + config, + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Option enum\n" + COMMENT_BLOCK_END "\n" + "typedef enum\n" + "{\n"); + + for (unsigned int optIdx = 0; optIdx < lstSize(bldCfg.optList); optIdx++) + { + const BldCfgOptionGroup *const opt = lstGet(bldCfg.optList, optIdx); + + strCatFmt(config, " %s,\n", strZ(bldEnumOpt(opt->name))); + } + + strCatZ( + config, + "} ConfigOption;\n"); + + // End and save + strCatZ( + config, + "\n" + "#endif\n"); + + storagePutP(storageNewWriteP(storageRepo, STRDEF("src/config/config.auto.h"), .noSyncPath = true), BUFSTR(config)); + + // Build C + // ----------------------------------------------------------------------------------------------------------------------------- + config = strNewFmt( + "%s" + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Command data\n" + COMMENT_BLOCK_END "\n" + "static const ConfigCommandData configCommandData[CFG_COMMAND_TOTAL] = CONFIG_COMMAND_LIST\n" + "(\n", + strZ(header)); + + for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldCfg.commandList); cmdIdx++) + { + const BldCfgCommand *const cmd = lstGet(bldCfg.commandList, cmdIdx); + + if (cmdIdx != 0) + strCatZ(config, "\n"); + + strCatFmt( + config, + " CONFIG_COMMAND\n" + " (\n" + " CONFIG_COMMAND_NAME(%s)\n" + "\n" + " CONFIG_COMMAND_LOG_FILE(%s)\n" + " CONFIG_COMMAND_LOG_LEVEL_DEFAULT(%s)\n" + " CONFIG_COMMAND_LOCK_REQUIRED(%s)\n" + " CONFIG_COMMAND_LOCK_REMOTE_REQUIRED(%s)\n" + " CONFIG_COMMAND_LOCK_TYPE(%s)\n" + " )\n", + strZ(bldConst("CFGCMD", cmd->name)), cvtBoolToConstZ(cmd->logFile), strZ(bldEnum("logLevel", cmd->logLevelDefault)), + cvtBoolToConstZ(cmd->lockRequired), cvtBoolToConstZ(cmd->lockRemoteRequired), strZ(bldEnum("lockType", cmd->lockType))); + } + + strCatZ( + config, + ")\n"); + + storagePutP(storageNewWriteP(storageRepo, STRDEF("src/config/config.auto.c"), .noSyncPath = true), BUFSTR(config)); +} diff --git a/src/build/config/render.h b/src/build/config/render.h new file mode 100644 index 0000000000..285fb3c436 --- /dev/null +++ b/src/build/config/render.h @@ -0,0 +1,15 @@ +/*********************************************************************************************************************************** +Render Configuration Yaml +***********************************************************************************************************************************/ +#ifndef BUILD_CONFIG_RENDER_H +#define BUILD_CONFIG_RENDER_H + +#include "build/config/parse.h" + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Render auto-generated config files +void bldCfgRender(const Storage *const storageRepo, const BldCfg bldCfg); + +#endif diff --git a/src/build/configure.ac b/src/build/configure.ac index b0ebeb2deb..e4e6523e78 100644 --- a/src/build/configure.ac +++ b/src/build/configure.ac @@ -119,6 +119,12 @@ fi AC_CHECK_LIB([xml2], [xmlSaveToBuffer], [], [AC_MSG_ERROR([library 'xml2' is required])]) AC_CHECK_HEADER(libxml/parser.h, [], [AC_MSG_ERROR([header file is required])]) +# Check required yaml library (only required for build) +# ---------------------------------------------------------------------------------------------------------------------------------- +AC_CHECK_LIB( + [yaml], [yaml_parser_initialize], [AC_SUBST(LIBS_BUILD, "${LIBS_BUILD} -lyaml")], [AC_MSG_ERROR([library 'yaml' is required])]) +AC_CHECK_HEADER(zlib.h, [], [AC_MSG_ERROR([header file is required])]) + # Check required gz library # ---------------------------------------------------------------------------------------------------------------------------------- AC_CHECK_LIB([z], [deflate], [], [AC_MSG_ERROR([library 'z' is required])]) diff --git a/src/config/config.auto.c b/src/config/config.auto.c index c1b7e2cb73..0544af70ff 100644 --- a/src/config/config.auto.c +++ b/src/config/config.auto.c @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** Command and Option Configuration -Automatically generated by Build.pm -- do not modify directly. +Automatically generated by 'make build-config' -- do not modify directly. ***********************************************************************************************************************************/ /*********************************************************************************************************************************** diff --git a/src/config/config.auto.h b/src/config/config.auto.h index d3dce6512f..3300450ad4 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** Command and Option Configuration -Automatically generated by Build.pm -- do not modify directly. +Automatically generated by 'make build-config' -- do not modify directly. ***********************************************************************************************************************************/ #ifndef CONFIG_CONFIG_AUTO_H #define CONFIG_CONFIG_AUTO_H diff --git a/src/configure b/src/configure index b96e5b89a5..e591bb87b7 100755 --- a/src/configure +++ b/src/configure @@ -621,6 +621,7 @@ ac_includes_default="\ ac_subst_vars='LTLIBOBJS LIBOBJS +LIBS_BUILD XML2_CONFIG_EXISTS XML2_CONFIG EGREP @@ -4077,6 +4078,60 @@ fi +# Check required yaml library (only required for build) +# ---------------------------------------------------------------------------------------------------------------------------------- +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for yaml_parser_initialize in -lyaml" >&5 +$as_echo_n "checking for yaml_parser_initialize in -lyaml... " >&6; } +if ${ac_cv_lib_yaml_yaml_parser_initialize+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lyaml $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char yaml_parser_initialize (); +int +main () +{ +return yaml_parser_initialize (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_yaml_yaml_parser_initialize=yes +else + ac_cv_lib_yaml_yaml_parser_initialize=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_yaml_yaml_parser_initialize" >&5 +$as_echo "$ac_cv_lib_yaml_yaml_parser_initialize" >&6; } +if test "x$ac_cv_lib_yaml_yaml_parser_initialize" = xyes; then : + LIBS_BUILD="${LIBS_BUILD} -lyaml" + +else + as_fn_error $? "library 'yaml' is required" "$LINENO" 5 +fi + +ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default" +if test "x$ac_cv_header_zlib_h" = xyes; then : + +else + as_fn_error $? "header file is required" "$LINENO" 5 +fi + + + # Check required gz library # ---------------------------------------------------------------------------------------------------------------------------------- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for deflate in -lz" >&5 @@ -5609,4 +5664,4 @@ if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} fi -# Generated from src/build/configure.ac sha1 dc219d0c0aa8df2e94dca9a451d15f319add2797 +# Generated from src/build/configure.ac sha1 57c7d495f9e82c4aa9aed1b3358b94c4b65c1baf diff --git a/test/Vagrantfile b/test/Vagrantfile index b8bd1ca35e..68c9fb02ca 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -148,13 +148,14 @@ Vagrant.configure(2) do |config| # Basic environment to build/test pgBackRest using homebrew installed in the local user account. #------------------------------------------------------------------------------------------------------------------------------- # mkdir ~/homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C ~/homebrew - # ~/homebrew/bin/brew install -q libpq libxml2 cpanm lcov + # ~/homebrew/bin/brew install -q libpq libxml2 libyaml cpanm lcov # ~/homebrew/bin/cpanm --force --local-lib=~/homebrew/perl5 install YAML::XS XML::Checker::Parser # # export PATH="${HOME?}/homebrew/bin:$PATH" # export LDFLAGS="-L${HOME?}/homebrew/opt/openssl@1.1/lib -L${HOME?}/homebrew/opt/libpq/lib -L${HOME?}/homebrew/opt/libxml2/lib" + # export LDFLAGS="${LDFLAGS?} -L${HOME?}/homebrew/opt/libyaml/lib" # export CPPFLAGS="-I${HOME?}/homebrew/opt/openssl@1.1/include -I/${HOME?}/homebrew/opt/libpq/include" - # export CPPFLAGS="${CPPFLAGS?} -I${HOME?}/homebrew/opt/libxml2/include/libxml2" + # export CPPFLAGS="${CPPFLAGS?} -I${HOME?}/homebrew/opt/libxml2/include/libxml2 -I/${HOME?}/homebrew/opt/libyaml/include" # export PERL5LIB=~/homebrew/perl5/lib/perl5"${PERL5LIB:+:${PERL5LIB}}" # Don't share the default vagrant folder diff --git a/test/ci.pl b/test/ci.pl index 193e88eeca..5adfa222f5 100755 --- a/test/ci.pl +++ b/test/ci.pl @@ -177,7 +177,7 @@ sub processEnd elsif ($ARGV[0] eq 'test') { # Build list of packages that need to be installed - my $strPackage = "rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev pkg-config"; + my $strPackage = "rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config"; # Add lcov when testing coverage if (vmCoverageC($strVm)) diff --git a/test/define.yaml b/test/define.yaml index d9123a8704..08dffb1812 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -603,6 +603,25 @@ unit: include: - storage/write + # ******************************************************************************************************************************** + - name: build + + test: + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common + total: 1 + + coverage: + - build/common/yaml + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: config + total: 1 + + coverage: + - build/config/parse + - build/config/render + # ******************************************************************************************************************************** - name: info diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index b333232775..662a5402d4 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -262,7 +262,7 @@ sub run "CFLAGS_CONFIG = \@CFLAGS\@\n" . "CPPFLAGS_CONFIG = \@CPPFLAGS\@\n" . "LDFLAGS_CONFIG = \@LDFLAGS\@\n" . - "LIBS_CONFIG = \@LIBS\@\n"; + "LIBS_CONFIG = \@LIBS\@ \@LIBS_BUILD\@\n"; # If Makefile.in has changed then configure needs to be run and all files cleaned if (buildPutDiffers($self->{oStorageTest}, $self->{strGCovPath} . "/Makefile.in", $strMakefileIn)) diff --git a/test/src/module/build/commonTest.c b/test/src/module/build/commonTest.c new file mode 100644 index 0000000000..906ddba6a4 --- /dev/null +++ b/test/src/module/build/commonTest.c @@ -0,0 +1,60 @@ +/*********************************************************************************************************************************** +Test Build Common +***********************************************************************************************************************************/ + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +void +testRun(void) +{ + FUNCTION_HARNESS_VOID(); + + // ***************************************************************************************************************************** + if (testBegin("Yaml")) + { + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("parse and error"); + + const Buffer *buffer = BUFSTRZ( + "test:\n" + " main: [0, 1]\n" + " default: text\n"); + + Yaml *yaml = NULL; + TEST_ASSIGN(yaml, yamlNew(buffer), "new yaml") + TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeMapBegin), "map begin event"); + TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeScalar), "scalar event"); + TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeMapBegin), "map begin event"); + TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeScalar), "scalar event"); + TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeSeqBegin), "seq begin event"); + TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeScalar), "scalar event"); + TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeScalar), "scalar event"); + TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeSeqEnd), "seq end event"); + TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeMapEnd), "map end event"); + TEST_ERROR( + yamlEventNextCheck(yaml, yamlEventTypeScalar), FormatError, + "yaml parse error: did not find expected key at line: 3 column: 3"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("boolean parse"); + + TEST_RESULT_BOOL(yamlBoolParse((YamlEvent){.value = STRDEF("true")}), true, "true"); + TEST_RESULT_BOOL(yamlBoolParse((YamlEvent){.value = STRDEF("false")}), false, "false"); + TEST_ERROR(yamlBoolParse((YamlEvent){.value = STRDEF("ack")}), FormatError, "invalid boolean 'ack'"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("type map (remaning types)"); + + TEST_RESULT_UINT(yamlEventType(YAML_STREAM_END_EVENT), yamlEventTypeStreamEnd, "stream end"); + TEST_RESULT_UINT(yamlEventType(YAML_DOCUMENT_END_EVENT), yamlEventTypeDocEnd, "doc end"); + TEST_RESULT_UINT(yamlEventType(YAML_ALIAS_EVENT), yamlEventTypeAlias, "alias"); + TEST_RESULT_UINT(yamlEventType(YAML_NO_EVENT), yamlEventTypeNone, "none"); + + TEST_ERROR( + yamlEventCheck((YamlEvent){.type = yamlEventTypeAlias}, yamlEventTypeScalar), FormatError, + "expected event type 'scalar' but got 'alias'"); + } + + FUNCTION_HARNESS_RETURN_VOID(); +} diff --git a/test/src/module/build/configTest.c b/test/src/module/build/configTest.c new file mode 100644 index 0000000000..51fda3bd34 --- /dev/null +++ b/test/src/module/build/configTest.c @@ -0,0 +1,300 @@ +/*********************************************************************************************************************************** +Test Build Config +***********************************************************************************************************************************/ +#include "common/harnessStorage.h" + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +void +testRun(void) +{ + FUNCTION_HARNESS_VOID(); + + // Create default storage object for testing + Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true); + + // ***************************************************************************************************************************** + if (testBegin("bldCfgParse() and bldCfgRender()")) + { + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("command parse errors"); + + HRN_STORAGE_PUT_Z( + storageTest, "src/build/config/config.yaml", + "command:\n" + " archive-get:\n" + " bogus: test\n"); + + TEST_ERROR(bldCfgParse(storageTest), FormatError, "unknown command definition 'bogus'"); + + #define TEST_COMMAND_VALID \ + "command:\n" \ + " archive-get:\n" \ + " internal: true\n" \ + "\n" + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("option group parse errors"); + + HRN_STORAGE_PUT_Z( + storageTest, "src/build/config/config.yaml", + TEST_COMMAND_VALID + "optionGroup:\n" + " repo:\n" + " bogus: test\n"); + + TEST_ERROR(bldCfgParse(storageTest), FormatError, "unknown option group definition 'bogus'"); + + #define TEST_OPTION_GROUP_VALID \ + "optionGroup:\n" \ + " repo:\n" \ + " prefix: repo\n" \ + "\n" + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("option parse errors"); + + HRN_STORAGE_PUT_Z( + storageTest, "src/build/config/config.yaml", + TEST_COMMAND_VALID + TEST_OPTION_GROUP_VALID + "option:\n" + " config:\n" + " bogus: test\n"); + + TEST_ERROR(bldCfgParse(storageTest), FormatError, "unknown option definition 'bogus'"); + + HRN_STORAGE_PUT_Z( + storageTest, "src/build/config/config.yaml", + TEST_COMMAND_VALID + TEST_OPTION_GROUP_VALID + "option:\n" + " config:\n" + " depend:\n" + " bogus: test\n"); + + TEST_ERROR(bldCfgParse(storageTest), FormatError, "unknown depend definition 'bogus'"); + + HRN_STORAGE_PUT_Z( + storageTest, "src/build/config/config.yaml", + TEST_COMMAND_VALID + TEST_OPTION_GROUP_VALID + "option:\n" + " config:\n" + " deprecate:\n" + " old: {bogus: test}\n"); + + TEST_ERROR(bldCfgParse(storageTest), FormatError, "unknown deprecate definition 'bogus'"); + + HRN_STORAGE_PUT_Z( + storageTest, "src/build/config/config.yaml", + TEST_COMMAND_VALID + TEST_OPTION_GROUP_VALID + "option:\n" + " config:\n" + " command:\n" + " backup:\n" + " bogus: test\n"); + + TEST_ERROR(bldCfgParse(storageTest), FormatError, "unknown option command definition 'bogus'"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("parse and render config"); + + HRN_STORAGE_PUT_Z( + storageTest, "src/build/config/config.yaml", + "command:\n" + " archive-get:\n" + " command-role:\n" + " async: {}\n" + " local: {}\n" + " remote: {}\n" + " lock-type: archive\n" + " log-file: false\n" + " log-level-default: debug\n" + " parameter-allowed: true\n" + "\n" + " backup:\n" + " internal: true\n" + " command-role:\n" + " local: {}\n" + " remote: {}\n" + " lock-type: backup\n" + " lock-required: true\n" + " lock-remote-required: true\n" + "\n" + " help: {}\n" + "\n" + "optionGroup:\n" + " pg:\n" + " indexTotal: 8\n" + " prefix: pg\n" + " repo:\n" + " indexTotal: 4\n" + " prefix: repo\n" + "\n" + "option:\n" + " compress-type:\n" + " section: global\n" + " type: string\n" + " default: gz\n" + " command-role: {}\n" + " deprecate:\n" + " compress: {}\n" + "\n" + " compress-level:\n" + " section: global\n" + " type: integer\n" + " required: false\n" + " allow-range: [0, 9]\n" + " command: compress-type\n" + " depend: compress-type\n" + "\n" + " compress-level-network:\n" + " inherit: compress-level\n" + " internal: true\n" + " secure: true\n" + " default: 3\n" + " depend:\n" + " option: compress-type\n" + " list:\n" + " - none\n" + " - gz\n" + "\n" + " config:\n" + " type: string\n" + " default: CFGOPTDEF_CONFIG_PATH \"/\" PROJECT_CONFIG_FILE\n" + " default-literal: true\n" + " negate: y\n" + "\n" + " log-level-console:\n" + " section: global\n" + " type: string\n" + " default: warn\n" + " allow-list:\n" + " - off\n" + " - error\n" + " - warn\n" + " - debug\n" + "\n" + " log-level-file:\n" + " section: global\n" + " type: string\n" + " default: info\n" + " allow-list: log-level-console\n" + " command:\n" + " backup:\n" + " internal: true\n" + " required: false\n" + " default: warn\n" + " allow-list:\n" + " - off\n" + " - warn\n" + " depend:\n" + " option: log-level-console\n" + " list:\n" + " - warn\n" + " command-role:\n" + " main: {}\n" + " archive-get: {}\n" + "\n" + " pg-path:\n" + " group: pg\n" + " type: path\n" + " deprecate:\n" + " db-path: {index: 1, reset: false}\n" + " db?-path: {reset: false}\n"); + + TEST_RESULT_VOID(bldCfgRender(storageTest, bldCfgParse(storageTest)), "parse and render"); + + TEST_STORAGE_GET( + storageTest, + "src/config/config.auto.h", + COMMENT_BLOCK_BEGIN "\n" + "Command and Option Configuration\n" + "\n" + "Automatically generated by 'make build-config' -- do not modify directly.\n" + COMMENT_BLOCK_END "\n" + "#ifndef CONFIG_CONFIG_AUTO_H\n" + "#define CONFIG_CONFIG_AUTO_H\n" + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Command constants\n" + COMMENT_BLOCK_END "\n" + "#define CFGCMD_ARCHIVE_GET \"archive-get\"\n" + "#define CFGCMD_BACKUP \"backup\"\n" + "#define CFGCMD_HELP \"help\"\n" + "\n" + "#define CFG_COMMAND_TOTAL 3\n" + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Option group constants\n" + COMMENT_BLOCK_END "\n" + "#define CFG_OPTION_GROUP_TOTAL 2\n" + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Option constants\n" + COMMENT_BLOCK_END "\n" + "#define CFGOPT_COMPRESS_LEVEL \"compress-level\"\n" + "#define CFGOPT_COMPRESS_LEVEL_NETWORK \"compress-level-network\"\n" + "#define CFGOPT_COMPRESS_TYPE \"compress-type\"\n" + "#define CFGOPT_CONFIG \"config\"\n" + "#define CFGOPT_LOG_LEVEL_CONSOLE \"log-level-console\"\n" + "#define CFGOPT_LOG_LEVEL_FILE \"log-level-file\"\n" + "\n" + "#define CFG_OPTION_TOTAL 7\n" + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Option value constants\n" + COMMENT_BLOCK_END "\n" + "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_DEBUG_Z \"debug\"\n" + "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_ERROR_Z \"error\"\n" + "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_OFF_Z \"off\"\n" + "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_WARN_Z \"warn\"\n" + "\n" + "#define CFGOPTVAL_LOG_LEVEL_FILE_DEBUG_Z \"debug\"\n" + "#define CFGOPTVAL_LOG_LEVEL_FILE_ERROR_Z \"error\"\n" + "#define CFGOPTVAL_LOG_LEVEL_FILE_OFF_Z \"off\"\n" + "#define CFGOPTVAL_LOG_LEVEL_FILE_WARN_Z \"warn\"\n" + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Command enum\n" + COMMENT_BLOCK_END "\n" + "typedef enum\n" + "{\n" + " cfgCmdArchiveGet,\n" + " cfgCmdBackup,\n" + " cfgCmdHelp,\n" + " cfgCmdNone,\n" + "} ConfigCommand;\n" + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Option group enum\n" + COMMENT_BLOCK_END "\n" + "typedef enum\n" + "{\n" + " cfgOptGrpPg,\n" + " cfgOptGrpRepo,\n" + "} ConfigOptionGroup;\n" + "\n" + COMMENT_BLOCK_BEGIN "\n" + "Option enum\n" + COMMENT_BLOCK_END "\n" + "typedef enum\n" + "{\n" + " cfgOptCompressLevel,\n" + " cfgOptCompressLevelNetwork,\n" + " cfgOptCompressType,\n" + " cfgOptConfig,\n" + " cfgOptLogLevelConsole,\n" + " cfgOptLogLevelFile,\n" + " cfgOptPgPath,\n" + "} ConfigOption;\n" + "\n" + "#endif\n" + ); + } + + FUNCTION_HARNESS_RETURN_VOID(); +} diff --git a/test/test.pl b/test/test.pl index 3b99c041cd..272763d122 100755 --- a/test/test.pl +++ b/test/test.pl @@ -16,6 +16,7 @@ use Digest::SHA qw(sha1_hex); use File::Basename qw(dirname); +use File::stat; use Getopt::Long qw(GetOptions); use Cwd qw(abs_path cwd); use JSON::PP; @@ -35,7 +36,6 @@ use pgBackRestBuild::Build; use pgBackRestBuild::Build::Common; -use pgBackRestBuild::Config::Build; use pgBackRestBuild::Config::BuildHelp; use pgBackRestBuild::Config::BuildParse; use pgBackRestBuild::Error::Build; @@ -423,6 +423,173 @@ =head1 SYNOPSIS ################################################################################################################################ if (!defined($iVmId)) { + # Auto-generate configure files unless --no-gen specified + #--------------------------------------------------------------------------------------------------------------------------- + if (!$bNoGen) + { + &log(INFO, "autogenerate configure"); + + # Auto-generate version for configure.ac script + #----------------------------------------------------------------------------------------------------------------------- + my $strConfigureAcOld = ${$oStorageTest->get("${strBackRestBase}/src/build/configure.ac")}; + my $strConfigureAcNew; + + foreach my $strLine (split("\n", $strConfigureAcOld)) + { + if ($strLine =~ /^AC_INIT\(/) + { + $strLine = 'AC_INIT([' . PROJECT_NAME . '], [' . PROJECT_VERSION . '])'; + } + + $strConfigureAcNew .= "${strLine}\n"; + } + + # Save into the src dir + my @stryBuilt; + my $strBuilt = 'src/build/configure.ac'; + + if (buildPutDiffers($oStorageBackRest, "${strBackRestBase}/${strBuilt}", $strConfigureAcNew)) + { + push(@stryBuilt, $strBuilt); + } + + # Error when checking that files have already been generated but they change + if ($bGenCheck && @stryBuilt) + { + confess &log( + ERROR, + "unexpected autogeneration of version in configure.ac script: " . join(', ', @stryBuilt) . ":\n" . + trim(executeTest("git -C ${strBackRestBase} diff"))); + } + + &log(INFO, + " autogenerated version in configure.ac script: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); + + # Auto-generate configure script + #----------------------------------------------------------------------------------------------------------------------- + # Set build file + @stryBuilt = (); + $strBuilt = 'src/configure'; + + # Get configure.ac and configure to see if anything has changed + my $strConfigureAc = ${$oStorageBackRest->get('src/build/configure.ac')}; + my $strConfigureAcHash = sha1_hex($strConfigureAc); + my $rstrConfigure = $oStorageBackRest->get($oStorageBackRest->openRead($strBuilt, {bIgnoreMissing => true})); + + # Check if configure needs to be regenerated + if (!defined($rstrConfigure) || !defined($$rstrConfigure) || + $strConfigureAcHash ne substr($$rstrConfigure, length($$rstrConfigure) - 41, 40)) + { + # Generate aclocal.m4 + my $strAcLocal = executeTest("cd ${strBackRestBase}/src/build && aclocal --OUT=-"); + $strAcLocal = trim($strAcLocal) . "\n"; + + buildPutDiffers($oStorageBackRest, "${strBackRestBase}/src/build/aclocal.m4", $strAcLocal); + + # Generate configure + my $strConfigure = executeTest("cd ${strBackRestBase}/src/build && autoconf --output=-"); + $strConfigure = + trim($strConfigure) . "\n\n# Generated from src/build/configure.ac sha1 ${strConfigureAcHash}\n"; + + # Remove cache created by autconf + executeTest("rm -rf ${strBackRestBase}/src/build/autom4te.cache"); + + # Remove unused options from help + my $strDirList = + "sbin|libexec|sysconf|sharedstate|localstate|runstate|lib|include|oldinclude|dataroot|data|info" . + "|locale|man|doc|html|dvi|pdf|ps"; + + $strConfigure =~ s/^ --(${strDirList})*dir=DIR.*\n//mg; + + # Save into the src dir + $oStorageBackRest->put( + $oStorageBackRest->openWrite("${strBackRestBase}/${strBuilt}", {strMode => '0755'}), $strConfigure); + + # Add to built list + push(@stryBuilt, $strBuilt); + } + + # Error when checking that files have already been generated but they change + if ($bGenCheck && @stryBuilt) + { + confess &log( + ERROR, + "unexpected autogeneration of configure script: " . join(', ', @stryBuilt) . ":\n" . + trim(executeTest("git -C ${strBackRestBase} diff"))); + } + + &log(INFO, " autogenerated configure script: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); + } + + # Create the build path + #--------------------------------------------------------------------------------------------------------------------------- + my $strBuildPath = "${strTestPath}/build"; + + if (!-e "${strBuildPath}/Makefile" || + stat("${strBackRestBase}/src/Makefile.in")->mtime > stat("${strBuildPath}/Makefile")->mtime || + stat("${strBackRestBase}/src/configure")->mtime > stat("${strBuildPath}/Makefile")->mtime || + stat("${strBackRestBase}/src/build.auto.h.in")->mtime > stat("${strBuildPath}/Makefile")->mtime) + { + &log(INFO, "configure build"); + + $oStorageTest->pathCreate("${strBuildPath}/repo", {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); + executeTest("find ${strBuildPath} -mindepth 1 -print0 | xargs -0 rm -rf"); + executeTest("cd ${strBuildPath} && ${strBackRestBase}/src/configure -q --enable-test"); + } + + # Auto-generate code files unless --no-gen specified + #--------------------------------------------------------------------------------------------------------------------------- + if (!$bNoGen) + { + &log(INFO, "autogenerate code"); + + # Auto-generate C files + #----------------------------------------------------------------------------------------------------------------------- + errorDefineLoad(${$oStorageBackRest->get("build/error.yaml")}); + + my $rhBuild = + { + 'configHelp' => + { + &BLD_DATA => buildConfigHelp(), + &BLD_PATH => 'command/help', + }, + + 'configParse' => + { + &BLD_DATA => buildConfigParse(), + &BLD_PATH => 'config', + }, + + 'error' => + { + &BLD_DATA => buildError(), + &BLD_PATH => 'common', + }, + }; + + my @stryBuilt = buildAll("${strBackRestBase}/src", $rhBuild); + + # Error when checking that files have already been generated but they change + if ($bGenCheck && @stryBuilt) + { + confess &log( + ERROR, + 'unexpected autogeneration of C code: ' . join(', ', @stryBuilt) . ":\n" . + trim(executeTest("git -C ${strBackRestBase} diff"))); + } + + &log(INFO, " autogenerated C code: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); + + # Build configuration + executeTest("make -C ${strBuildPath} build-config"); + + if ($bGenOnly) + { + exit 0; + } + } + # Make a copy of the repo to track which files have been changed. Eventually all builds will be done from this directory. #--------------------------------------------------------------------------------------------------------------------------- my $strRepoCachePath = "${strTestPath}/repo"; @@ -480,181 +647,6 @@ =head1 SYNOPSIS exit 0; } - # Auto-generate files unless --no-gen specified - #--------------------------------------------------------------------------------------------------------------------------- - if (!$bNoGen) - { - my @stryBuiltAll; - &log(INFO, "check code autogenerate"); - - # Auto-generate version for configure.ac script - #----------------------------------------------------------------------------------------------------------------------- - if (!$bSmart || grep(/^src\/version\.h/, @stryModifiedList)) - { - my $strConfigureAcOld = ${$oStorageTest->get("${strBackRestBase}/src/build/configure.ac")}; - my $strConfigureAcNew; - - foreach my $strLine (split("\n", $strConfigureAcOld)) - { - if ($strLine =~ /^AC_INIT\(/) - { - $strLine = 'AC_INIT([' . PROJECT_NAME . '], [' . PROJECT_VERSION . '])'; - } - - $strConfigureAcNew .= "${strLine}\n"; - } - - # Save into the src dir - my @stryBuilt; - my $strBuilt = 'src/build/configure.ac'; - - if (buildPutDiffers($oStorageBackRest, "${strBackRestBase}/${strBuilt}", $strConfigureAcNew)) - { - push(@stryBuilt, $strBuilt); - push(@stryBuiltAll, @stryBuilt); - push(@stryModifiedList, @stryBuilt); - } - - # Error when checking that files have already been generated but they change - if ($bGenCheck && @stryBuilt) - { - confess &log( - ERROR, - "unexpected autogeneration of version in configure.ac script: " . join(', ', @stryBuilt) . ":\n" . - trim(executeTest("git -C ${strBackRestBase} diff"))); - } - - &log(INFO, - " autogenerated version in configure.ac script: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); - } - - # Auto-generate configure script - #----------------------------------------------------------------------------------------------------------------------- - if (!$bSmart || grep(/^src\/build\/configure\.ac/, @stryModifiedList)) - { - # Set build file - my @stryBuilt; - my $strBuilt = 'src/configure'; - - # Get configure.ac and configure to see if anything has changed - my $strConfigureAc = ${$oStorageBackRest->get('src/build/configure.ac')}; - my $strConfigureAcHash = sha1_hex($strConfigureAc); - my $rstrConfigure = $oStorageBackRest->get($oStorageBackRest->openRead($strBuilt, {bIgnoreMissing => true})); - - # Check if configure needs to be regenerated - if (!defined($rstrConfigure) || !defined($$rstrConfigure) || - $strConfigureAcHash ne substr($$rstrConfigure, length($$rstrConfigure) - 41, 40)) - { - # Generate aclocal.m4 - my $strAcLocal = executeTest("cd ${strBackRestBase}/src/build && aclocal --OUT=-"); - $strAcLocal = trim($strAcLocal) . "\n"; - - buildPutDiffers($oStorageBackRest, "${strBackRestBase}/src/build/aclocal.m4", $strAcLocal); - - # Generate configure - my $strConfigure = executeTest("cd ${strBackRestBase}/src/build && autoconf --output=-"); - $strConfigure = - trim($strConfigure) . "\n\n# Generated from src/build/configure.ac sha1 ${strConfigureAcHash}\n"; - - # Remove cache created by autconf - executeTest("rm -rf ${strBackRestBase}/src/build/autom4te.cache"); - - # Remove unused options from help - my $strDirList = - "sbin|libexec|sysconf|sharedstate|localstate|runstate|lib|include|oldinclude|dataroot|data|info" . - "|locale|man|doc|html|dvi|pdf|ps"; - - $strConfigure =~ s/^ --(${strDirList})*dir=DIR.*\n//mg; - - # Save into the src dir - $oStorageBackRest->put( - $oStorageBackRest->openWrite("${strBackRestBase}/${strBuilt}", {strMode => '0755'}), $strConfigure); - - # Add to built list - push(@stryBuilt, $strBuilt); - push(@stryBuiltAll, @stryBuilt); - push(@stryModifiedList, @stryBuilt); - } - - # Error when checking that files have already been generated but they change - if ($bGenCheck && @stryBuilt) - { - confess &log( - ERROR, - "unexpected autogeneration of configure script: " . join(', ', @stryBuilt) . ":\n" . - trim(executeTest("git -C ${strBackRestBase} diff"))); - } - - &log(INFO, " autogenerated configure script: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); - } - - # Auto-generate C files - #----------------------------------------------------------------------------------------------------------------------- - if (!$bSmart || grep(/^build\//, @stryModifiedList) || grep(/^doc\/xml\/reference\.xml/, @stryModifiedList) || - grep(/^src\/build\/config\/config\.yaml/, @stryModifiedList)) - { - errorDefineLoad(${$oStorageBackRest->get("build/error.yaml")}); - - my $rhBuild = - { - 'config' => - { - &BLD_DATA => buildConfig(), - &BLD_PATH => 'config', - }, - - 'configHelp' => - { - &BLD_DATA => buildConfigHelp(), - &BLD_PATH => 'command/help', - }, - - 'configParse' => - { - &BLD_DATA => buildConfigParse(), - &BLD_PATH => 'config', - }, - - 'error' => - { - &BLD_DATA => buildError(), - &BLD_PATH => 'common', - }, - }; - - my @stryBuilt = buildAll("${strBackRestBase}/src", $rhBuild); - - # Error when checking that files have already been generated but they change - if ($bGenCheck && @stryBuilt) - { - confess &log( - ERROR, - 'unexpected autogeneration of C code: ' . join(', ', @stryBuilt) . ":\n" . - trim(executeTest("git -C ${strBackRestBase} diff"))); - } - - &log(INFO, " autogenerated C code: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); - - if (@stryBuilt) - { - push(@stryBuiltAll, @stryBuilt); - push(@stryModifiedList, @stryBuilt); - } - } - - # Copy the files that were auto-generated to the repo cache so they will be included in the current build - #----------------------------------------------------------------------------------------------------------------------- - foreach my $strBuilt (@stryBuiltAll) - { - executeTest("cp -p ${strBackRestBase}/${strBuilt} ${strRepoCachePath}/${strBuilt}"); - } - - if ($bGenOnly) - { - exit 0; - } - } - # Clean up #--------------------------------------------------------------------------------------------------------------------------- my $iTestFail = 0; @@ -894,7 +886,7 @@ =head1 SYNOPSIS # Patch files in debian package builds # # Use these commands to create a new patch (may need to modify first line): - # BRDIR=/backrest;BRVM=u18;BRPATCHFILE=${BRDIR?}/test/patch/debian-package.patch + # BRDIR=/backrest;BRVM=u20;BRPATCHFILE=${BRDIR?}/test/patch/debian-package.patch # DBDIR=${BRDIR?}/test/result/package/${BRVM}/debian # diff -Naur ${DBDIR?}.old ${DBDIR}.new > ${BRPATCHFILE?} my $strDebianPackagePatch = "${strBackRestBase}/test/patch/debian-package.patch";