From fd63a9455a2cc0addccab9000d779d4dbae75992 Mon Sep 17 00:00:00 2001 From: Andreas Rogge Date: Tue, 9 Jul 2019 16:16:19 +0200 Subject: [PATCH 1/2] cats: adapt create_queryfiles.sh for new indent create_queryfiles.sh generated files complying with the old indentation style. This patch makes sure the generated files match out new indentation style. --- core/src/cats/dml/create_queryfiles.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/src/cats/dml/create_queryfiles.sh b/core/src/cats/dml/create_queryfiles.sh index 2b7af950cba..5be09045bf0 100755 --- a/core/src/cats/dml/create_queryfiles.sh +++ b/core/src/cats/dml/create_queryfiles.sh @@ -48,8 +48,9 @@ printf "const char *BareosDb::query_names[] = {\n" >> $QUERY_NAMES_FILE printf "%s\n\n" "$NOTE" >> $QUERY_ENUM_FILE cat >> $QUERY_ENUM_FILE << EOF class BareosDbQueryEnum { -public: - typedef enum { + public: + typedef enum + { EOF @@ -68,7 +69,7 @@ let i=0 for query in `ls ????_* | sed 's#\..*##g' | sort | uniq`; do queryname=`sed 's/[0-9]*_//' <<< $query` printf '"%s",\n' "$queryname" >> $QUERY_NAMES_FILE - printf " SQL_QUERY_%s = %s,\n" "$queryname" "$i" >> $QUERY_ENUM_FILE + printf " SQL_QUERY_%s = %s,\n" "$queryname" "$i" >> $QUERY_ENUM_FILE let i++ for db in $DATABASES; do @@ -90,8 +91,8 @@ done printf "NULL\n};\n" >> $QUERY_NAMES_FILE cat >> $QUERY_ENUM_FILE << EOF - SQL_QUERY_NUMBER = $i - } SQL_QUERY_ENUM; + SQL_QUERY_NUMBER = $i + } SQL_QUERY_ENUM; }; EOF From a9a0bdd911349d4b61612495dc00cfec469fb883 Mon Sep 17 00:00:00 2001 From: Andreas Rogge Date: Tue, 9 Jul 2019 16:31:56 +0200 Subject: [PATCH 2/2] cats: Fix .bvfs_lsdirs when using limit and offset Fixes #975: .bvfs_lsdirs limit offset command parameters do not work properly The .bvfs_lsdirs command did not return the correct amount of entries when it was used with multiple jobids together with limit and offset. The problem was that the SQL query returned multiple rows with the same path and Bvfs::_handle_path() removed the duplicates. Also, no matter what offset was given, the result did always start with the . and .. entries. This is fixed now when using the PostgreSQL DB for Catalog by a modified SQL query that integrates the formerly separate SQL query for the . and .. entries with the query for the normal directory entries and the new query no longer contains duplicates. This version will not work with a MySQL Catalog DB because MySQL/MariaDB lacks the DISTINCT ON feature. Original Author: Stephan Duehr --- core/src/cats/bvfs.cc | 2 +- .../0074_bvfs_ls_special_dirs_3.postgresql | 31 +++++++++ core/src/cats/dml/0075_bvfs_ls_sub_dirs_5 | 2 +- .../dml/0075_bvfs_ls_sub_dirs_5.postgresql | 65 +++++++++++++++++++ core/src/cats/mysql_queries.inc | 2 +- core/src/cats/postgresql_queries.inc | 14 ++-- core/src/cats/sqlite_queries.inc | 2 +- 7 files changed, 109 insertions(+), 9 deletions(-) create mode 100644 core/src/cats/dml/0074_bvfs_ls_special_dirs_3.postgresql create mode 100644 core/src/cats/dml/0075_bvfs_ls_sub_dirs_5.postgresql diff --git a/core/src/cats/bvfs.cc b/core/src/cats/bvfs.cc index 61800124783..9970e41f0f3 100644 --- a/core/src/cats/bvfs.cc +++ b/core/src/cats/bvfs.cc @@ -655,7 +655,7 @@ bool Bvfs::ls_dirs() db->FillQuery(filter, BareosDb::SQL_QUERY_match_query, pattern); } db->FillQuery(sub_dirs_query, BareosDb::SQL_QUERY_bvfs_ls_sub_dirs_5, pathid, - jobids, filter.c_str(), jobids, jobids); + jobids, jobids, filter.c_str(), jobids); db->FillQuery(union_query, BareosDb::SQL_QUERY_bvfs_lsdirs_4, special_dirs_query.c_str(), sub_dirs_query.c_str(), limit, diff --git a/core/src/cats/dml/0074_bvfs_ls_special_dirs_3.postgresql b/core/src/cats/dml/0074_bvfs_ls_special_dirs_3.postgresql new file mode 100644 index 00000000000..ab915efb50f --- /dev/null +++ b/core/src/cats/dml/0074_bvfs_ls_special_dirs_3.postgresql @@ -0,0 +1,31 @@ +# +# for .bvfs_lsdir +# +# Retrieve special directors "." and "..". +# If attributes of this paths are known, include them, +# otherwise result for extra columns will be empty. +# +# parameter: +# %s PathId +# %s PathId +# %s JobIds ("1,2,...") +# +# row 0 1 2 3 4 5 +SELECT DISTINCT ON (PathId) 'D', SpecialDir.PathId, SpecialDir.Path, JobId, LStat, FileId +FROM ( + SELECT %s AS PathId, '.' AS Path + + UNION + + SELECT PPathId AS PathId, '..' AS Path + FROM PathHierarchy + WHERE PathId = %s +) AS SpecialDir +LEFT JOIN ( + SELECT PathId, JobId, LStat, FileId + FROM File + WHERE File.Name = '' + AND File.JobId IN (%s) + ORDER BY PathId ASC, JobId DESC +) AS DirAttribute +ON (SpecialDir.PathId = DirAttribute.PathId) diff --git a/core/src/cats/dml/0075_bvfs_ls_sub_dirs_5 b/core/src/cats/dml/0075_bvfs_ls_sub_dirs_5 index 430c099be86..dfb5246093c 100644 --- a/core/src/cats/dml/0075_bvfs_ls_sub_dirs_5 +++ b/core/src/cats/dml/0075_bvfs_ls_sub_dirs_5 @@ -35,7 +35,6 @@ FROM ( ON (PathHierarchy1.PathId = PathVisibility1.PathId) WHERE PathHierarchy1.PPathId = %s AND PathVisibility1.JobId IN (%s) - %s ) AS listpath1 LEFT JOIN ( SELECT PVD1.PathId AS PathId @@ -48,6 +47,7 @@ FROM ( ) AS listpath2 ON (listpath1.PathId = listpath2.PathId) WHERE listpath2.PathId IS NULL + %s ) AS listpath3 INNER JOIN Path AS Path1 ON (listpath3.PathId = Path1.PathId) LEFT JOIN ( diff --git a/core/src/cats/dml/0075_bvfs_ls_sub_dirs_5.postgresql b/core/src/cats/dml/0075_bvfs_ls_sub_dirs_5.postgresql new file mode 100644 index 00000000000..65f2bdb6f20 --- /dev/null +++ b/core/src/cats/dml/0075_bvfs_ls_sub_dirs_5.postgresql @@ -0,0 +1,65 @@ +# for .bvfs_lsdirs +# +# Let's retrieve the list of the visible dirs in this dir ... +# +# First, the empty filename to locate efficiently +# the dirs in the file table. +# +# Then get all the dir entries from File ... +# +# parameter: +# %s PathId +# %s JobIds ("1,2,...") +# %s extra filter +# %s JobIds ("1,2,...") +# %s JobIds ("1,2,...") +# +# row 0 1 2 3 4 5 +( +SELECT DISTINCT ON (Path) 'D', PathId, Path, JobId, LStat, FileId +FROM ( + SELECT + Path1.PathId AS PathId, + Path1.Path AS Path, + lower(Path1.Path) AS lpath, + listfile1.JobId AS JobId, + listfile1.LStat AS LStat, + listfile1.FileId AS FileId + FROM ( + SELECT listpath1.PathId AS PathId + FROM ( + SELECT DISTINCT PathHierarchy1.PathId AS PathId + FROM PathHierarchy AS PathHierarchy1 + INNER JOIN Path AS Path2 + ON (PathHierarchy1.PathId = Path2.PathId) + INNER JOIN PathVisibility AS PathVisibility1 + ON (PathHierarchy1.PathId = PathVisibility1.PathId) + WHERE PathHierarchy1.PPathId = %s + AND PathVisibility1.JobId IN (%s) + ) AS listpath1 + LEFT JOIN ( + SELECT PVD1.PathId AS PathId + FROM ( + SELECT PV1.PathId AS PathId, MAX(JobId) AS MaxJobId + FROM PathVisibility AS PV1 WHERE JobId IN (%s) GROUP BY PathId + ) AS PVD1 + INNER JOIN File AS F2 + ON (F2.PathId = PVD1.PathId AND F2.JobId = PVD1.MaxJobId AND F2.FileIndex = 0 AND F2.Name = '') + ) AS listpath2 + ON (listpath1.PathId = listpath2.PathId) + WHERE listpath2.PathId IS NULL + %s + ) AS listpath3 + INNER JOIN Path AS Path1 ON (listpath3.PathId = Path1.PathId) + LEFT JOIN ( + SELECT File1.PathId AS PathId, File1.JobId AS JobId, File1.LStat AS LStat, File1.FileId AS FileId + FROM File AS File1 + WHERE File1.Name = '' + AND File1.JobId IN (%s) + ) AS listfile1 + ON (listpath3.PathId = listfile1.PathId) +) AS A +ORDER BY Path ASC, JobId DESC +) + + diff --git a/core/src/cats/mysql_queries.inc b/core/src/cats/mysql_queries.inc index 60d470fdc7d..783e0c8d97c 100644 --- a/core/src/cats/mysql_queries.inc +++ b/core/src/cats/mysql_queries.inc @@ -930,7 +930,6 @@ const char *BareosDbMysql::query_definitions[] = { "ON (PathHierarchy1.PathId = PathVisibility1.PathId) " "WHERE PathHierarchy1.PPathId = %s " "AND PathVisibility1.JobId IN (%s) " - "%s " ") AS listpath1 " "LEFT JOIN ( " "SELECT PVD1.PathId AS PathId " @@ -943,6 +942,7 @@ const char *BareosDbMysql::query_definitions[] = { ") AS listpath2 " "ON (listpath1.PathId = listpath2.PathId) " "WHERE listpath2.PathId IS NULL " + "%s " ") AS listpath3 " "INNER JOIN Path AS Path1 ON (listpath3.PathId = Path1.PathId) " "LEFT JOIN ( " diff --git a/core/src/cats/postgresql_queries.inc b/core/src/cats/postgresql_queries.inc index 1b9281186c8..0c17c43d757 100644 --- a/core/src/cats/postgresql_queries.inc +++ b/core/src/cats/postgresql_queries.inc @@ -864,8 +864,8 @@ const char *BareosDbPostgresql::query_definitions[] = { "AND Path NOT LIKE '%%/' " , -/* 0074_bvfs_ls_special_dirs_3 */ -"SELECT 'D', SpecialDir.PathId, SpecialDir.Path, JobId, LStat, FileId " +/* 0074_bvfs_ls_special_dirs_3.postgresql */ +"SELECT DISTINCT ON (PathId) 'D', SpecialDir.PathId, SpecialDir.Path, JobId, LStat, FileId " "FROM ( " "SELECT %s AS PathId, '.' AS Path " " " @@ -880,12 +880,14 @@ const char *BareosDbPostgresql::query_definitions[] = { "FROM File " "WHERE File.Name = '' " "AND File.JobId IN (%s) " + "ORDER BY PathId ASC, JobId DESC " ") AS DirAttribute " "ON (SpecialDir.PathId = DirAttribute.PathId) " , -/* 0075_bvfs_ls_sub_dirs_5 */ -"SELECT 'D', PathId, Path, JobId, LStat, FileId " +/* 0075_bvfs_ls_sub_dirs_5.postgresql */ +"( " +"SELECT DISTINCT ON (Path) 'D', PathId, Path, JobId, LStat, FileId " "FROM ( " "SELECT " "Path1.PathId AS PathId, " @@ -905,7 +907,6 @@ const char *BareosDbPostgresql::query_definitions[] = { "ON (PathHierarchy1.PathId = PathVisibility1.PathId) " "WHERE PathHierarchy1.PPathId = %s " "AND PathVisibility1.JobId IN (%s) " - "%s " ") AS listpath1 " "LEFT JOIN ( " "SELECT PVD1.PathId AS PathId " @@ -918,6 +919,7 @@ const char *BareosDbPostgresql::query_definitions[] = { ") AS listpath2 " "ON (listpath1.PathId = listpath2.PathId) " "WHERE listpath2.PathId IS NULL " + "%s " ") AS listpath3 " "INNER JOIN Path AS Path1 ON (listpath3.PathId = Path1.PathId) " "LEFT JOIN ( " @@ -928,6 +930,8 @@ const char *BareosDbPostgresql::query_definitions[] = { ") AS listfile1 " "ON (listpath3.PathId = listfile1.PathId) " ") AS A " +"ORDER BY Path ASC, JobId DESC " +") " , /* 0076_list_volumes_select_0 */ diff --git a/core/src/cats/sqlite_queries.inc b/core/src/cats/sqlite_queries.inc index d79e5b63f02..e36a31f9be8 100644 --- a/core/src/cats/sqlite_queries.inc +++ b/core/src/cats/sqlite_queries.inc @@ -915,7 +915,6 @@ const char *BareosDbSqlite::query_definitions[] = { "ON (PathHierarchy1.PathId = PathVisibility1.PathId) " "WHERE PathHierarchy1.PPathId = %s " "AND PathVisibility1.JobId IN (%s) " - "%s " ") AS listpath1 " "LEFT JOIN ( " "SELECT PVD1.PathId AS PathId " @@ -928,6 +927,7 @@ const char *BareosDbSqlite::query_definitions[] = { ") AS listpath2 " "ON (listpath1.PathId = listpath2.PathId) " "WHERE listpath2.PathId IS NULL " + "%s " ") AS listpath3 " "INNER JOIN Path AS Path1 ON (listpath3.PathId = Path1.PathId) " "LEFT JOIN ( "