Skip to content
Permalink
Browse files

cats: Fix .bvfs_lsdirs when using limit and offset

Fixes #975: .bvfs_lsdirs limit offset command parameters do not work properly

The .bvfs_lsdirs command did not return the correct amount of entries
when it was used with multiple jobids together with limit and offset.
The problem was that the SQL query returned multiple rows with the same
path and Bvfs::_handle_path() removed the duplicates. Also, no matter
what offset was given, the result did always start with the . and ..
entries.

This is fixed now when using the PostgreSQL DB for Catalog by a modified
SQL query that integrates the formerly separate SQL query for the . and
.. entries with the query for the normal directory entries and the new
query no longer contains duplicates.

This version will not work with a MySQL Catalog DB because MySQL/MariaDB
lacks the DISTINCT ON feature.

Original Author: Stephan Duehr <stephan.duehr@bareos.com>
  • Loading branch information...
arogge committed Jul 9, 2019
1 parent fd63a94 commit a9a0bdd911349d4b61612495dc00cfec469fb883
@@ -655,7 +655,7 @@ bool Bvfs::ls_dirs()
db->FillQuery(filter, BareosDb::SQL_QUERY_match_query, pattern);
}
db->FillQuery(sub_dirs_query, BareosDb::SQL_QUERY_bvfs_ls_sub_dirs_5, pathid,
jobids, filter.c_str(), jobids, jobids);
jobids, jobids, filter.c_str(), jobids);

db->FillQuery(union_query, BareosDb::SQL_QUERY_bvfs_lsdirs_4,
special_dirs_query.c_str(), sub_dirs_query.c_str(), limit,
@@ -0,0 +1,31 @@
#
# for .bvfs_lsdir
#
# Retrieve special directors "." and "..".
# If attributes of this paths are known, include them,
# otherwise result for extra columns will be empty.
#
# parameter:
# %s PathId
# %s PathId
# %s JobIds ("1,2,...")
#
# row 0 1 2 3 4 5
SELECT DISTINCT ON (PathId) 'D', SpecialDir.PathId, SpecialDir.Path, JobId, LStat, FileId
FROM (
SELECT %s AS PathId, '.' AS Path

UNION

SELECT PPathId AS PathId, '..' AS Path
FROM PathHierarchy
WHERE PathId = %s
) AS SpecialDir
LEFT JOIN (
SELECT PathId, JobId, LStat, FileId
FROM File
WHERE File.Name = ''
AND File.JobId IN (%s)
ORDER BY PathId ASC, JobId DESC
) AS DirAttribute
ON (SpecialDir.PathId = DirAttribute.PathId)
@@ -35,7 +35,6 @@ FROM (
ON (PathHierarchy1.PathId = PathVisibility1.PathId)
WHERE PathHierarchy1.PPathId = %s
AND PathVisibility1.JobId IN (%s)
%s
) AS listpath1
LEFT JOIN (
SELECT PVD1.PathId AS PathId
@@ -48,6 +47,7 @@ FROM (
) AS listpath2
ON (listpath1.PathId = listpath2.PathId)
WHERE listpath2.PathId IS NULL
%s
) AS listpath3
INNER JOIN Path AS Path1 ON (listpath3.PathId = Path1.PathId)
LEFT JOIN (
@@ -0,0 +1,65 @@
# for .bvfs_lsdirs
#
# Let's retrieve the list of the visible dirs in this dir ...
#
# First, the empty filename to locate efficiently
# the dirs in the file table.
#
# Then get all the dir entries from File ...
#
# parameter:
# %s PathId
# %s JobIds ("1,2,...")
# %s extra filter
# %s JobIds ("1,2,...")
# %s JobIds ("1,2,...")
#
# row 0 1 2 3 4 5
(
SELECT DISTINCT ON (Path) 'D', PathId, Path, JobId, LStat, FileId
FROM (
SELECT
Path1.PathId AS PathId,
Path1.Path AS Path,
lower(Path1.Path) AS lpath,
listfile1.JobId AS JobId,
listfile1.LStat AS LStat,
listfile1.FileId AS FileId
FROM (
SELECT listpath1.PathId AS PathId
FROM (
SELECT DISTINCT PathHierarchy1.PathId AS PathId
FROM PathHierarchy AS PathHierarchy1
INNER JOIN Path AS Path2
ON (PathHierarchy1.PathId = Path2.PathId)
INNER JOIN PathVisibility AS PathVisibility1
ON (PathHierarchy1.PathId = PathVisibility1.PathId)
WHERE PathHierarchy1.PPathId = %s
AND PathVisibility1.JobId IN (%s)
) AS listpath1
LEFT JOIN (
SELECT PVD1.PathId AS PathId
FROM (
SELECT PV1.PathId AS PathId, MAX(JobId) AS MaxJobId
FROM PathVisibility AS PV1 WHERE JobId IN (%s) GROUP BY PathId
) AS PVD1
INNER JOIN File AS F2
ON (F2.PathId = PVD1.PathId AND F2.JobId = PVD1.MaxJobId AND F2.FileIndex = 0 AND F2.Name = '')
) AS listpath2
ON (listpath1.PathId = listpath2.PathId)
WHERE listpath2.PathId IS NULL
%s
) AS listpath3
INNER JOIN Path AS Path1 ON (listpath3.PathId = Path1.PathId)
LEFT JOIN (
SELECT File1.PathId AS PathId, File1.JobId AS JobId, File1.LStat AS LStat, File1.FileId AS FileId
FROM File AS File1
WHERE File1.Name = ''
AND File1.JobId IN (%s)
) AS listfile1
ON (listpath3.PathId = listfile1.PathId)
) AS A
ORDER BY Path ASC, JobId DESC
)


@@ -930,7 +930,6 @@ const char *BareosDbMysql::query_definitions[] = {
"ON (PathHierarchy1.PathId = PathVisibility1.PathId) "
"WHERE PathHierarchy1.PPathId = %s "
"AND PathVisibility1.JobId IN (%s) "
"%s "
") AS listpath1 "
"LEFT JOIN ( "
"SELECT PVD1.PathId AS PathId "
@@ -943,6 +942,7 @@ const char *BareosDbMysql::query_definitions[] = {
") AS listpath2 "
"ON (listpath1.PathId = listpath2.PathId) "
"WHERE listpath2.PathId IS NULL "
"%s "
") AS listpath3 "
"INNER JOIN Path AS Path1 ON (listpath3.PathId = Path1.PathId) "
"LEFT JOIN ( "
@@ -864,8 +864,8 @@ const char *BareosDbPostgresql::query_definitions[] = {
"AND Path NOT LIKE '%%/' "
,

/* 0074_bvfs_ls_special_dirs_3 */
"SELECT 'D', SpecialDir.PathId, SpecialDir.Path, JobId, LStat, FileId "
/* 0074_bvfs_ls_special_dirs_3.postgresql */
"SELECT DISTINCT ON (PathId) 'D', SpecialDir.PathId, SpecialDir.Path, JobId, LStat, FileId "
"FROM ( "
"SELECT %s AS PathId, '.' AS Path "
" "
@@ -880,12 +880,14 @@ const char *BareosDbPostgresql::query_definitions[] = {
"FROM File "
"WHERE File.Name = '' "
"AND File.JobId IN (%s) "
"ORDER BY PathId ASC, JobId DESC "
") AS DirAttribute "
"ON (SpecialDir.PathId = DirAttribute.PathId) "
,

/* 0075_bvfs_ls_sub_dirs_5 */
"SELECT 'D', PathId, Path, JobId, LStat, FileId "
/* 0075_bvfs_ls_sub_dirs_5.postgresql */
"( "
"SELECT DISTINCT ON (Path) 'D', PathId, Path, JobId, LStat, FileId "
"FROM ( "
"SELECT "
"Path1.PathId AS PathId, "
@@ -905,7 +907,6 @@ const char *BareosDbPostgresql::query_definitions[] = {
"ON (PathHierarchy1.PathId = PathVisibility1.PathId) "
"WHERE PathHierarchy1.PPathId = %s "
"AND PathVisibility1.JobId IN (%s) "
"%s "
") AS listpath1 "
"LEFT JOIN ( "
"SELECT PVD1.PathId AS PathId "
@@ -918,6 +919,7 @@ const char *BareosDbPostgresql::query_definitions[] = {
") AS listpath2 "
"ON (listpath1.PathId = listpath2.PathId) "
"WHERE listpath2.PathId IS NULL "
"%s "
") AS listpath3 "
"INNER JOIN Path AS Path1 ON (listpath3.PathId = Path1.PathId) "
"LEFT JOIN ( "
@@ -928,6 +930,8 @@ const char *BareosDbPostgresql::query_definitions[] = {
") AS listfile1 "
"ON (listpath3.PathId = listfile1.PathId) "
") AS A "
"ORDER BY Path ASC, JobId DESC "
") "
,

/* 0076_list_volumes_select_0 */
@@ -915,7 +915,6 @@ const char *BareosDbSqlite::query_definitions[] = {
"ON (PathHierarchy1.PathId = PathVisibility1.PathId) "
"WHERE PathHierarchy1.PPathId = %s "
"AND PathVisibility1.JobId IN (%s) "
"%s "
") AS listpath1 "
"LEFT JOIN ( "
"SELECT PVD1.PathId AS PathId "
@@ -928,6 +927,7 @@ const char *BareosDbSqlite::query_definitions[] = {
") AS listpath2 "
"ON (listpath1.PathId = listpath2.PathId) "
"WHERE listpath2.PathId IS NULL "
"%s "
") AS listpath3 "
"INNER JOIN Path AS Path1 ON (listpath3.PathId = Path1.PathId) "
"LEFT JOIN ( "

0 comments on commit a9a0bdd

Please sign in to comment.
You can’t perform that action at this time.