From c39f3fcd9b1df52f2cf4a17260fd5a890f44de2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Thu, 3 Jun 2021 10:30:59 +0200 Subject: [PATCH 1/7] Add proper "is shallow scanned" state to avoid rescanning folders --- lib/private/Files/Cache/Cache.php | 23 ++++++++++++++----- lib/private/Files/Cache/Propagator.php | 2 ++ lib/private/Files/Cache/Scanner.php | 23 ++++++++++++------- lib/private/Files/Cache/Wrapper/CacheJail.php | 2 +- .../Files/Cache/Wrapper/CacheWrapper.php | 2 +- lib/private/Files/View.php | 3 ++- lib/public/Files/Cache/ICache.php | 8 +++++++ lib/public/Files/Cache/IScanner.php | 3 +++ tests/lib/Files/Cache/CacheTest.php | 2 ++ 9 files changed, 51 insertions(+), 17 deletions(-) diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index ad9c050b3976..3f16c1a4ec79 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -38,6 +38,7 @@ use Doctrine\DBAL\Platforms\OraclePlatform; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\ICache; +use OCP\Files\Cache\IScanner; use OCP\Files\Cache\ICacheEntry; use \OCP\Files\IMimeTypeLoader; use OCP\IDBConnection; @@ -681,10 +682,11 @@ public function clear() { * - Cache::PARTIAL: File is not stored in the cache but some incomplete data is known * - Cache::SHALLOW: The folder and it's direct children are in the cache but not all sub folders are fully scanned * - Cache::COMPLETE: The file or folder, with all it's children) are fully scanned + * - Cache::NOT_SCANNED: Only the folder is in the cache. The contents are unknown, so the folder needs a scan * * @param string $file * - * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW, Cache::COMPLETE or Cache::NOT_SCANNED */ public function getStatus($file) { // normalize file @@ -694,7 +696,10 @@ public function getStatus($file) { $sql = 'SELECT `size` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'; $result = $this->connection->executeQuery($sql, [$this->getNumericStorageId(), $pathHash]); if ($row = $result->fetch()) { - if ((int)$row['size'] === -1) { + $size = (int)$row['size']; + if ($size === IScanner::SIZE_NEEDS_SCAN) { + return self::NOT_SCANNED; + } elseif ($size === IScanner::SIZE_SHALLOW_SCANNED) { return self::SHALLOW; } else { return self::COMPLETE; @@ -853,10 +858,16 @@ public function calculateFolderSize($path, $entry = null) { if ($row = $result->fetch()) { $result->closeCursor(); list($sum, $min) = \array_values($row); + if ($min === null) { + // could happen if the folder hasn't been scanned. + // we don't have any data, so return the SIZE_NEEDS_SCAN + return IScanner::SIZE_NEEDS_SCAN; + } $sum = 0 + $sum; $min = 0 + $min; - if ($min === -1) { - $totalSize = $min; + if ($min === IScanner::SIZE_NEEDS_SCAN || $min === IScanner::SIZE_SHALLOW_SCANNED) { + // current folder is shallow scanned + $totalSize = IScanner::SIZE_SHALLOW_SCANNED; } else { $totalSize = $sum; } @@ -900,8 +911,8 @@ public function getAll() { */ public function getIncomplete() { $query = $this->connection->prepare('SELECT `path` FROM `*PREFIX*filecache`' - . ' WHERE `storage` = ? AND `size` = -1 ORDER BY `fileid` DESC', 1); - $query->execute([$this->getNumericStorageId()]); + . ' WHERE `storage` = ? AND `size` = ? ORDER BY `fileid` DESC', 1); + $query->execute([$this->getNumericStorageId(), IScanner::SIZE_NEEDS_SCAN]); if ($row = $query->fetch()) { return $row['path']; } else { diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php index 5d98037d5ef4..bf2013771182 100644 --- a/lib/private/Files/Cache/Propagator.php +++ b/lib/private/Files/Cache/Propagator.php @@ -87,6 +87,7 @@ public function propagateChange($internalPath, $time, $sizeDifference = 0) { if ($sizeDifference !== 0) { // if we need to update size, only update the records with calculated size (>-1) + // special cases (IScanner::SIZE_*) have negative values, so they won't interfere $builder->set('size', $builder->createFunction( 'CASE' . ' WHEN ' . $builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT)) . @@ -168,6 +169,7 @@ public function commitBatch() { ->set('size', $sizeQuery->createFunction('`size` + ' . $sizeQuery->createParameter('size'))) ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT))) ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))) + // special cases (IScanner::SIZE_*) have negative values, so they won't interfere ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->expr()->literal(-1, IQueryBuilder::PARAM_INT))); foreach ($this->batch as $item) { diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index d8a4f95e10db..0dedf98f154e 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -190,7 +190,7 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = // only reuse data if the file hasn't explicitly changed if (isset($data['storage_mtime'], $cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) { $data['mtime'] = $cacheData['mtime']; - if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { + if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === IScanner::SIZE_NEEDS_SCAN)) { $data['size'] = $cacheData['size']; } if ($reuseExisting & self::REUSE_ETAG) { @@ -399,12 +399,19 @@ protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse foreach ($childQueue as $child => $childId) { $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock); - if ($childSize === -1) { - $size = -1; - } elseif ($size !== -1) { + if ($childSize === IScanner::SIZE_NEEDS_SCAN) { + $size = IScanner::SIZE_NEEDS_SCAN; + } elseif ($size !== IScanner::SIZE_NEEDS_SCAN) { $size += $childSize; } } + + if ($size === IScanner::SIZE_NEEDS_SCAN) { + // Mark this folder as shallow scanned to avoid hitting the storage. + // The shallow contents are known, but not the whole. + $size = IScanner::SIZE_SHALLOW_SCANNED; + } + if ($this->cacheActive) { $this->cache->update($folderId, ['size' => $size]); } @@ -431,12 +438,12 @@ private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$s if ($data) { if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) { $childQueue[$child] = $data['fileid']; - } elseif ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE_INCOMPLETE and $data['size'] === -1) { + } elseif ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE_INCOMPLETE and $data['size'] === IScanner::SIZE_NEEDS_SCAN) { // only recurse into folders which aren't fully scanned $childQueue[$child] = $data['fileid']; - } elseif (!isset($data['size']) || $data['size'] === -1) { - $size = -1; - } elseif ($size !== -1) { + } elseif (!isset($data['size']) || $data['size'] === IScanner::SIZE_NEEDS_SCAN) { + $size = IScanner::SIZE_NEEDS_SCAN; + } elseif ($size !== IScanner::SIZE_NEEDS_SCAN) { $size += $data['size']; } } diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php index b64134c210e5..834fd6b274f6 100644 --- a/lib/private/Files/Cache/Wrapper/CacheJail.php +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -187,7 +187,7 @@ public function clear() { /** * @param string $file * - * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW, Cache::COMPLETE or Cache::NOT_SCANNED */ public function getStatus($file) { return $this->cache->getStatus($this->getSourcePath($file)); diff --git a/lib/private/Files/Cache/Wrapper/CacheWrapper.php b/lib/private/Files/Cache/Wrapper/CacheWrapper.php index 2c5948fcd730..09818978d1a0 100644 --- a/lib/private/Files/Cache/Wrapper/CacheWrapper.php +++ b/lib/private/Files/Cache/Wrapper/CacheWrapper.php @@ -198,7 +198,7 @@ public function clear() { /** * @param string $file * - * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW, Cache::COMPLETE or Cache::NOT_SCANNED */ public function getStatus($file) { return $this->cache->getStatus($file); diff --git a/lib/private/Files/View.php b/lib/private/Files/View.php index 7055c463ddd9..0be05a3ac714 100644 --- a/lib/private/Files/View.php +++ b/lib/private/Files/View.php @@ -53,6 +53,7 @@ use OCP\Constants; use OCP\Events\EventEmitterTrait; use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\IScanner; use OCP\Files\FileNameTooLongException; use OCP\Files\InvalidCharacterInPathException; use OCP\Files\InvalidPathException; @@ -1399,7 +1400,7 @@ private function getCacheEntry($storage, $internalPath, $relativePath) { try { // if the file is not in the cache or needs to be updated, trigger the scanner and reload the data - if (!$data || (isset($data['size']) && ($data['size'] === -1))) { + if (!$data || (isset($data['size']) && ($data['size'] === IScanner::SIZE_NEEDS_SCAN))) { $this->lockFile($relativePath, ILockingProvider::LOCK_SHARED); if (!$storage->file_exists($internalPath)) { $this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED); diff --git a/lib/public/Files/Cache/ICache.php b/lib/public/Files/Cache/ICache.php index cc91498c96a7..6de59fddf124 100644 --- a/lib/public/Files/Cache/ICache.php +++ b/lib/public/Files/Cache/ICache.php @@ -38,6 +38,14 @@ interface ICache { public const PARTIAL = 1; //only partial data available, file not cached in the database public const SHALLOW = 2; //folder in cache, but not all child files are completely scanned public const COMPLETE = 3; + /** + * folder in cache, but contents not scanned yet. This is different than SHALLOW + * because SHALLOW implies a partial scan. + * If there are "/A/D1" (folder) and "/A/F1" (file), "/A" should be with SHALLOW status + * (we have data for D1 and F1), but "/A/D1" should be with NOT_SCANNED status because + * we don't know what is inside. + */ + public const NOT_SCANNED = 4; /** * Get the numeric storage id for this cache's storage diff --git a/lib/public/Files/Cache/IScanner.php b/lib/public/Files/Cache/IScanner.php index f04676ba13b3..4785480785bd 100644 --- a/lib/public/Files/Cache/IScanner.php +++ b/lib/public/Files/Cache/IScanner.php @@ -36,6 +36,9 @@ interface IScanner { public const REUSE_SIZE = 2; public const REUSE_ONLY_FOR_FILES = 4; // apply the etag reuse only to files, not folders + const SIZE_NEEDS_SCAN = -1; + const SIZE_SHALLOW_SCANNED = -2; // current folder might be scanned but deeper folders not + /** * scan a single file and store it in the cache * diff --git a/tests/lib/Files/Cache/CacheTest.php b/tests/lib/Files/Cache/CacheTest.php index 2b691586a787..f160326d4dd5 100644 --- a/tests/lib/Files/Cache/CacheTest.php +++ b/tests/lib/Files/Cache/CacheTest.php @@ -264,6 +264,8 @@ public function testStatus() { $this->cache->put('foo', ['size' => -1]); $this->assertEquals(Cache::PARTIAL, $this->cache->getStatus('foo')); $this->cache->put('foo', ['size' => -1, 'mtime' => 20, 'mimetype' => 'foo/file']); + $this->assertEquals(Cache::NOT_SCANNED, $this->cache->getStatus('foo')); + $this->cache->put('foo', ['size' => -2, 'mtime' => 20, 'mimetype' => 'foo/file']); $this->assertEquals(Cache::SHALLOW, $this->cache->getStatus('foo')); $this->cache->put('foo', ['size' => 10]); $this->assertEquals(Cache::COMPLETE, $this->cache->getStatus('foo')); From 45f6675b784fb56af0039477bab4192c5a4107fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Thu, 3 Jun 2021 11:29:49 +0200 Subject: [PATCH 2/7] Recalculate folder size if it's shallow scanned --- lib/private/Files/View.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/private/Files/View.php b/lib/private/Files/View.php index 0be05a3ac714..01ffa2e32f95 100644 --- a/lib/private/Files/View.php +++ b/lib/private/Files/View.php @@ -1416,6 +1416,8 @@ private function getCacheEntry($storage, $internalPath, $relativePath) { $storage->getPropagator()->propagateChange($internalPath, \time()); $data = $cache->get($internalPath); $this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED); + } elseif ($data['size'] === IScanner::SIZE_SHALLOW_SCANNED) { + $cache->correctFolderSize($internalPath, $data); } } catch (LockedException $e) { // if the file is locked we just use the old cache info From 732afca545c73954c0c75e639d640e03d651e5e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Thu, 3 Jun 2021 14:18:31 +0200 Subject: [PATCH 3/7] Don't change size to "shallow scanned" if folder has been scanned before --- lib/private/Files/Cache/Cache.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 3f16c1a4ec79..d9270b15cfff 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -858,9 +858,11 @@ public function calculateFolderSize($path, $entry = null) { if ($row = $result->fetch()) { $result->closeCursor(); list($sum, $min) = \array_values($row); - if ($min === null) { + if ($min === null && $entry['size'] < 0) { // could happen if the folder hasn't been scanned. // we don't have any data, so return the SIZE_NEEDS_SCAN + // if the size of the entry is positive it means that the entry has been scanned, + // so the folder is empty (no need to be scanned) return IScanner::SIZE_NEEDS_SCAN; } $sum = 0 + $sum; From 53dcf3da5963ef94dbdef1bd95a8234e45d10a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Thu, 3 Jun 2021 14:19:31 +0200 Subject: [PATCH 4/7] Adjust unit tests --- tests/lib/Files/Cache/CacheTest.php | 9 ++++++--- tests/lib/Files/Cache/ScannerTest.php | 19 ++++++++++++++----- tests/lib/Files/ViewTest.php | 3 ++- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/lib/Files/Cache/CacheTest.php b/tests/lib/Files/Cache/CacheTest.php index f160326d4dd5..c3d1a7269014 100644 --- a/tests/lib/Files/Cache/CacheTest.php +++ b/tests/lib/Files/Cache/CacheTest.php @@ -135,7 +135,8 @@ public function testFolder($folder) { $fileData['unkownSize'] = ['size' => -1, 'mtime' => 25, 'mimetype' => 'foo/file']; $this->cache->put($file4, $fileData['unkownSize']); - $this->assertEquals(-1, $this->cache->calculateFolderSize($folder)); + // -2 = IScanner::SIZE_SHALLOW_SCANNED + $this->assertEquals(-2, $this->cache->calculateFolderSize($folder)); $fileData['unkownSize'] = ['size' => 5, 'mtime' => 25, 'mimetype' => 'foo/file']; $this->cache->put($file4, $fileData['unkownSize']); @@ -208,7 +209,8 @@ public function testEncryptedFolder() { $fileData['unkownSize'] = ['size' => -1, 'mtime' => 25, 'mimetype' => 'foo/file']; $this->cache->put($file4, $fileData['unkownSize']); - $this->assertEquals(-1, $this->cache->calculateFolderSize($file1)); + // -2 = IScanner::SIZE_SHALLOW_SCANNED + $this->assertEquals(-2, $this->cache->calculateFolderSize($file1)); $fileData['unkownSize'] = ['size' => 5, 'mtime' => 25, 'mimetype' => 'foo/file']; $this->cache->put($file4, $fileData['unkownSize']); @@ -244,7 +246,8 @@ public function testRootFolderSizeForNonHomeStorage() { $this->assertTrue($this->cache->inCache($dir2)); // check that root size ignored the unknown sizes - $this->assertEquals(-1, $this->cache->calculateFolderSize('')); + // -2 = IScanner::SIZE_SHALLOW_SCANNED + $this->assertEquals(-2, $this->cache->calculateFolderSize('')); // clean up $this->cache->remove(''); diff --git a/tests/lib/Files/Cache/ScannerTest.php b/tests/lib/Files/Cache/ScannerTest.php index db628f32640e..c0c31ff3a7fc 100644 --- a/tests/lib/Files/Cache/ScannerTest.php +++ b/tests/lib/Files/Cache/ScannerTest.php @@ -118,7 +118,9 @@ public function testShallow() { $cachedDataFolder = $this->cache->get(''); $cachedDataFolder2 = $this->cache->get('folder'); - $this->assertEquals(-1, $cachedDataFolder['size']); + // -2 = IScanner::SIZE_SHALLOW_SCANNED + // -1 = IScanner::SIZE_NEEDS_SCAN + $this->assertEquals(-2, $cachedDataFolder['size']); $this->assertEquals(-1, $cachedDataFolder2['size']); $this->scanner->scan('folder', \OC\Files\Cache\Scanner::SCAN_SHALLOW); @@ -142,7 +144,8 @@ public function testBackgroundScan() { $this->assertFalse($this->cache->inCache('folder/bar.txt')); $this->assertFalse($this->cache->inCache('folder/2bar.txt')); $cachedData = $this->cache->get(''); - $this->assertEquals(-1, $cachedData['size']); + // -2 = IScanner::SIZE_SHALLOW_SCANNED + $this->assertEquals(-2, $cachedData['size']); $this->scanner->backgroundScan(); @@ -150,7 +153,10 @@ public function testBackgroundScan() { $this->assertTrue($this->cache->inCache('folder/bar.txt')); $cachedData = $this->cache->get(''); - $this->assertnotEquals(-1, $cachedData['size']); + // -2 = IScanner::SIZE_SHALLOW_SCANNED + // -1 = IScanner::SIZE_NEEDS_SCAN + $this->assertNotEquals(-1, $cachedData['size']); + $this->assertNotEquals(-2, $cachedData['size']); $this->assertFalse($this->cache->getIncomplete()); } @@ -167,7 +173,8 @@ public function testBackgroundScanOnlyRecurseIncomplete() { $this->cache->put('folder2', ['size' => 1]); // mark as complete $cachedData = $this->cache->get(''); - $this->assertEquals(-1, $cachedData['size']); + // -2 = IScanner::SIZE_SHALLOW_SCANNED + $this->assertEquals(-2, $cachedData['size']); $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE_INCOMPLETE, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); @@ -177,6 +184,7 @@ public function testBackgroundScanOnlyRecurseIncomplete() { $cachedData = $this->cache->get(''); $this->assertNotEquals(-1, $cachedData['size']); + $this->assertNotEquals(-2, $cachedData['size']); $this->assertFalse($this->cache->getIncomplete()); } @@ -199,11 +207,12 @@ public function testReuseExisting() { $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_ETAG); $newData = $this->cache->get(''); $this->assertSame($oldData['etag'], $newData['etag']); - $this->assertEquals(-1, $newData['size']); + $this->assertEquals(-2, $newData['size']); $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE); $oldData = $this->cache->get(''); $this->assertNotEquals(-1, $oldData['size']); + $this->assertNotEquals(-2, $oldData['size']); $this->scanner->scanFile('', \OC\Files\Cache\Scanner::REUSE_ETAG + \OC\Files\Cache\Scanner::REUSE_SIZE); $newData = $this->cache->get(''); $this->assertSame($oldData['etag'], $newData['etag']); diff --git a/tests/lib/Files/ViewTest.php b/tests/lib/Files/ViewTest.php index 8bb3dba3749a..83baa4a13829 100644 --- a/tests/lib/Files/ViewTest.php +++ b/tests/lib/Files/ViewTest.php @@ -347,7 +347,8 @@ public function testAutoScan() { $cachedData = $rootView->getFileInfo('/'); $this->assertEquals('httpd/unix-directory', $cachedData['mimetype']); - $this->assertEquals(-1, $cachedData['size']); + // -2 = IScanner::SIZE_SHALLOW_SCANNED + $this->assertEquals(-2, $cachedData['size']); $folderData = $rootView->getDirectoryContent('/substorage/folder'); $this->assertEquals('text/plain', $folderData[0]['mimetype']); From 130faa902be6070e1a409e1d1e2771182e69e74a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Fri, 4 Jun 2021 11:15:58 +0200 Subject: [PATCH 5/7] Remove code not needed any longer --- lib/private/Files/Cache/Scanner.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index 0dedf98f154e..5db1e17d270a 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -506,14 +506,10 @@ public function backgroundScan() { $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG); }, ''); } else { - $lastPath = null; - while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { + while (($path = $this->cache->getIncomplete()) !== false) { $this->runBackgroundScanJob(function () use ($path) { $this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE); }, $path); - // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() - // to make this possible - $lastPath = $path; } } } From e300cbd03e048fc3dd6ee1ec66b08015dde8898e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Fri, 4 Jun 2021 14:16:50 +0200 Subject: [PATCH 6/7] Add changelog entry --- changelog/unreleased/38804 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 changelog/unreleased/38804 diff --git a/changelog/unreleased/38804 b/changelog/unreleased/38804 new file mode 100644 index 000000000000..3708cfd51bb8 --- /dev/null +++ b/changelog/unreleased/38804 @@ -0,0 +1,15 @@ +Enhancement: Introduce new state to prevent scanning of shallow scanned folders + +Folders can be partially scanned, this means that a folder could have its closest +contents scanned (the first level), but not deeper contents. Folder "/A" could be +scanned but not "/A/B/C". + +Previously, we couldn't detect that a folder had been partially scanned, so we +triggered another scan on that folder even though we already had data in the DB. + +Now, we can detect that the folder has been partially scanned to avoid another +scan if it isn't needed. This leads to notable performance improvements in cases +where a FS hasn't been scanned fully. Note that an initial scan is still required, +and the performance will remain the same in this case. + +https://github.com/owncloud/core/pull/38804 From f877a96a94b72a78e7ed604d4407fd645a8bd792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Fri, 11 Jun 2021 09:06:36 +0200 Subject: [PATCH 7/7] CI fixes --- lib/public/Files/Cache/IScanner.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/public/Files/Cache/IScanner.php b/lib/public/Files/Cache/IScanner.php index 4785480785bd..fe2d944642eb 100644 --- a/lib/public/Files/Cache/IScanner.php +++ b/lib/public/Files/Cache/IScanner.php @@ -36,8 +36,8 @@ interface IScanner { public const REUSE_SIZE = 2; public const REUSE_ONLY_FOR_FILES = 4; // apply the etag reuse only to files, not folders - const SIZE_NEEDS_SCAN = -1; - const SIZE_SHALLOW_SCANNED = -2; // current folder might be scanned but deeper folders not + public const SIZE_NEEDS_SCAN = -1; + public const SIZE_SHALLOW_SCANNED = -2; // current folder might be scanned but deeper folders not /** * scan a single file and store it in the cache