Allow to mesure free space in object storages, based on used size and…

… quota

If the object storage backend doesn't return quota or used bytes
information, use a config override for the quota and count the size used
in Nextcloud instead.

Signed-off-by: Thomas Citharel <tcit@tcit.fr>

lib/private/Files/ObjectStore/Azure.php

@@ -25,6 +25,7 @@
 use MicrosoftAzure\Storage\Blob\BlobRestProxy;
 use MicrosoftAzure\Storage\Blob\Models\CreateBlockBlobOptions;
 use MicrosoftAzure\Storage\Common\Exceptions\ServiceException;
+use OCP\Files\FileInfo;
 use OCP\Files\ObjectStore\IObjectStore;
 
 class Azure implements IObjectStore {
@@ -133,4 +134,17 @@ public function objectExists($urn) {
 	public function copyObject($from, $to) {
 		$this->getBlobClient()->copyBlob($this->containerName, $to, $this->containerName, $from);
 	}
+
+	public function bytesUsed(): int {
+		// The only way to get the bytes used is by listing every object
+		// in the blob and sum their size
+		return FileInfo::SPACE_UNKNOWN;
+	}
+
+	public function bytesQuota(): int {
+		// Azure doesn't have a way to set a quota on a specific blob container,
+		// only on a storage account
+		// https://learn.microsoft.com/en-us/answers/questions/627442/blob-general-v2-container-quota.html
+		return FileInfo::SPACE_UNLIMITED;
+	}
 }

lib/private/Files/ObjectStore/HomeObjectStoreStorage.php

@@ -26,8 +26,9 @@
 namespace OC\Files\ObjectStore;
 
 use OC\User\User;
+use OCP\Files\IHomeStorage;
 
-class HomeObjectStoreStorage extends ObjectStoreStorage implements \OCP\Files\IHomeStorage {
+class HomeObjectStoreStorage extends ObjectStoreStorage implements IHomeStorage {
 
 	/**
 	 * The home user storage requires a user object to create a unique storage id
@@ -60,7 +61,7 @@ public function getOwner($path) {
 
 	/**
 	 * @param string $path, optional
-	 * @return \OC\User\User
+	 * @return User
 	 */
 	public function getUser($path = null) {
 		return $this->user;

lib/private/Files/ObjectStore/ObjectStoreStorage.php

@@ -40,6 +40,10 @@
 use OCP\Files\NotFoundException;
 use OCP\Files\ObjectStore\IObjectStore;
 use OCP\Files\Storage\IStorage;
+use OCP\ICacheFactory;
+use OCP\IConfig;
+use OCP\IDBConnection;
+use OCP\Server;
 
 class ObjectStoreStorage extends \OC\Files\Storage\Common {
 	use CopyDirectory;
@@ -627,4 +631,68 @@ private function copyFile(ICacheEntry $sourceEntry, string $to) {
 			throw $e;
 		}
 	}
+
+	/**
+	 * In case the admin has overwritten the global instance quota,
+	 * default to the quota minus the used object storage size known by Nextcloud
+	 * instead of unknown
+	 *
+	 * @param $path
+	 */
+	public function free_space($path) {
+		$cacheFactory = Server::get(ICacheFactory::class);
+		$memcache = $cacheFactory->createLocal('object_storage_info');
+		$cacheKey = 'object_store_used_size';
+		$size = $memcache->get($cacheKey);
+		if ($size) {
+			return $size;
+		}
+
+		// Ask for the object store backend
+		$quota = $this->objectStore->bytesQuota();
+
+		// If not given, by the object store backend, check if we have an override
+		if ($quota < 0) {
+			/** @var IConfig $config */
+			$config = Server::get(IConfig::class);
+			$quota = (int) $config->getAppValue('core', 'quota_instance_global', '0');
+		}
+
+		// If there's a quota in place
+		if ($quota > 0) {
+			// The used size, directly given by the object store backend
+			$size = $this->objectStore->bytesUsed();
+			if ($size < 0) {
+				// Otherwise we just make the sum of all of the storages sizes currently in Nextcloud
+				// Note: This will not take into account object versioning handled
+				// on the object store backend's side such as when using files_versions_s3
+				$size = $this->getUsedObjectStorageSizeInNextcloud();
+			}
+			$freeSpace = $quota - $size;
+			$memcache->set($cacheKey, $freeSpace, 5 * 60);
+			return $freeSpace;
+		}
+		// Otherwise default to unknown
+		return parent::free_space($path);
+	}
+
+	/**
+	 * Returns the sum of file sizes from all of the users storages using object storage,
+	 * as well as the global object store storage.
+	 */
+	public function getUsedObjectStorageSizeInNextcloud(): ?int {
+		$qb = Server::get(IDBConnection::class)->getQueryBuilder();
+		$result = $qb->select($qb->func()->sum('size'))
+			->from('filecache', 'fc')
+			->join('fc', 'storages', 's', $qb->expr()->eq('fc.storage', 's.numeric_id'))
+			->where($qb->expr()->like('s.id', $qb->createNamedParameter('object::%')))
+			->andWhere($qb->expr()->eq('fc.path', $qb->createNamedParameter('')))
+			->executeQuery();
+
+		$row = $result->fetchOne();
+		if ($row) {
+			return (int) $row;
+		}
+		return null;
+	}
 }

lib/private/Files/ObjectStore/S3.php

@@ -23,6 +23,7 @@
  */
 namespace OC\Files\ObjectStore;
 
+use OCP\Files\FileInfo;
 use OCP\Files\ObjectStore\IObjectStore;
 
 class S3 implements IObjectStore {
@@ -41,4 +42,16 @@ public function __construct($parameters) {
 	public function getStorageId() {
 		return $this->id;
 	}
+
+	public function bytesUsed(): int {
+		// The only way to get the bytes used is by listing every object
+		// in the bucket and sum their size
+		return FileInfo::SPACE_UNKNOWN;
+	}
+
+	public function bytesQuota(): int {
+		// The quota is not obtainable through the S3 API, only through the
+		// specific Amazon Service Quotas API
+		return FileInfo::SPACE_UNLIMITED;
+	}
 }

lib/private/Files/ObjectStore/StorageObjectStore.php

@@ -23,6 +23,7 @@
  */
 namespace OC\Files\ObjectStore;
 
+use OCP\Files\FileInfo;
 use OCP\Files\ObjectStore\IObjectStore;
 use OCP\Files\Storage\IStorage;
 use function is_resource;
@@ -91,4 +92,12 @@ public function objectExists($urn) {
 	public function copyObject($from, $to) {
 		$this->storage->copy($from, $to);
 	}
+
+	public function bytesUsed(): int {
+		return FileInfo::SPACE_UNKNOWN;
+	}
+
+	public function bytesQuota(): int {
+		return FileInfo::SPACE_UNLIMITED;
+	}
 }

lib/private/Files/ObjectStore/Swift.php

@@ -29,6 +29,7 @@
 use GuzzleHttp\Exception\BadResponseException;
 use GuzzleHttp\Psr7\Utils;
 use Icewind\Streams\RetryWrapper;
+use OCP\Files\FileInfo;
 use OCP\Files\NotFoundException;
 use OCP\Files\ObjectStore\IObjectStore;
 use OCP\Files\StorageAuthException;
@@ -152,4 +153,12 @@ public function copyObject($from, $to) {
 			'destination' => $this->getContainer()->name . '/' . $to
 		]);
 	}
+
+	public function bytesUsed(): int {
+		return $this->getContainer()->bytesUsed;
+	}
+
+	public function bytesQuota(): int {
+		return $this->getContainer()->getMetadata()['Quota-Bytes'] ?? FileInfo::SPACE_UNLIMITED;
+	}
 }

lib/public/Files/ObjectStore/IObjectStore.php

@@ -81,4 +81,18 @@ public function objectExists($urn);
 	 * @since 21.0.0
 	 */
 	public function copyObject($from, $to);
+
+	/**
+	 * Returns the number of bytes used in the object store
+	 *
+	 * @since 26.0.0
+	 */
+	public function bytesUsed(): int;
+
+	/**
+	 * Returns the eventual quota of the object store, in bytes
+	 *
+	 * @since 26.0.0
+	 */
+	public function bytesQuota(): int;
 }

tests/lib/Files/ObjectStore/FailDeleteObjectStore.php

@@ -23,6 +23,7 @@
 
 namespace Test\Files\ObjectStore;
 
+use OCP\Files\FileInfo;
 use OCP\Files\ObjectStore\IObjectStore;
 
 class FailDeleteObjectStore implements IObjectStore {
@@ -55,4 +56,12 @@ public function objectExists($urn) {
 	public function copyObject($from, $to) {
 		$this->objectStore->copyObject($from, $to);
 	}
+
+	public function bytesUsed(): int {
+		return FileInfo::SPACE_UNKNOWN;
+	}
+
+	public function bytesQuota(): int {
+		return FileInfo::SPACE_UNLIMITED;
+	}
 }

tests/lib/Files/ObjectStore/FailWriteObjectStore.php

@@ -23,6 +23,7 @@
 
 namespace Test\Files\ObjectStore;
 
+use OCP\Files\FileInfo;
 use OCP\Files\ObjectStore\IObjectStore;
 
 class FailWriteObjectStore implements IObjectStore {
@@ -56,4 +57,12 @@ public function objectExists($urn) {
 	public function copyObject($from, $to) {
 		$this->objectStore->copyObject($from, $to);
 	}
+
+	public function bytesUsed(): int {
+		return FileInfo::SPACE_UNKNOWN;
+	}
+
+	public function bytesQuota(): int {
+		return FileInfo::SPACE_UNLIMITED;
+	}
 }