add utility command for object store objects

Signed-off-by: Robin Appelman <robin@icewind.nl>

apps/files/appinfo/info.xml

@@ -38,6 +38,9 @@
 		<command>OCA\Files\Command\Get</command>
 		<command>OCA\Files\Command\Put</command>
 		<command>OCA\Files\Command\Delete</command>
+		<command>OCA\Files\Command\Object\Delete</command>
+		<command>OCA\Files\Command\Object\Get</command>
+		<command>OCA\Files\Command\Object\Put</command>
 	</commands>
 
 	<activity>

apps/files/composer/composer/autoload_classmap.php

@@ -30,6 +30,10 @@
     'OCA\\Files\\Command\\Delete' => $baseDir . '/../lib/Command/Delete.php',
     'OCA\\Files\\Command\\DeleteOrphanedFiles' => $baseDir . '/../lib/Command/DeleteOrphanedFiles.php',
     'OCA\\Files\\Command\\Get' => $baseDir . '/../lib/Command/Get.php',
+    'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php',
+    'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php',
+    'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
+    'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',
     'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php',
     'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php',
     'OCA\\Files\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php',

apps/files/composer/composer/autoload_static.php

@@ -45,6 +45,10 @@ class ComposerStaticInitFiles
         'OCA\\Files\\Command\\Delete' => __DIR__ . '/..' . '/../lib/Command/Delete.php',
         'OCA\\Files\\Command\\DeleteOrphanedFiles' => __DIR__ . '/..' . '/../lib/Command/DeleteOrphanedFiles.php',
         'OCA\\Files\\Command\\Get' => __DIR__ . '/..' . '/../lib/Command/Get.php',
+        'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php',
+        'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php',
+        'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
+        'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',
         'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php',
         'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php',
         'OCA\\Files\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php',

apps/files/lib/Command/Object/Delete.php

@@ -0,0 +1,80 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Files\Command\Object;
+
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\IDBConnection;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\QuestionHelper;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Question\ConfirmationQuestion;
+
+class Delete extends Command {
+	private ObjectUtil $objectUtils;
+	private IDBConnection $connection;
+
+	public function __construct(ObjectUtil $objectUtils, IDBConnection $connection) {
+		$this->objectUtils = $objectUtils;
+		$this->connection = $connection;
+		parent::__construct();
+	}
+
+	protected function configure(): void {
+		$this
+			->setName('files:object:delete')
+			->setDescription('Delete an object from the object store')
+			->addArgument('object', InputArgument::REQUIRED, "Object to delete")
+			->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to delete the object from, only required in cases where it can't be determined from the config");
+	}
+
+	public function execute(InputInterface $input, OutputInterface $output): int {
+		$object = $input->getArgument('object');
+		$objectStore = $this->objectUtils->getObjectStore($input->getOption("bucket"), $output);
+		if (!$objectStore) {
+			return -1;
+		}
+
+		if ($fileId = $this->objectUtils->objectExistsInDb($object)) {
+			$output->writeln("<error>Warning, object $object belongs to an existing file, deleting the object will lead to unexpected behavior if not replaced</error>");
+			$output->writeln("  Note: use <info>occ files:delete $fileId</info> to delete the file cleanly or <info>occ info:file $fileId</info> for more information about the file");
+			$output->writeln("");
+		}
+
+		if (!$objectStore->objectExists($object)) {
+			$output->writeln("<error>Object $object does not exist</error>");
+			return -1;
+		}
+
+		/** @var QuestionHelper $helper */
+		$helper = $this->getHelper('question');
+		$question = new ConfirmationQuestion("Delete $object? [y/N] ", false);
+		if ($helper->ask($input, $output, $question)) {
+			$objectStore->deleteObject($object);
+		}
+		return 0;
+	}
+}

apps/files/lib/Command/Object/Get.php

@@ -0,0 +1,80 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Files\Command\Object;
+
+use OCP\Files\File;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Get extends Command {
+	private ObjectUtil $objectUtils;
+
+	public function __construct(ObjectUtil $objectUtils) {
+		$this->objectUtils = $objectUtils;
+		parent::__construct();
+	}
+
+	protected function configure(): void {
+		$this
+			->setName('files:object:get')
+			->setDescription('Get the contents of an object')
+			->addArgument('object', InputArgument::REQUIRED, "Object to get")
+			->addArgument('output', InputArgument::REQUIRED, "Target local file to output to, use - for STDOUT")
+			->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config");
+	}
+
+	public function execute(InputInterface $input, OutputInterface $output): int {
+		$object = $input->getArgument('object');
+		$outputName = $input->getArgument('output');
+		$objectStore = $this->objectUtils->getObjectStore($input->getOption("bucket"), $output);
+		if (!$objectStore) {
+			return 1;
+		}
+
+		if (!$objectStore->objectExists($object)) {
+			$output->writeln("<error>Object $object does not exist</error>");
+			return 1;
+		} else {
+			try {
+				$source = $objectStore->readObject($object);
+			} catch (\Exception $e) {
+				$msg = $e->getMessage();
+				$output->writeln("<error>Failed to read $object from object store: $msg</error>");
+				return 1;
+			}
+			$target = $outputName === '-' ? STDOUT : fopen($outputName, 'w');
+			if (!$target) {
+				$output->writeln("<error>Failed to open $outputName for writing</error>");
+				return 1;
+			}
+
+			stream_copy_to_stream($source, $target);
+			return 0;
+		}
+	}
+
+}

apps/files/lib/Command/Object/ObjectUtil.php

@@ -0,0 +1,110 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Files\Command\Object;
+
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\Files\ObjectStore\IObjectStore;
+use OCP\IConfig;
+use OCP\IDBConnection;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class ObjectUtil {
+	private IConfig $config;
+	private IDBConnection $connection;
+
+	public function __construct(IConfig $config, IDBConnection $connection) {
+		$this->config = $config;
+		$this->connection = $connection;
+	}
+
+	private function getObjectStoreConfig(): ?array {
+		$config = $this->config->getSystemValue('objectstore_multibucket');
+		if (is_array($config)) {
+			$config['multibucket'] = true;
+			return $config;
+		}
+		$config = $this->config->getSystemValue('objectstore');
+		if (is_array($config)) {
+			if (!isset($config['multibucket'])) {
+				$config['multibucket'] = false;
+			}
+			return $config;
+		} else {
+			return null;
+		}
+	}
+
+	public function getObjectStore(?string $bucket, OutputInterface $output): ?IObjectStore {
+		$config = $this->getObjectStoreConfig();
+		if (!$config) {
+			$output->writeln("<error>Instance is not using primary object store</error>");
+			return null;
+		}
+		if ($config['multibucket'] && !$bucket) {
+			$output->writeln("<error>--bucket option required</error> because <info>multi bucket</info> is enabled.");
+			return null;
+		}
+
+		if (!isset($config['arguments'])) {
+			throw new \Exception("no arguments configured for object store configuration");
+		}
+		if (!isset($config['class'])) {
+			throw new \Exception("no class configured for object store configuration");
+		}
+
+		if ($bucket) {
+			// s3, swift
+			$config['arguments']['bucket'] = $bucket;
+			// azure
+			$config['arguments']['container'] = $bucket;
+		}
+
+		$store = new $config['class']($config['arguments']);
+		if (!$store instanceof IObjectStore) {
+			throw new \Exception("configured object store class is not an object store implementation");
+		}
+		return $store;
+	}
+
+	/**
+	 * Check if an object is referenced in the database
+	 */
+	public function objectExistsInDb(string $object): int|false {
+		if (str_starts_with($object, 'urn:oid:')) {
+			$fileId = (int)substr($object, strlen('urn:oid:'));
+			$query = $this->connection->getQueryBuilder();
+			$query->select('fileid')
+				->from('filecache')
+				->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)));
+			$result = $query->executeQuery();
+			if ($result->fetchOne() !== false) {
+				return $fileId;
+			} else {
+				return false;
+			}
+		} else {
+			return false;
+		}
+	}
+}

apps/files/lib/Command/Object/Put.php

@@ -0,0 +1,84 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Files\Command\Object;
+
+use OCP\Files\IMimeTypeDetector;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\QuestionHelper;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Question\ConfirmationQuestion;
+
+class Put extends Command {
+	private ObjectUtil $objectUtils;
+	private IMimeTypeDetector $mimeTypeDetector;
+
+	public function __construct(ObjectUtil $objectUtils, IMimeTypeDetector $mimeTypeDetector) {
+		$this->objectUtils = $objectUtils;
+		$this->mimeTypeDetector = $mimeTypeDetector;
+		parent::__construct();
+	}
+
+	protected function configure(): void {
+		$this
+			->setName('files:object:put')
+			->setDescription('Write a file to the object store')
+			->addArgument('input', InputArgument::REQUIRED, "Source local path, use - to read from STDIN")
+			->addArgument('object', InputArgument::REQUIRED, "Object to write")
+			->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket where to store the object, only required in cases where it can't be determined from the config");;
+	}
+
+	public function execute(InputInterface $input, OutputInterface $output): int {
+		$object = $input->getArgument('object');
+		$inputName = (string)$input->getArgument('input');
+		$objectStore = $this->objectUtils->getObjectStore($input->getOption("bucket"), $output);
+		if (!$objectStore) {
+			return -1;
+		}
+
+		if ($fileId = $this->objectUtils->objectExistsInDb($object)) {
+			$output->writeln("<error>Warning, object $object belongs to an existing file, overwriting the object contents can lead to unexpected behavior.</error>");
+			$output->writeln("You can use <info>occ files:put $inputName $fileId</info> to write to the file safely.");
+			$output->writeln("");
+
+			/** @var QuestionHelper $helper */
+			$helper = $this->getHelper('question');
+			$question = new ConfirmationQuestion("Write to the object anyway? [y/N] ", false);
+			if (!$helper->ask($input, $output, $question)) {
+				return -1;
+			}
+		}
+
+		$source = ($inputName === null || $inputName === '-') ? STDIN : fopen($inputName, 'r');
+		if (!$source) {
+			$output->writeln("<error>Failed to open $inputName</error>");
+			return 1;
+		}
+		$objectStore->writeObject($object, $source, $this->mimeTypeDetector->detectPath($inputName));
+		return 0;
+	}
+
+}