Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow scanning for metadata with occ scan:file --metadata #32309

Closed
wants to merge 1 commit into from
Closed

Allow scanning for metadata with occ scan:file --metadata #32309

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 64 additions & 14 deletions apps/files/lib/Command/Scan.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,17 @@
use OC\Core\Command\InterruptedException;
use OC\DB\Connection;
use OC\DB\ConnectionAdapter;
use OC\Files\Filesystem;
use OC\Files\Node\File;
use OC\Files\Node\Folder;
use OC\Files\Node\NonExistingFile;
use OC\Files\Node\NonExistingFolder;
use OC\Files\View;
use OC\ForbiddenException;
use OC\Metadata\MetadataManager;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\FileInfo;
use OCP\Files\IRootFolder;
use OCP\Files\Mount\IMountPoint;
use OCP\Files\NotFoundException;
use OCP\Files\StorageNotAvailableException;
Expand All @@ -51,19 +60,18 @@
use Symfony\Component\Console\Output\OutputInterface;

class Scan extends Base {
private IUserManager $userManager;
protected float $execTime = 0;
protected int $foldersCounter = 0;
protected int $filesCounter = 0;
private IRootFolder $root;
private View $view;

/** @var IUserManager $userManager */
private $userManager;
/** @var float */
protected $execTime = 0;
/** @var int */
protected $foldersCounter = 0;
/** @var int */
protected $filesCounter = 0;

public function __construct(IUserManager $userManager) {
public function __construct(IUserManager $userManager, IRootFolder $rootFolder, View $view) {
$this->userManager = $userManager;
parent::__construct();
$this->root = $rootFolder;
$this->view = $view;
}

protected function configure() {
Expand All @@ -83,6 +91,13 @@ protected function configure() {
InputArgument::OPTIONAL,
'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored'
)
->addOption(
'metadata',
null,
InputOption::VALUE_NONE,
'Add missing file metadata',
false
)
->addOption(
'all',
null,
Expand All @@ -106,21 +121,28 @@ protected function configure() {
);
}

protected function scanFiles($user, $path, OutputInterface $output, $backgroundScan = false, $recursive = true, $homeOnly = false) {
protected function scanFiles($user, $path, bool $scanMetadata, OutputInterface $output, $backgroundScan = false, $recursive = true, $homeOnly = false) {
$connection = $this->reconnectToDatabase($output);
$scanner = new \OC\Files\Utils\Scanner(
$user,
new ConnectionAdapter($connection),
\OC::$server->query(IEventDispatcher::class),
\OC::$server->get(IEventDispatcher::class),
\OC::$server->get(LoggerInterface::class)
);

# check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception

$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) {
/** @var MetadataManager $metadataManager */
$metadataManager = \OC::$server->get(MetadataManager::class);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should be injected instead of possible


$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output, $metadataManager, $scanMetadata) {
$output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
++$this->filesCounter;
$this->abortIfInterrupted();
if ($scanMetadata) {
$node = $this->getNodeForPath($path);
$metadataManager->generateMetadata($node, true);
}
});

$scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output) {
Expand Down Expand Up @@ -197,7 +219,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
++$user_count;
if ($this->userManager->userExists($user)) {
$output->writeln("Starting scan for user $user_count out of $users_total ($user)");
$this->scanFiles($user, $path, $output, $input->getOption('unscanned'), !$input->getOption('shallow'), $input->getOption('home-only'));
$this->scanFiles($user, $path, $input->getOption('metadata'), $output, $input->getOption('unscanned'), !$input->getOption('shallow'), $input->getOption('home-only'));
$output->writeln('', OutputInterface::VERBOSITY_VERBOSE);
} else {
$output->writeln("<error>Unknown user $user_count $user</error>");
Expand Down Expand Up @@ -312,4 +334,32 @@ protected function reconnectToDatabase(OutputInterface $output): Connection {
}
return $connection;
}

private function getNodeForPath($path) {
$pathParts = explode('/', $path);
// FIXME ugly hack to get it working for local file
array_shift($pathParts);
array_shift($pathParts);
array_shift($pathParts);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you can add an argument to explode to only do so for the first three or something and then keep the rest

did you check if using Filesystem::resolvePath would work here ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The issue is that these array_shift shouldn't even exists :( This should work:

 		$info = Filesystem::getFileInfo($path);

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

iirc $path is absolute here right?

Inject a IRootFolder into the command and use IRootFolder::get instead of using the old static filesystem apis

CarlSchwan marked this conversation as resolved.
Show resolved Hide resolved
$info = Filesystem::getFileInfo('/' . implode('/', $pathParts));
if (!$info) {
$fullPath = Filesystem::getView()->getAbsolutePath($path);
if (isset($this->deleteMetaCache[$fullPath])) {
$info = $this->deleteMetaCache[$fullPath];
} else {
$info = null;
}
if (Filesystem::is_dir($path)) {
return new NonExistingFolder($this->root, $this->view, $fullPath, $info);
} else {
return new NonExistingFile($this->root, $this->view, $fullPath, $info);
}
}
if ($info->getType() === FileInfo::TYPE_FILE) {
return new File($this->root, $this->view, $info->getPath(), $info);
} else {
return new Folder($this->root, $this->view, $info->getPath(), $info);
}
}

}