From cb0f918d2101bf7644d7d84f811b37e3e672f091 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 28 Jun 2023 14:59:20 +0200 Subject: [PATCH] Add tasks::last_updated column and vacate tasks after a week Signed-off-by: Marcel Klehr --- .../Version28000Date20230616104802.php | 7 +++ lib/private/LanguageModel/Db/Task.php | 10 +++- lib/private/LanguageModel/Db/TaskMapper.php | 18 ++++++ .../RemoveOldTasksBackgroundJob.php | 59 +++++++++++++++++++ lib/private/Repair.php | 2 + .../Repair/AddRemoveOldTasksBackgroundJob.php | 47 +++++++++++++++ lib/private/Setup.php | 2 + 7 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 lib/private/LanguageModel/RemoveOldTasksBackgroundJob.php create mode 100644 lib/private/Repair/AddRemoveOldTasksBackgroundJob.php diff --git a/core/Migrations/Version28000Date20230616104802.php b/core/Migrations/Version28000Date20230616104802.php index 76d8173861f41..ab7347608e200 100644 --- a/core/Migrations/Version28000Date20230616104802.php +++ b/core/Migrations/Version28000Date20230616104802.php @@ -78,9 +78,16 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt 'length' => 32, 'default' => '', ]); + $table->addColumn('last_updated', 'integer', [ + 'notnull' => false, + 'length' => 4, + 'default' => 0, + 'unsigned' => true, + ]); $table->setPrimaryKey(['id'], 'llm_tasks_id_index'); $table->addUniqueIndex(['status', 'type'], 'llm_tasks_status_type'); + $table->addIndex(['last_updated'], 'llm_tasks_updated'); } return $schema; diff --git a/lib/private/LanguageModel/Db/Task.php b/lib/private/LanguageModel/Db/Task.php index fe77d5595307a..895969e08bb91 100644 --- a/lib/private/LanguageModel/Db/Task.php +++ b/lib/private/LanguageModel/Db/Task.php @@ -8,6 +8,8 @@ /** * @method setType(string $type) * @method string getType() + * @method setLastUpdated(int $lastUpdated) + * @method int getLastUpdated() * @method setInput(string $type) * @method string getInput() * @method setStatus(int $type) @@ -18,6 +20,8 @@ * @method string getAppId() */ class Task extends Entity { + protected $lastUpdated; + protected $type; protected $input; protected $status; @@ -27,17 +31,18 @@ class Task extends Entity { /** * @var string[] */ - public static array $columns = ['id', 'type', 'input', 'output', 'status', 'user_id', 'app_id']; + public static array $columns = ['id', 'last_updated', 'type', 'input', 'output', 'status', 'user_id', 'app_id']; /** * @var string[] */ - public static array $fields = ['id', 'type', 'input', 'output', 'status', 'userId', 'appId']; + public static array $fields = ['id', 'lastUpdated', 'type', 'input', 'output', 'status', 'userId', 'appId']; public function __construct() { // add types in constructor $this->addType('id', 'integer'); + $this->addType('lastUpdated', 'integer'); $this->addType('type', 'string'); $this->addType('input', 'string'); $this->addType('status', 'integer'); @@ -48,6 +53,7 @@ public function __construct() { public static function fromLanguageModelTask(ILanguageModelTask $task): Task { return Task::fromParams([ 'type' => $task->getType(), + 'lastUpdated' => time(), 'status' => $task->getStatus(), 'input' => $task->getInput(), 'output' => $task->getOutput(), diff --git a/lib/private/LanguageModel/Db/TaskMapper.php b/lib/private/LanguageModel/Db/TaskMapper.php index d7122ea794165..e0b06a1b62b3f 100644 --- a/lib/private/LanguageModel/Db/TaskMapper.php +++ b/lib/private/LanguageModel/Db/TaskMapper.php @@ -3,6 +3,7 @@ namespace OC\LanguageModel\Db; use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\Entity; use OCP\AppFramework\Db\MultipleObjectsReturnedException; use OCP\AppFramework\Db\QBMapper; use OCP\DB\Exception; @@ -30,4 +31,21 @@ public function find(int $id): Task { ->where($qb->expr()->eq('id', $qb->createPositionalParameter($id))); return $this->findEntity($qb); } + + /** + * @param int $timeout + * @return int the number of deleted tasks + * @throws Exception + */ + public function deleteOlderThan(int $timeout): int { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->tableName) + ->where($qb->expr()->lt('last_updated', $qb->createPositionalParameter(time() - $timeout))); + return $qb->executeStatement(); + } + + public function update(Entity $entity): Entity { + $entity->setLastUpdated(time()); + return parent::update($entity); + } } diff --git a/lib/private/LanguageModel/RemoveOldTasksBackgroundJob.php b/lib/private/LanguageModel/RemoveOldTasksBackgroundJob.php new file mode 100644 index 0000000000000..fa3a716a2c608 --- /dev/null +++ b/lib/private/LanguageModel/RemoveOldTasksBackgroundJob.php @@ -0,0 +1,59 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +namespace OC\LanguageModel; + +use OC\LanguageModel\Db\TaskMapper; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\TimedJob; +use OCP\DB\Exception; +use Psr\Log\LoggerInterface; + +class RemoveOldTasksBackgroundJob extends TimedJob { + public const MAX_TASK_AGE_SECONDS = 60 * 50 * 24 * 7; // 1 week + + public function __construct( + ITimeFactory $timeFactory, + private TaskMapper $taskMapper, + private LoggerInterface $logger, + + ) { + parent::__construct($timeFactory); + $this->setInterval(60 * 60 * 24); + } + + /** + * @param mixed $argument + * @inheritDoc + */ + protected function run($argument) { + try { + $this->taskMapper->deleteOlderThan(self::MAX_TASK_AGE_SECONDS); + } catch (Exception $e) { + $this->logger->warning('Failed to delete stale language model tasks', ['exception' => $e]); + } + } +} diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 9c6a6cd00f270..05624a2423a22 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -34,6 +34,7 @@ */ namespace OC; +use OC\Repair\AddRemoveOldTasksBackgroundJob; use OC\Repair\CleanUpAbandonedApps; use OCP\AppFramework\QueryException; use OCP\AppFramework\Utility\ITimeFactory; @@ -210,6 +211,7 @@ public static function getRepairSteps(): array { \OCP\Server::get(AddTokenCleanupJob::class), \OCP\Server::get(CleanUpAbandonedApps::class), \OCP\Server::get(AddMissingSecretJob::class), + \OCP\Server::get(AddRemoveOldTasksBackgroundJob::class), ]; } diff --git a/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php b/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php new file mode 100644 index 0000000000000..713192b06f957 --- /dev/null +++ b/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php @@ -0,0 +1,47 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OC\Repair; + +use OC\LanguageModel\RemoveOldTasksBackgroundJob; +use OCP\BackgroundJob\IJobList; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; + +class AddRemoveOldTasksBackgroundJob implements IRepairStep { + private IJobList $jobList; + + public function __construct(IJobList $jobList) { + $this->jobList = $jobList; + } + + public function getName(): string { + return 'Add language model tasks cleanup job'; + } + + public function run(IOutput $output) { + $this->jobList->add(RemoveOldTasksBackgroundJob::class); + } +} diff --git a/lib/private/Setup.php b/lib/private/Setup.php index d847125cc7983..76bd5e6c615d1 100644 --- a/lib/private/Setup.php +++ b/lib/private/Setup.php @@ -53,6 +53,7 @@ use InvalidArgumentException; use OC\Authentication\Token\PublicKeyTokenProvider; use OC\Authentication\Token\TokenCleanupJob; +use OC\LanguageModel\RemoveOldTasksBackgroundJob; use OC\Log\Rotate; use OC\Preview\BackgroundCleanupJob; use OCP\AppFramework\Utility\ITimeFactory; @@ -453,6 +454,7 @@ public static function installBackgroundJobs() { $jobList->add(TokenCleanupJob::class); $jobList->add(Rotate::class); $jobList->add(BackgroundCleanupJob::class); + $jobList->add(RemoveOldTasksBackgroundJob::class); } /**