Mirrors
/
nextcloud-server
mirror of https://github.com/nextcloud/server
3
0
Fork 0
Code Releases Activity
Browse Source

Merge pull request #52867 from nextcloud/backport/51603/stable30 

[stable30] Add command to list orphan objects
pull/52924/head
Robin Appelman 7 months ago
committed by GitHub
parent
921c98e4df 86206e9bac
commit
09e1ecd15b
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
12 changed files with 367 additions and 3 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      apps/files/appinfo/info.xml
  2. 3
      apps/files/composer/composer/autoload_classmap.php
  3. 3
      apps/files/composer/composer/autoload_static.php
  4. 80
      apps/files/lib/Command/Object/Info.php
  5. 50
      apps/files/lib/Command/Object/ListObject.php
  6. 21
      apps/files/lib/Command/Object/ObjectUtil.php
  7. 79
      apps/files/lib/Command/Object/Orphans.php
  8. 52
      core/Command/Base.php
  9. 1
      lib/composer/composer/autoload_classmap.php
  10. 1
      lib/composer/composer/autoload_static.php
  11. 36
      lib/private/Files/ObjectStore/IObjectStoreMetaData.php
  12. 41
      lib/private/Files/ObjectStore/S3.php

3
apps/files/appinfo/info.xml
View File

@ -49,6 +49,9 @@
<command>OCA\Files\Command\Object\Delete</command>
<command>OCA\Files\Command\Object\Get</command>
<command>OCA\Files\Command\Object\Put</command>
<command>OCA\Files\Command\Object\Info</command>
<command>OCA\Files\Command\Object\ListObject</command>
<command>OCA\Files\Command\Object\Orphans</command>
</commands>
<settings>

3
apps/files/composer/composer/autoload_classmap.php
View File

@ -34,7 +34,10 @@ return array(
'OCA\\Files\\Command\\Move' => $baseDir . '/../lib/Command/Move.php',
'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php',
'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php',
'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php',
'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php',
'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php',

3
apps/files/composer/composer/autoload_static.php
View File

@ -49,7 +49,10 @@ class ComposerStaticInitFiles
'OCA\\Files\\Command\\Move' => __DIR__ . '/..' . '/../lib/Command/Move.php',
'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php',
'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php',
'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php',
'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php',
'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php',

80
apps/files/lib/Command/Object/Info.php
View File

@ -0,0 +1,80 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OCA\Files\Command\Object;
use OC\Core\Command\Base;
use OC\Files\ObjectStore\IObjectStoreMetaData;
use OCP\Files\IMimeTypeDetector;
use OCP\Util;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class Info extends Base {
public function __construct(
private ObjectUtil $objectUtils,
private IMimeTypeDetector $mimeTypeDetector,
) {
parent::__construct();
}
protected function configure(): void {
parent::configure();
$this
->setName('files:object:info')
->setDescription('Get the metadata of an object')
->addArgument('object', InputArgument::REQUIRED, 'Object to get')
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config");
}
public function execute(InputInterface $input, OutputInterface $output): int {
$object = $input->getArgument('object');
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
if (!$objectStore) {
return self::FAILURE;
}
if (!$objectStore instanceof IObjectStoreMetaData) {
$output->writeln('<error>Configured object store does currently not support retrieve metadata</error>');
return self::FAILURE;
}
if (!$objectStore->objectExists($object)) {
$output->writeln("<error>Object $object does not exist</error>");
return self::FAILURE;
}
try {
$meta = $objectStore->getObjectMetaData($object);
} catch (\Exception $e) {
$msg = $e->getMessage();
$output->writeln("<error>Failed to read $object from object store: $msg</error>");
return self::FAILURE;
}
if ($input->getOption('output') === 'plain' && isset($meta['size'])) {
$meta['size'] = Util::humanFileSize($meta['size']);
}
if (isset($meta['mtime'])) {
$meta['mtime'] = $meta['mtime']->format(\DateTimeImmutable::ATOM);
}
if (!isset($meta['mimetype'])) {
$handle = $objectStore->readObject($object);
$head = fread($handle, 8192);
fclose($handle);
$meta['mimetype'] = $this->mimeTypeDetector->detectString($head);
}
$this->writeArrayInOutputFormat($input, $output, $meta);
return self::SUCCESS;
}
}

50
apps/files/lib/Command/Object/ListObject.php
View File

@ -0,0 +1,50 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OCA\Files\Command\Object;
use OC\Core\Command\Base;
use OC\Files\ObjectStore\IObjectStoreMetaData;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class ListObject extends Base {
private const CHUNK_SIZE = 100;
public function __construct(
private readonly ObjectUtil $objectUtils,
) {
parent::__construct();
}
protected function configure(): void {
parent::configure();
$this
->setName('files:object:list')
->setDescription('List all objects in the object store')
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
}
public function execute(InputInterface $input, OutputInterface $output): int {
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
if (!$objectStore) {
return self::FAILURE;
}
if (!$objectStore instanceof IObjectStoreMetaData) {
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
return self::FAILURE;
}
$objects = $objectStore->listObjects();
$objects = $this->objectUtils->formatObjects($objects, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN);
$this->writeStreamingTableInOutputFormat($input, $output, $objects, self::CHUNK_SIZE);
return self::SUCCESS;
}
}

21
apps/files/lib/Command/Object/ObjectUtil.php
View File

@ -12,6 +12,7 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\ObjectStore\IObjectStore;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\Util;
use Symfony\Component\Console\Output\OutputInterface;
class ObjectUtil {
@ -91,4 +92,24 @@ class ObjectUtil {
return $fileId;
}
public function formatObjects(\Iterator $objects, bool $humanOutput): \Iterator {
foreach ($objects as $object) {
yield $this->formatObject($object, $humanOutput);
}
}
public function formatObject(array $object, bool $humanOutput): array {
$row = array_merge([
'urn' => $object['urn'],
], ($object['metadata'] ?? []));
if ($humanOutput && isset($row['size'])) {
$row['size'] = Util::humanFileSize($row['size']);
}
if (isset($row['mtime'])) {
$row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM);
}
return $row;
}
}

79
apps/files/lib/Command/Object/Orphans.php
View File

@ -0,0 +1,79 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OCA\Files\Command\Object;
use OC\Core\Command\Base;
use OC\Files\ObjectStore\IObjectStoreMetaData;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class Orphans extends Base {
private const CHUNK_SIZE = 100;
private ?IQueryBuilder $query = null;
public function __construct(
private readonly ObjectUtil $objectUtils,
private readonly IDBConnection $connection,
) {
parent::__construct();
}
private function getQuery(): IQueryBuilder {
if (!$this->query) {
$this->query = $this->connection->getQueryBuilder();
$this->query->select('fileid')
->from('filecache')
->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id')));
}
return $this->query;
}
protected function configure(): void {
parent::configure();
$this
->setName('files:object:orphans')
->setDescription('List all objects in the object store that don\'t have a matching entry in the database')
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
}
public function execute(InputInterface $input, OutputInterface $output): int {
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
if (!$objectStore) {
return self::FAILURE;
}
if (!$objectStore instanceof IObjectStoreMetaData) {
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
return self::FAILURE;
}
$prefixLength = strlen('urn:oid:');
$objects = $objectStore->listObjects('urn:oid:');
$orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) {
$fileId = (int) substr($object['urn'], $prefixLength);
return !$this->fileIdInDb($fileId);
});
$orphans = $this->objectUtils->formatObjects($orphans, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN);
$this->writeStreamingTableInOutputFormat($input, $output, $orphans, self::CHUNK_SIZE);
return self::SUCCESS;
}
private function fileIdInDb(int $fileId): bool {
$query = $this->getQuery();
$query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT);
$result = $query->executeQuery();
return $result->fetchOne() !== false;
}
}

52
core/Command/Base.php
View File

@ -88,6 +88,58 @@ class Base extends Command implements CompletionAwareInterface {
}
}
protected function writeStreamingTableInOutputFormat(InputInterface $input, OutputInterface $output, \Iterator $items, int $tableGroupSize): void {
switch ($input->getOption('output')) {
case self::OUTPUT_FORMAT_JSON:
case self::OUTPUT_FORMAT_JSON_PRETTY:
$this->writeStreamingJsonArray($input, $output, $items);
break;
default:
foreach ($this->chunkIterator($items, $tableGroupSize) as $chunk) {
$this->writeTableInOutputFormat($input, $output, $chunk);
}
break;
}
}
protected function writeStreamingJsonArray(InputInterface $input, OutputInterface $output, \Iterator $items): void {
$first = true;
$outputType = $input->getOption('output');
$output->writeln('[');
foreach ($items as $item) {
if (!$first) {
$output->writeln(',');
}
if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) {
$output->write(json_encode($item, JSON_PRETTY_PRINT));
} else {
$output->write(json_encode($item));
}
$first = false;
}
$output->writeln("\n]");
}
public function chunkIterator(\Iterator $iterator, int $count): \Iterator {
$chunk = [];
for ($i = 0; $iterator->valid(); $i++) {
$chunk[] = $iterator->current();
$iterator->next();
if (count($chunk) == $count) {
// Got a full chunk, yield and start a new one
yield $chunk;
$chunk = [];
}
}
if (count($chunk)) {
// Yield the last chunk even if incomplete
yield $chunk;
}
}
/**
* @param mixed $item

1
lib/composer/composer/autoload_classmap.php
View File

@ -1551,6 +1551,7 @@ return array(
'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Azure' => $baseDir . '/lib/private/Files/ObjectStore/Azure.php',
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\IObjectStoreMetaData' => $baseDir . '/lib/private/Files/ObjectStore/IObjectStoreMetaData.php',
'OC\\Files\\ObjectStore\\Mapper' => $baseDir . '/lib/private/Files/ObjectStore/Mapper.php',
'OC\\Files\\ObjectStore\\ObjectStoreScanner' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
'OC\\Files\\ObjectStore\\ObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',

1
lib/composer/composer/autoload_static.php
View File

@ -1584,6 +1584,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Azure' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Azure.php',
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\IObjectStoreMetaData' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/IObjectStoreMetaData.php',
'OC\\Files\\ObjectStore\\Mapper' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Mapper.php',
'OC\\Files\\ObjectStore\\ObjectStoreScanner' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
'OC\\Files\\ObjectStore\\ObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',

36
lib/private/Files/ObjectStore/IObjectStoreMetaData.php
View File

@ -0,0 +1,36 @@
<?php
/**
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
namespace OC\Files\ObjectStore;
/**
* Interface IObjectStoreMetaData
*
* @psalm-type ObjectMetaData = array{mtime?: \DateTime, etag?: string, size?: int, mimetype?: string, filename?: string}
*/
interface IObjectStoreMetaData {
/**
* Get metadata for an object.
*
* @param string $urn
* @return ObjectMetaData
*
* @since 32.0.0
*/
public function getObjectMetaData(string $urn): array;
/**
* List all objects in the object store.
*
* If the object store implementation can do it efficiently, the metadata for each object is also included.
*
* @param string $prefix
* @return \Iterator<array{urn: string, metadata: ?ObjectMetaData}>
*
* @since 32.0.0
*/
public function listObjects(string $prefix = ''): \Iterator;
}

41
lib/private/Files/ObjectStore/S3.php
View File

@ -4,6 +4,7 @@
* SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OC\Files\ObjectStore;
use Aws\Result;
@ -11,7 +12,7 @@ use Exception;
use OCP\Files\ObjectStore\IObjectStore;
use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
class S3 implements IObjectStore, IObjectStoreMultiPartUpload {
class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData {
use S3ConnectionTrait;
use S3ObjectTrait;
@ -62,7 +63,7 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload {
'Key' => $urn,
'UploadId' => $uploadId,
'MaxParts' => 1000,
'PartNumberMarker' => $partNumberMarker
'PartNumberMarker' => $partNumberMarker,
] + $this->getSSECParameters());
$parts = array_merge($parts, $result->get('Parts') ?? []);
$isTruncated = $result->get('IsTruncated');
@ -90,7 +91,41 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload {
$this->getConnection()->abortMultipartUpload([
'Bucket' => $this->bucket,
'Key' => $urn,
'UploadId' => $uploadId
'UploadId' => $uploadId,
]);
}
public function getObjectMetaData(string $urn): array {
$object = $this->getConnection()->headObject([
'Bucket' => $this->bucket,
'Key' => $urn
] + $this->getSSECParameters())->toArray();
return [
'mtime' => $object['LastModified'],
'etag' => trim($object['ETag'], '"'),
'size' => (int) ($object['Size'] ?? $object['ContentLength']),
];
}
public function listObjects(string $prefix = ''): \Iterator {
$results = $this->getConnection()->getPaginator('ListObjectsV2', [
'Bucket' => $this->bucket,
'Prefix' => $prefix,
] + $this->getSSECParameters());
foreach ($results as $result) {
if (is_array($result['Contents'])) {
foreach ($result['Contents'] as $object) {
yield [
'urn' => basename($object['Key']),
'metadata' => [
'mtime' => $object['LastModified'],
'etag' => trim($object['ETag'], '"'),
'size' => (int) ($object['Size'] ?? $object['ContentLength']),
],
];
}
}
}
}
}
Write Preview
Loading…
Cancel
Save