Browse Source
Merge pull request #52786 from nextcloud/multi-object-store
Merge pull request #52786 from nextcloud/multi-object-store
allow configuring multiple object store backendspull/54058/head
Robin Appelman
3 months ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 616 additions and 19 deletions
-
2apps/files/appinfo/info.xml
-
2apps/files/composer/composer/autoload_classmap.php
-
2apps/files/composer/composer/autoload_static.php
-
108apps/files/lib/Command/Object/Multi/Rename.php
-
98apps/files/lib/Command/Object/Multi/Users.php
-
1lib/composer/composer/autoload_classmap.php
-
1lib/composer/composer/autoload_static.php
-
13lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php
-
121lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php
-
2tests/lib/Files/Mount/ObjectHomeMountProviderTest.php
-
285tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php
@ -0,0 +1,108 @@ |
|||
<?php |
|||
|
|||
declare(strict_types=1); |
|||
/** |
|||
* SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl> |
|||
* SPDX-License-Identifier: AGPL-3.0-or-later |
|||
*/ |
|||
|
|||
namespace OCA\Files\Command\Object\Multi; |
|||
|
|||
use OC\Core\Command\Base; |
|||
use OC\Files\ObjectStore\PrimaryObjectStoreConfig; |
|||
use OCP\IConfig; |
|||
use OCP\IDBConnection; |
|||
use Symfony\Component\Console\Helper\QuestionHelper; |
|||
use Symfony\Component\Console\Input\InputArgument; |
|||
use Symfony\Component\Console\Input\InputInterface; |
|||
use Symfony\Component\Console\Output\OutputInterface; |
|||
use Symfony\Component\Console\Question\ConfirmationQuestion; |
|||
|
|||
class Rename extends Base { |
|||
public function __construct( |
|||
private readonly IDBConnection $connection, |
|||
private readonly PrimaryObjectStoreConfig $objectStoreConfig, |
|||
private readonly IConfig $config, |
|||
) { |
|||
parent::__construct(); |
|||
} |
|||
|
|||
protected function configure(): void { |
|||
parent::configure(); |
|||
$this |
|||
->setName('files:object:multi:rename-config') |
|||
->setDescription('Rename an object store configuration and move all users over to the new configuration,') |
|||
->addArgument('source', InputArgument::REQUIRED, 'Object store configuration to rename') |
|||
->addArgument('target', InputArgument::REQUIRED, 'New name for the object store configuration'); |
|||
} |
|||
|
|||
public function execute(InputInterface $input, OutputInterface $output): int { |
|||
$source = $input->getArgument('source'); |
|||
$target = $input->getArgument('target'); |
|||
|
|||
$configs = $this->objectStoreConfig->getObjectStoreConfigs(); |
|||
if (!isset($configs[$source])) { |
|||
$output->writeln('<error>Unknown object store configuration: ' . $source . '</error>'); |
|||
return 1; |
|||
} |
|||
|
|||
if ($source === 'root') { |
|||
$output->writeln('<error>Renaming the root configuration is not supported.</error>'); |
|||
return 1; |
|||
} |
|||
|
|||
if ($source === 'default') { |
|||
$output->writeln('<error>Renaming the default configuration is not supported.</error>'); |
|||
return 1; |
|||
} |
|||
|
|||
if (!isset($configs[$target])) { |
|||
$output->writeln('<comment>Target object store configuration ' . $target . ' doesn\'t exist yet.</comment>'); |
|||
$output->writeln('The target configuration can be created automatically.'); |
|||
$output->writeln('However, as this depends on modifying the config.php, this only works as long as the instance runs on a single node or all nodes in a clustered setup have a shared config file (such as from a shared network mount).'); |
|||
$output->writeln('If the different nodes have a separate copy of the config.php file, the automatic object store configuration creation will lead to the configuration going out of sync.'); |
|||
$output->writeln('If these requirements are not met, you can manually create the target object store configuration in each node\'s configuration before running the command.'); |
|||
$output->writeln(''); |
|||
$output->writeln('<error>Failure to check these requirements will lead to data loss for users.</error>'); |
|||
|
|||
/** @var QuestionHelper $helper */ |
|||
$helper = $this->getHelper('question'); |
|||
$question = new ConfirmationQuestion('Automatically create target object store configuration? [y/N] ', false); |
|||
if ($helper->ask($input, $output, $question)) { |
|||
$configs[$target] = $configs[$source]; |
|||
|
|||
// update all aliases
|
|||
foreach ($configs as &$config) { |
|||
if ($config === $source) { |
|||
$config = $target; |
|||
} |
|||
} |
|||
$this->config->setSystemValue('objectstore', $configs); |
|||
} else { |
|||
return 0; |
|||
} |
|||
} elseif (($configs[$source] !== $configs[$target]) || $configs[$source] !== $target) { |
|||
$output->writeln('<error>Source and target configuration differ.</error>'); |
|||
$output->writeln(''); |
|||
$output->writeln('To ensure proper migration of users, the source and target configuration must be the same to ensure that the objects for the moved users exist on the target configuration.'); |
|||
$output->writeln('The usual migration process consists of creating a clone of the old configuration, moving the users from the old configuration to the new one, and then adjust the old configuration that is longer used.'); |
|||
return 1; |
|||
} |
|||
|
|||
$query = $this->connection->getQueryBuilder(); |
|||
$query->update('preferences') |
|||
->set('configvalue', $query->createNamedParameter($target)) |
|||
->where($query->expr()->eq('appid', $query->createNamedParameter('homeobjectstore'))) |
|||
->andWhere($query->expr()->eq('configkey', $query->createNamedParameter('objectstore'))) |
|||
->andWhere($query->expr()->eq('configvalue', $query->createNamedParameter($source))); |
|||
$count = $query->executeStatement(); |
|||
|
|||
if ($count > 0) { |
|||
$output->writeln('Moved <info>' . $count . '</info> users'); |
|||
} else { |
|||
$output->writeln('No users moved'); |
|||
} |
|||
|
|||
return 0; |
|||
} |
|||
} |
|||
@ -0,0 +1,98 @@ |
|||
<?php |
|||
|
|||
declare(strict_types=1); |
|||
/** |
|||
* SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl> |
|||
* SPDX-License-Identifier: AGPL-3.0-or-later |
|||
*/ |
|||
|
|||
namespace OCA\Files\Command\Object\Multi; |
|||
|
|||
use OC\Core\Command\Base; |
|||
use OC\Files\ObjectStore\PrimaryObjectStoreConfig; |
|||
use OCP\IConfig; |
|||
use OCP\IUser; |
|||
use OCP\IUserManager; |
|||
use Symfony\Component\Console\Input\InputInterface; |
|||
use Symfony\Component\Console\Input\InputOption; |
|||
use Symfony\Component\Console\Output\OutputInterface; |
|||
|
|||
class Users extends Base { |
|||
public function __construct( |
|||
private readonly IUserManager $userManager, |
|||
private readonly PrimaryObjectStoreConfig $objectStoreConfig, |
|||
private readonly IConfig $config, |
|||
) { |
|||
parent::__construct(); |
|||
} |
|||
|
|||
protected function configure(): void { |
|||
parent::configure(); |
|||
$this |
|||
->setName('files:object:multi:users') |
|||
->setDescription('Get the mapping between users and object store buckets') |
|||
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, 'Only list users using the specified bucket') |
|||
->addOption('object-store', 'o', InputOption::VALUE_REQUIRED, 'Only list users using the specified object store configuration') |
|||
->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Only show the mapping for the specified user, ignores all other options'); |
|||
} |
|||
|
|||
public function execute(InputInterface $input, OutputInterface $output): int { |
|||
if ($userId = $input->getOption('user')) { |
|||
$user = $this->userManager->get($userId); |
|||
if (!$user) { |
|||
$output->writeln("<error>User $userId not found</error>"); |
|||
return 1; |
|||
} |
|||
$users = new \ArrayIterator([$user]); |
|||
} else { |
|||
$bucket = (string)$input->getOption('bucket'); |
|||
$objectStore = (string)$input->getOption('object-store'); |
|||
if ($bucket !== '' && $objectStore === '') { |
|||
$users = $this->getUsers($this->config->getUsersForUserValue('homeobjectstore', 'bucket', $bucket)); |
|||
} elseif ($bucket === '' && $objectStore !== '') { |
|||
$users = $this->getUsers($this->config->getUsersForUserValue('homeobjectstore', 'objectstore', $objectStore)); |
|||
} elseif ($bucket) { |
|||
$users = $this->getUsers(array_intersect( |
|||
$this->config->getUsersForUserValue('homeobjectstore', 'bucket', $bucket), |
|||
$this->config->getUsersForUserValue('homeobjectstore', 'objectstore', $objectStore) |
|||
)); |
|||
} else { |
|||
$users = $this->userManager->getSeenUsers(); |
|||
} |
|||
} |
|||
|
|||
$this->writeStreamingTableInOutputFormat($input, $output, $this->infoForUsers($users), 100); |
|||
return 0; |
|||
} |
|||
|
|||
/** |
|||
* @param string[] $userIds |
|||
* @return \Iterator<IUser> |
|||
*/ |
|||
private function getUsers(array $userIds): \Iterator { |
|||
foreach ($userIds as $userId) { |
|||
$user = $this->userManager->get($userId); |
|||
if ($user) { |
|||
yield $user; |
|||
} |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* @param \Iterator<IUser> $users |
|||
* @return \Iterator<array> |
|||
*/ |
|||
private function infoForUsers(\Iterator $users): \Iterator { |
|||
foreach ($users as $user) { |
|||
yield $this->infoForUser($user); |
|||
} |
|||
} |
|||
|
|||
private function infoForUser(IUser $user): array { |
|||
return [ |
|||
'user' => $user->getUID(), |
|||
'object-store' => $this->objectStoreConfig->getObjectStoreForUser($user), |
|||
'bucket' => $this->objectStoreConfig->getSetBucketForUser($user) ?? 'unset', |
|||
]; |
|||
} |
|||
} |
|||
@ -0,0 +1,13 @@ |
|||
<?php |
|||
|
|||
declare(strict_types=1); |
|||
/** |
|||
* SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl> |
|||
* SPDX-License-Identifier: AGPL-3.0-or-later |
|||
*/ |
|||
|
|||
namespace OC\Files\ObjectStore; |
|||
|
|||
class InvalidObjectStoreConfigurationException extends \Exception { |
|||
|
|||
} |
|||
@ -0,0 +1,285 @@ |
|||
<?php |
|||
|
|||
declare(strict_types=1); |
|||
/** |
|||
* SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl> |
|||
* SPDX-License-Identifier: AGPL-3.0-or-later |
|||
*/ |
|||
|
|||
namespace lib\Files\ObjectStore; |
|||
|
|||
use OC\Files\ObjectStore\PrimaryObjectStoreConfig; |
|||
use OC\Files\ObjectStore\StorageObjectStore; |
|||
use OCP\App\IAppManager; |
|||
use OCP\IConfig; |
|||
use OCP\IUser; |
|||
use PHPUnit\Framework\MockObject\MockObject; |
|||
use Test\TestCase; |
|||
|
|||
class PrimaryObjectStoreConfigTest extends TestCase { |
|||
private array $systemConfig = []; |
|||
private array $userConfig = []; |
|||
private IConfig&MockObject $config; |
|||
private IAppManager&MockObject $appManager; |
|||
private PrimaryObjectStoreConfig $objectStoreConfig; |
|||
|
|||
protected function setUp(): void { |
|||
parent::setUp(); |
|||
|
|||
$this->systemConfig = []; |
|||
$this->config = $this->createMock(IConfig::class); |
|||
$this->appManager = $this->createMock(IAppManager::class); |
|||
$this->config->method('getSystemValue') |
|||
->willReturnCallback(function ($key, $default = '') { |
|||
if (isset($this->systemConfig[$key])) { |
|||
return $this->systemConfig[$key]; |
|||
} else { |
|||
return $default; |
|||
} |
|||
}); |
|||
$this->config->method('getUserValue') |
|||
->willReturnCallback(function ($userId, $appName, $key, $default = '') { |
|||
if (isset($this->userConfig[$userId][$appName][$key])) { |
|||
return $this->userConfig[$userId][$appName][$key]; |
|||
} else { |
|||
return $default; |
|||
} |
|||
}); |
|||
$this->config->method('setUserValue') |
|||
->willReturnCallback(function ($userId, $appName, $key, $value) { |
|||
$this->userConfig[$userId][$appName][$key] = $value; |
|||
}); |
|||
|
|||
$this->objectStoreConfig = new PrimaryObjectStoreConfig($this->config, $this->appManager); |
|||
} |
|||
|
|||
private function getUser(string $uid): IUser { |
|||
$user = $this->createMock(IUser::class); |
|||
$user->method('getUID') |
|||
->willReturn($uid); |
|||
return $user; |
|||
} |
|||
|
|||
private function setConfig(string $key, $value) { |
|||
$this->systemConfig[$key] = $value; |
|||
} |
|||
|
|||
public function testNewUserGetsDefault() { |
|||
$this->setConfig('objectstore', [ |
|||
'default' => 'server1', |
|||
'server1' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
], |
|||
], |
|||
]); |
|||
|
|||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test')); |
|||
$this->assertEquals('server1', $result['arguments']['host']); |
|||
|
|||
$this->assertEquals('server1', $this->config->getUserValue('test', 'homeobjectstore', 'objectstore', null)); |
|||
} |
|||
|
|||
public function testExistingUserKeepsStorage() { |
|||
// setup user with `server1` as storage
|
|||
$this->testNewUserGetsDefault(); |
|||
|
|||
$this->setConfig('objectstore', [ |
|||
'default' => 'server2', |
|||
'server1' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
], |
|||
], |
|||
'server2' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server2', |
|||
], |
|||
], |
|||
]); |
|||
|
|||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test')); |
|||
$this->assertEquals('server1', $result['arguments']['host']); |
|||
|
|||
$this->assertEquals('server1', $this->config->getUserValue('test', 'homeobjectstore', 'objectstore', null)); |
|||
|
|||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('other-user')); |
|||
$this->assertEquals('server2', $result['arguments']['host']); |
|||
} |
|||
|
|||
public function testNestedAliases() { |
|||
$this->setConfig('objectstore', [ |
|||
'default' => 'a1', |
|||
'a1' => 'a2', |
|||
'a2' => 'server1', |
|||
'server1' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
], |
|||
], |
|||
]); |
|||
$this->assertEquals('server1', $this->objectStoreConfig->resolveAlias('default')); |
|||
} |
|||
|
|||
public function testMultibucketChangedConfig() { |
|||
$this->setConfig('objectstore', [ |
|||
'default' => 'server1', |
|||
'server1' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
'multibucket' => true, |
|||
'num_buckets' => 8, |
|||
'bucket' => 'bucket-' |
|||
], |
|||
], |
|||
]); |
|||
|
|||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test')); |
|||
$this->assertEquals('server1', $result['arguments']['host']); |
|||
$this->assertEquals('bucket-7', $result['arguments']['bucket']); |
|||
|
|||
$this->setConfig('objectstore', [ |
|||
'default' => 'server1', |
|||
'server1' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
'multibucket' => true, |
|||
'num_buckets' => 64, |
|||
'bucket' => 'bucket-' |
|||
], |
|||
], |
|||
]); |
|||
|
|||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test')); |
|||
$this->assertEquals('server1', $result['arguments']['host']); |
|||
$this->assertEquals('bucket-7', $result['arguments']['bucket']); |
|||
|
|||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test-foo')); |
|||
$this->assertEquals('server1', $result['arguments']['host']); |
|||
$this->assertEquals('bucket-40', $result['arguments']['bucket']); |
|||
|
|||
$this->setConfig('objectstore', [ |
|||
'default' => 'server2', |
|||
'server1' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
'multibucket' => true, |
|||
'num_buckets' => 64, |
|||
'bucket' => 'bucket-' |
|||
], |
|||
], |
|||
'server2' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server2', |
|||
'multibucket' => true, |
|||
'num_buckets' => 16, |
|||
'bucket' => 'bucket-' |
|||
], |
|||
], |
|||
]); |
|||
|
|||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test')); |
|||
$this->assertEquals('server1', $result['arguments']['host']); |
|||
$this->assertEquals('bucket-7', $result['arguments']['bucket']); |
|||
|
|||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test-bar')); |
|||
$this->assertEquals('server2', $result['arguments']['host']); |
|||
$this->assertEquals('bucket-4', $result['arguments']['bucket']); |
|||
} |
|||
|
|||
public function testMultibucketOldConfig() { |
|||
$this->setConfig('objectstore_multibucket', [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
'multibucket' => true, |
|||
'num_buckets' => 8, |
|||
'bucket' => 'bucket-' |
|||
], |
|||
]); |
|||
$configs = $this->objectStoreConfig->getObjectStoreConfigs(); |
|||
$this->assertEquals([ |
|||
'default' => 'server1', |
|||
'root' => 'server1', |
|||
'server1' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
'multibucket' => true, |
|||
'num_buckets' => 8, |
|||
'bucket' => 'bucket-' |
|||
], |
|||
], |
|||
], $configs); |
|||
} |
|||
|
|||
public function testSingleObjectStore() { |
|||
$this->setConfig('objectstore', [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
], |
|||
]); |
|||
$configs = $this->objectStoreConfig->getObjectStoreConfigs(); |
|||
$this->assertEquals([ |
|||
'default' => 'server1', |
|||
'root' => 'server1', |
|||
'server1' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
'multibucket' => false, |
|||
], |
|||
], |
|||
], $configs); |
|||
} |
|||
|
|||
public function testRoot() { |
|||
$this->setConfig('objectstore', [ |
|||
'default' => 'server1', |
|||
'server1' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
], |
|||
], |
|||
'server2' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server2', |
|||
], |
|||
], |
|||
]); |
|||
|
|||
$result = $this->objectStoreConfig->getObjectStoreConfigForRoot(); |
|||
$this->assertEquals('server1', $result['arguments']['host']); |
|||
|
|||
$this->setConfig('objectstore', [ |
|||
'default' => 'server1', |
|||
'root' => 'server2', |
|||
'server1' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server1', |
|||
], |
|||
], |
|||
'server2' => [ |
|||
'class' => StorageObjectStore::class, |
|||
'arguments' => [ |
|||
'host' => 'server2', |
|||
], |
|||
], |
|||
]); |
|||
|
|||
$result = $this->objectStoreConfig->getObjectStoreConfigForRoot(); |
|||
$this->assertEquals('server2', $result['arguments']['host']); |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue