Mirrors
/
nextcloud-server
mirror of https://github.com/nextcloud/server
3
0
Fork 0
Code Releases Activity
Browse Source

add utility command for object store objects 

Signed-off-by: Robin Appelman <robin@icewind.nl>
pull/38226/head
Robin Appelman 3 years ago
parent
faf0e634db
commit
ea88ec1350
No known key found for this signature in database GPG Key ID: 42B69D8A64526EFB
8 changed files with 368 additions and 1 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      apps/files/appinfo/info.xml
  2. 4
      apps/files/composer/composer/autoload_classmap.php
  3. 4
      apps/files/composer/composer/autoload_static.php
  4. 78
      apps/files/lib/Command/Object/Delete.php
  5. 80
      apps/files/lib/Command/Object/Get.php
  6. 110
      apps/files/lib/Command/Object/ObjectUtil.php
  7. 84
      apps/files/lib/Command/Object/Put.php
  8. 6
      lib/private/Files/ObjectStore/S3ObjectTrait.php

3
apps/files/appinfo/info.xml
View File

@ -38,6 +38,9 @@
<command>OCA\Files\Command\Get</command>
<command>OCA\Files\Command\Put</command>
<command>OCA\Files\Command\Delete</command>
<command>OCA\Files\Command\Object\Delete</command>
<command>OCA\Files\Command\Object\Get</command>
<command>OCA\Files\Command\Object\Put</command>
</commands>
<activity>

4
apps/files/composer/composer/autoload_classmap.php
View File

@ -30,6 +30,10 @@ return array(
'OCA\\Files\\Command\\Delete' => $baseDir . '/../lib/Command/Delete.php',
'OCA\\Files\\Command\\DeleteOrphanedFiles' => $baseDir . '/../lib/Command/DeleteOrphanedFiles.php',
'OCA\\Files\\Command\\Get' => $baseDir . '/../lib/Command/Get.php',
'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php',
'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php',
'OCA\\Files\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php',

4
apps/files/composer/composer/autoload_static.php
View File

@ -45,6 +45,10 @@ class ComposerStaticInitFiles
'OCA\\Files\\Command\\Delete' => __DIR__ . '/..' . '/../lib/Command/Delete.php',
'OCA\\Files\\Command\\DeleteOrphanedFiles' => __DIR__ . '/..' . '/../lib/Command/DeleteOrphanedFiles.php',
'OCA\\Files\\Command\\Get' => __DIR__ . '/..' . '/../lib/Command/Get.php',
'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php',
'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php',
'OCA\\Files\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php',

78
apps/files/lib/Command/Object/Delete.php
View File

@ -0,0 +1,78 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Files\Command\Object;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\QuestionHelper;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Question\ConfirmationQuestion;
class Delete extends Command {
private ObjectUtil $objectUtils;
public function __construct(ObjectUtil $objectUtils) {
$this->objectUtils = $objectUtils;
parent::__construct();
}
protected function configure(): void {
$this
->setName('files:object:delete')
->setDescription('Delete an object from the object store')
->addArgument('object', InputArgument::REQUIRED, "Object to delete")
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to delete the object from, only required in cases where it can't be determined from the config");
}
public function execute(InputInterface $input, OutputInterface $output): int {
$object = $input->getArgument('object');
$objectStore = $this->objectUtils->getObjectStore($input->getOption("bucket"), $output);
if (!$objectStore) {
return -1;
}
if ($fileId = $this->objectUtils->objectExistsInDb($object)) {
$output->writeln("<error>Warning, object $object belongs to an existing file, deleting the object will lead to unexpected behavior if not replaced</error>");
$output->writeln(" Note: use <info>occ files:delete $fileId</info> to delete the file cleanly or <info>occ info:file $fileId</info> for more information about the file");
$output->writeln("");
}
if (!$objectStore->objectExists($object)) {
$output->writeln("<error>Object $object does not exist</error>");
return -1;
}
/** @var QuestionHelper $helper */
$helper = $this->getHelper('question');
$question = new ConfirmationQuestion("Delete $object? [y/N] ", false);
if ($helper->ask($input, $output, $question)) {
$objectStore->deleteObject($object);
}
return 0;
}
}

80
apps/files/lib/Command/Object/Get.php
View File

@ -0,0 +1,80 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Files\Command\Object;
use OCP\Files\File;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class Get extends Command {
private ObjectUtil $objectUtils;
public function __construct(ObjectUtil $objectUtils) {
$this->objectUtils = $objectUtils;
parent::__construct();
}
protected function configure(): void {
$this
->setName('files:object:get')
->setDescription('Get the contents of an object')
->addArgument('object', InputArgument::REQUIRED, "Object to get")
->addArgument('output', InputArgument::REQUIRED, "Target local file to output to, use - for STDOUT")
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config");
}
public function execute(InputInterface $input, OutputInterface $output): int {
$object = $input->getArgument('object');
$outputName = $input->getArgument('output');
$objectStore = $this->objectUtils->getObjectStore($input->getOption("bucket"), $output);
if (!$objectStore) {
return 1;
}
if (!$objectStore->objectExists($object)) {
$output->writeln("<error>Object $object does not exist</error>");
return 1;
} else {
try {
$source = $objectStore->readObject($object);
} catch (\Exception $e) {
$msg = $e->getMessage();
$output->writeln("<error>Failed to read $object from object store: $msg</error>");
return 1;
}
$target = $outputName === '-' ? STDOUT : fopen($outputName, 'w');
if (!$target) {
$output->writeln("<error>Failed to open $outputName for writing</error>");
return 1;
}
stream_copy_to_stream($source, $target);
return 0;
}
}
}

110
apps/files/lib/Command/Object/ObjectUtil.php
View File

@ -0,0 +1,110 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Files\Command\Object;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\ObjectStore\IObjectStore;
use OCP\IConfig;
use OCP\IDBConnection;
use Symfony\Component\Console\Output\OutputInterface;
class ObjectUtil {
private IConfig $config;
private IDBConnection $connection;
public function __construct(IConfig $config, IDBConnection $connection) {
$this->config = $config;
$this->connection = $connection;
}
private function getObjectStoreConfig(): ?array {
$config = $this->config->getSystemValue('objectstore_multibucket');
if (is_array($config)) {
$config['multibucket'] = true;
return $config;
}
$config = $this->config->getSystemValue('objectstore');
if (is_array($config)) {
if (!isset($config['multibucket'])) {
$config['multibucket'] = false;
}
return $config;
} else {
return null;
}
}
public function getObjectStore(?string $bucket, OutputInterface $output): ?IObjectStore {
$config = $this->getObjectStoreConfig();
if (!$config) {
$output->writeln("<error>Instance is not using primary object store</error>");
return null;
}
if ($config['multibucket'] && !$bucket) {
$output->writeln("<error>--bucket option required</error> because <info>multi bucket</info> is enabled.");
return null;
}
if (!isset($config['arguments'])) {
throw new \Exception("no arguments configured for object store configuration");
}
if (!isset($config['class'])) {
throw new \Exception("no class configured for object store configuration");
}
if ($bucket) {
// s3, swift
$config['arguments']['bucket'] = $bucket;
// azure
$config['arguments']['container'] = $bucket;
}
$store = new $config['class']($config['arguments']);
if (!$store instanceof IObjectStore) {
throw new \Exception("configured object store class is not an object store implementation");
}
return $store;
}
/**
* Check if an object is referenced in the database
*/
public function objectExistsInDb(string $object): int|false {
if (str_starts_with($object, 'urn:oid:')) {
$fileId = (int)substr($object, strlen('urn:oid:'));
$query = $this->connection->getQueryBuilder();
$query->select('fileid')
->from('filecache')
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)));
$result = $query->executeQuery();
if ($result->fetchOne() !== false) {
return $fileId;
} else {
return false;
}
} else {
return false;
}
}
}

84
apps/files/lib/Command/Object/Put.php
View File

@ -0,0 +1,84 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Files\Command\Object;
use OCP\Files\IMimeTypeDetector;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\QuestionHelper;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Question\ConfirmationQuestion;
class Put extends Command {
private ObjectUtil $objectUtils;
private IMimeTypeDetector $mimeTypeDetector;
public function __construct(ObjectUtil $objectUtils, IMimeTypeDetector $mimeTypeDetector) {
$this->objectUtils = $objectUtils;
$this->mimeTypeDetector = $mimeTypeDetector;
parent::__construct();
}
protected function configure(): void {
$this
->setName('files:object:put')
->setDescription('Write a file to the object store')
->addArgument('input', InputArgument::REQUIRED, "Source local path, use - to read from STDIN")
->addArgument('object', InputArgument::REQUIRED, "Object to write")
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket where to store the object, only required in cases where it can't be determined from the config");;
}
public function execute(InputInterface $input, OutputInterface $output): int {
$object = $input->getArgument('object');
$inputName = (string)$input->getArgument('input');
$objectStore = $this->objectUtils->getObjectStore($input->getOption("bucket"), $output);
if (!$objectStore) {
return -1;
}
if ($fileId = $this->objectUtils->objectExistsInDb($object)) {
$output->writeln("<error>Warning, object $object belongs to an existing file, overwriting the object contents can lead to unexpected behavior.</error>");
$output->writeln("You can use <info>occ files:put $inputName $fileId</info> to write to the file safely.");
$output->writeln("");
/** @var QuestionHelper $helper */
$helper = $this->getHelper('question');
$question = new ConfirmationQuestion("Write to the object anyway? [y/N] ", false);
if (!$helper->ask($input, $output, $question)) {
return -1;
}
}
$source = $inputName === '-' ? STDIN : fopen($inputName, 'r');
if (!$source) {
$output->writeln("<error>Failed to open $inputName</error>");
return 1;
}
$objectStore->writeObject($object, $source, $this->mimeTypeDetector->detectPath($inputName));
return 0;
}
}

6
lib/private/Files/ObjectStore/S3ObjectTrait.php
View File

@ -54,7 +54,7 @@ trait S3ObjectTrait {
* @since 7.0.0
*/
public function readObject($urn) {
return SeekableHttpStream::open(function ($range) use ($urn) {
$fh = SeekableHttpStream::open(function ($range) use ($urn) {
$command = $this->getConnection()->getCommand('GetObject', [
'Bucket' => $this->bucket,
'Key' => $urn,
@ -88,6 +88,10 @@ trait S3ObjectTrait {
$context = stream_context_create($opts);
return fopen($request->getUri(), 'r', false, $context);
});
if (!$fh) {
throw new \Exception("Failed to read object $urn");
}
return $fh;
}

Write Preview
Loading…
Cancel
Save