Browse Source

feat(TaskProcessing): Allow setting task results for file slots

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
pull/46368/head
Marcel Klehr 1 year ago
parent
commit
4ac7f8275b
  1. 51
      core/Controller/TaskProcessingApiController.php
  2. 129
      lib/private/TaskProcessing/Manager.php
  3. 3
      lib/private/TaskProcessing/SynchronousBackgroundJob.php
  4. 6
      lib/public/TaskProcessing/IManager.php

51
core/Controller/TaskProcessingApiController.php

@ -11,6 +11,7 @@ declare(strict_types=1);
namespace OC\Core\Controller;
use OC\Core\ResponseDefinitions;
use OC\Files\SimpleFS\SimpleFile;
use OCP\AppFramework\Http;
use OCP\AppFramework\Http\Attribute\AnonRateLimit;
use OCP\AppFramework\Http\Attribute\ApiRoute;
@ -22,6 +23,7 @@ use OCP\AppFramework\Http\DataDownloadResponse;
use OCP\AppFramework\Http\DataResponse;
use OCP\Files\File;
use OCP\Files\GenericFileException;
use OCP\Files\IAppData;
use OCP\Files\IRootFolder;
use OCP\Files\NotPermittedException;
use OCP\IL10N;
@ -50,6 +52,7 @@ class TaskProcessingApiController extends \OCP\AppFramework\OCSController {
private IL10N $l,
private ?string $userId,
private IRootFolder $rootFolder,
private IAppData $appData,
) {
parent::__construct($appName, $request);
}
@ -286,6 +289,39 @@ class TaskProcessingApiController extends \OCP\AppFramework\OCSController {
}
}
/**
* Upload a file so it can be referenced in a task result (ExApp route version)
*
* Use field 'file' for the file upload
*
* @param int $taskId The id of the task
* @return DataDownloadResponse<Http::STATUS_CREATED, array{fileId: int}, array{}>|DataResponse<Http::STATUS_INTERNAL_SERVER_ERROR|Http::STATUS_NOT_FOUND, array{message: string}, array{}>
*
* 201: File created
* 404: Task not found
*/
#[ExAppRequired]
#[ApiRoute(verb: 'POST', url: '/tasks_provider/{taskId}/file', root: '/taskprocessing')]
public function setFileContentsExApp(int $taskId): DataResponse {
try {
$task = $this->taskProcessingManager->getTask($taskId);
$file = $this->request->getUploadedFile('file');
if (!isset($file['tmp_name'])) {
return new DataResponse(['message' => $this->l->t('Bad request')], Http::STATUS_BAD_REQUEST);
}
$data = file_get_contents($file['tmp_name']);
if (!$data) {
return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR);
}
$fileId = $this->setFileContentsInternal($task, $data);
return new DataResponse(['fileId' => $fileId], Http::STATUS_CREATED);
} catch (NotFoundException) {
return new DataResponse(['message' => $this->l->t('Not found')], Http::STATUS_NOT_FOUND);
} catch (Exception) {
return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR);
}
}
/**
* @throws NotPermittedException
* @throws NotFoundException
@ -384,7 +420,7 @@ class TaskProcessingApiController extends \OCP\AppFramework\OCSController {
* Sets the task result
*
* @param int $taskId The id of the task
* @param array<string,mixed>|null $output The resulting task output
* @param array<string,mixed>|null $output The resulting task output, files are represented by their IDs
* @param string|null $errorMessage An error message if the task failed
* @return DataResponse<Http::STATUS_OK, array{task: CoreTaskProcessingTask}, array{}>|DataResponse<Http::STATUS_INTERNAL_SERVER_ERROR|Http::STATUS_NOT_FOUND, array{message: string}, array{}>
*
@ -396,7 +432,7 @@ class TaskProcessingApiController extends \OCP\AppFramework\OCSController {
public function setResult(int $taskId, ?array $output = null, ?string $errorMessage = null): DataResponse {
try {
// set result
$this->taskProcessingManager->setTaskResult($taskId, $errorMessage, $output);
$this->taskProcessingManager->setTaskResult($taskId, $errorMessage, $output, true);
$task = $this->taskProcessingManager->getTask($taskId);
/** @var CoreTaskProcessingTask $json */
@ -493,4 +529,15 @@ class TaskProcessingApiController extends \OCP\AppFramework\OCSController {
return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR);
}
}
private function setFileContentsInternal(Task $task, string $data) {
try {
$folder = $this->appData->getFolder('TaskProcessing');
} catch (\OCP\Files\NotFoundException) {
$folder = $this->appData->newFolder('TaskProcessing');
}
/** @var SimpleFile $file */
$file = $folder->newFile((string) rand(0, 10000000), $data);
return $file->getId();
}
}

129
lib/private/TaskProcessing/Manager.php

@ -18,10 +18,12 @@ use OCP\BackgroundJob\IJobList;
use OCP\DB\Exception;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\AppData\IAppDataFactory;
use OCP\Files\Config\IUserMountCache;
use OCP\Files\File;
use OCP\Files\GenericFileException;
use OCP\Files\IAppData;
use OCP\Files\IRootFolder;
use OCP\Files\Node;
use OCP\Files\NotPermittedException;
use OCP\Files\SimpleFS\ISimpleFile;
use OCP\IL10N;
@ -77,7 +79,7 @@ class Manager implements IManager {
private \OCP\TextProcessing\IManager $textProcessingManager,
private \OCP\TextToImage\IManager $textToImageManager,
private \OCP\SpeechToText\ISpeechToTextManager $speechToTextManager,
private \OCP\Share\IManager $shareManager,
private IUserMountCache $userMountCache,
) {
$this->appData = $appDataFactory->get('core');
}
@ -561,19 +563,8 @@ class Manager implements IManager {
}
}
foreach ($ids as $fileId) {
$node = $this->rootFolder->getFirstNodeById($fileId);
if ($node === null) {
$node = $this->rootFolder->getFirstNodeByIdInPath($fileId, '/' . $this->rootFolder->getAppDataDirectoryName() . '/');
if ($node === null) {
throw new ValidationException('Could not find file ' . $fileId);
}
}
/** @var array{users:array<string,array{node_id:int, node_path: string}>, remote: array<string,array{node_id:int, node_path: string}>, mail: array<string,array{node_id:int, node_path: string}>} $accessList */
$accessList = $this->shareManager->getAccessList($node, true, true);
$userIds = array_map(fn ($id) => strval($id), array_keys($accessList['users']));
if (!in_array($task->getUserId(), $userIds)) {
throw new UnauthorizedException('User ' . $task->getUserId() . ' does not have access to file ' . $fileId);
}
$this->validateFileId($fileId);
$this->validateUserAccessToFile($fileId, $task->getUserId());
}
// remove superfluous keys and set input
$task->setInput($this->removeSuperfluousArrayKeys($task->getInput(), $inputShape, $optionalInputShape));
@ -643,7 +634,7 @@ class Manager implements IManager {
return true;
}
public function setTaskResult(int $id, ?string $error, ?array $result): void {
public function setTaskResult(int $id, ?string $error, ?array $result, bool $isUsingFileIds = false): void {
// TODO: Not sure if we should rather catch the exceptions of getTask here and fail silently
$task = $this->getTask($id);
if ($task->getStatus() === Task::STATUS_CANCELLED) {
@ -664,7 +655,11 @@ class Manager implements IManager {
$this->validateOutput($optionalOutputShape, $result, true);
$output = $this->removeSuperfluousArrayKeys($result, $outputShape, $optionalOutputShape);
// extract raw data and put it in files, replace it with file ids
$output = $this->encapsulateOutputFileData($output, $outputShape, $optionalOutputShape);
if (!$isUsingFileIds) {
$output = $this->encapsulateOutputFileData($output, $outputShape, $optionalOutputShape);
} else {
$output = $this->validateOutputFileIds($output, $outputShape, $optionalOutputShape);
}
$task->setOutput($output);
$task->setProgress(1);
$task->setStatus(Task::STATUS_SUCCESSFUL);
@ -711,16 +706,13 @@ class Manager implements IManager {
}
/**
* Takes task input or output data and replaces fileIds with base64 data
* Takes task input data and replaces fileIds with File objects
*
* @param string|null $userId
* @param array<array-key, list<numeric|string>|numeric|string> $input
* @param ShapeDescriptor[] ...$specs the specs
* @return array<array-key, list<File|numeric|string>|numeric|string|File>
* @throws GenericFileException
* @throws LockedException
* @throws NotPermittedException
* @throws ValidationException
* @throws GenericFileException|LockedException|NotPermittedException|ValidationException|UnauthorizedException
*/
public function fillInputFileData(?string $userId, array $input, ...$specs): array {
if ($userId !== null) {
@ -738,30 +730,14 @@ class Manager implements IManager {
continue;
}
if ($type->value < 10) {
$node = $this->rootFolder->getFirstNodeById((int)$input[$key]);
if ($node === null) {
$node = $this->rootFolder->getFirstNodeByIdInPath((int)$input[$key], '/' . $this->rootFolder->getAppDataDirectoryName() . '/');
if (!$node instanceof File) {
throw new ValidationException('File id given for key "' . $key . '" is not a file');
}
} elseif (!$node instanceof File) {
throw new ValidationException('File id given for key "' . $key . '" is not a file');
}
// TODO: Validate if userId has access to this file
$node = $this->validateFileId((int)$input[$key]);
$this->validateUserAccessToFile($input[$key], $userId);
$newInputOutput[$key] = $node;
} else {
$newInputOutput[$key] = [];
foreach ($input[$key] as $item) {
$node = $this->rootFolder->getFirstNodeById((int)$item);
if ($node === null) {
$node = $this->rootFolder->getFirstNodeByIdInPath((int)$item, '/' . $this->rootFolder->getAppDataDirectoryName() . '/');
if (!$node instanceof File) {
throw new ValidationException('File id given for key "' . $key . '" is not a file');
}
} elseif (!$node instanceof File) {
throw new ValidationException('File id given for key "' . $key . '" is not a file');
}
// TODO: Validate if userId has access to this file
$node = $this->validateFileId((int)$item);
$this->validateUserAccessToFile($item, $userId);
$newInputOutput[$key][] = $node;
}
}
@ -851,7 +827,7 @@ class Manager implements IManager {
* @throws GenericFileException
* @throws LockedException
* @throws NotPermittedException
* @throws ValidationException
* @throws ValidationException|UnauthorizedException
*/
public function prepareInputData(Task $task): array {
$taskTypes = $this->getAvailableTaskTypes();
@ -884,4 +860,73 @@ class Manager implements IManager {
$taskEntity = \OC\TaskProcessing\Db\Task::fromPublicTask($task);
$this->taskMapper->update($taskEntity);
}
/**
* @param array $output
* @param ShapeDescriptor[] ...$specs the specs that define which keys to keep
* @return array
* @throws NotPermittedException
*/
private function validateOutputFileIds(array $output, ...$specs): array {
$newOutput = [];
$spec = array_reduce($specs, fn ($carry, $spec) => $carry + $spec, []);
foreach($spec as $key => $descriptor) {
$type = $descriptor->getShapeType();
if (!isset($output[$key])) {
continue;
}
if (!in_array(EShapeType::getScalarType($type), [EShapeType::Image, EShapeType::Audio, EShapeType::Video, EShapeType::File], true)) {
$newOutput[$key] = $output[$key];
continue;
}
if ($type->value < 10) {
// Is scalar file ID
$newOutput[$key] = $this->validateFileId($output[$key]);
} else {
// Is list of file IDs
$newOutput = [];
foreach ($output[$key] as $item) {
$newOutput[$key][] = $this->validateFileId($item);
}
}
}
return $newOutput;
}
/**
* @param mixed $id
* @return Node
* @throws ValidationException
*/
private function validateFileId(mixed $id): Node {
$node = $this->rootFolder->getFirstNodeById($id);
if ($node === null) {
$node = $this->rootFolder->getFirstNodeByIdInPath($id, '/' . $this->rootFolder->getAppDataDirectoryName() . '/');
if ($node === null) {
throw new ValidationException('Could not find file ' . $id);
} elseif (!$node instanceof File) {
throw new ValidationException('File with id "' . $id . '" is not a file');
}
} elseif (!$node instanceof File) {
throw new ValidationException('File with id "' . $id . '" is not a file');
}
return $node;
}
/**
* @param mixed $fileId
* @param string $userId
* @return void
* @throws UnauthorizedException
*/
private function validateUserAccessToFile(mixed $fileId, ?string $userId): void {
if ($userId === null) {
throw new UnauthorizedException('User does not have access to file ' . $fileId);
}
$mounts = $this->userMountCache->getMountsForFileId($fileId);
$userIds = array_map(fn ($mount) => $mount->getUser()->getUID(), $mounts);
if (!in_array($userId, $userIds)) {
throw new UnauthorizedException('User ' . $userId . ' does not have access to file ' . $fileId);
}
}
}

3
lib/private/TaskProcessing/SynchronousBackgroundJob.php

@ -15,6 +15,7 @@ use OCP\Lock\LockedException;
use OCP\TaskProcessing\Exception\Exception;
use OCP\TaskProcessing\Exception\NotFoundException;
use OCP\TaskProcessing\Exception\ProcessingException;
use OCP\TaskProcessing\Exception\UnauthorizedException;
use OCP\TaskProcessing\Exception\ValidationException;
use OCP\TaskProcessing\IManager;
use OCP\TaskProcessing\ISynchronousProvider;
@ -54,7 +55,7 @@ class SynchronousBackgroundJob extends QueuedJob {
try {
try {
$input = $this->taskProcessingManager->prepareInputData($task);
} catch (GenericFileException|NotPermittedException|LockedException|ValidationException $e) {
} catch (GenericFileException|NotPermittedException|LockedException|ValidationException|UnauthorizedException $e) {
$this->logger->warning('Failed to prepare input data for a TaskProcessing task with synchronous provider ' . $provider->getId(), ['exception' => $e]);
$this->taskProcessingManager->setTaskResult($task->getId(), $e->getMessage(), null);
// Schedule again

6
lib/public/TaskProcessing/IManager.php

@ -91,11 +91,12 @@ interface IManager {
* @param int $id The id of the task
* @param string|null $error
* @param array|null $result
* @param bool $isUsingFileIds
* @throws Exception If the query failed
* @throws NotFoundException If the task could not be found
* @since 30.0.0
*/
public function setTaskResult(int $id, ?string $error, ?array $result): void;
public function setTaskResult(int $id, ?string $error, ?array $result, bool $isUsingFileIds = false): void;
/**
* @param int $id
@ -152,7 +153,7 @@ interface IManager {
/**
* Prepare the task's input data, so it can be processed by the provider
* ie. this replaces file ids with base64 data
* ie. this replaces file ids with File objects
*
* @param Task $task
* @return array<array-key, list<numeric|string|File>|numeric|string|File>
@ -160,6 +161,7 @@ interface IManager {
* @throws GenericFileException
* @throws LockedException
* @throws ValidationException
* @throws UnauthorizedException
* @since 30.0.0
*/
public function prepareInputData(Task $task): array;

Loading…
Cancel
Save