James Cole
4 years ago
No known key found for this signature in database
GPG Key ID: BDE6667570EADBD5
14 changed files with 617 additions and 92 deletions
-
1app/Http/Controllers/Import/ConfigurationController.php
-
153app/Http/Controllers/Import/ConversionController.php
-
68app/Services/CSV/Configuration/Configuration.php
-
126app/Services/CSV/Conversion/RoutineManager.php
-
2app/Services/Import/ImportJobStatus/ImportJobStatusManager.php
-
1app/Services/Import/ImportRoutineManager.php
-
4app/Services/Session/Constants.php
-
84app/Services/Shared/Conversion/ConversionStatus.php
-
47app/Services/Shared/Conversion/RoutineManagerInterface.php
-
205app/Services/Shared/Conversion/RoutineStatusManager.php
-
4config/filesystems.php
-
4resources/views/index.twig
-
8routes/web.php
-
2storage/conversion-routines/.gitignore
@ -0,0 +1,126 @@ |
|||
<?php |
|||
/* |
|||
* RoutineManager.php |
|||
* Copyright (c) 2021 james@firefly-iii.org |
|||
* |
|||
* This file is part of the Firefly III Data Importer |
|||
* (https://github.com/firefly-iii/data-importer). |
|||
* |
|||
* This program is free software: you can redistribute it and/or modify |
|||
* it under the terms of the GNU Affero General Public License as |
|||
* published by the Free Software Foundation, either version 3 of the |
|||
* License, or (at your option) any later version. |
|||
* |
|||
* This program is distributed in the hope that it will be useful, |
|||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|||
* GNU Affero General Public License for more details. |
|||
* |
|||
* You should have received a copy of the GNU Affero General Public License |
|||
* along with this program. If not, see <https://www.gnu.org/licenses/>. |
|||
*/ |
|||
|
|||
namespace App\Services\CSV\Conversion; |
|||
|
|||
use App\Services\CSV\Configuration\Configuration; |
|||
use App\Services\Import\Routine\CSVFileProcessor; |
|||
use App\Services\Shared\Conversion\RoutineManagerInterface; |
|||
use Log; |
|||
use Storage; |
|||
use Str; |
|||
|
|||
/** |
|||
* Class RoutineManager |
|||
*/ |
|||
class RoutineManager implements RoutineManagerInterface |
|||
{ |
|||
private const DISK_NAME = 'conversion-routines'; |
|||
|
|||
private Configuration $configuration; |
|||
private string $identifier; |
|||
private CSVFileProcessor $csvFileProcessor; |
|||
|
|||
/** |
|||
* |
|||
*/ |
|||
public function __construct(?string $identifier) |
|||
{ |
|||
if (null === $identifier) { |
|||
$this->generateIdentifier(); |
|||
} |
|||
if (null !== $identifier) { |
|||
$this->identifier = $identifier; |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* @inheritDoc |
|||
*/ |
|||
public function setConfiguration(Configuration $configuration): void |
|||
{ |
|||
// save config
|
|||
$this->configuration = $configuration; |
|||
|
|||
// share config
|
|||
$this->csvFileProcessor = new CSVFileProcessor($this->configuration); |
|||
|
|||
// set identifier:
|
|||
$this->csvFileProcessor->setIdentifier($this->identifier); |
|||
} |
|||
|
|||
/** |
|||
* @inheritDoc |
|||
*/ |
|||
public function start(): void |
|||
{ |
|||
Log::debug(sprintf('Now in %s', __METHOD__)); |
|||
|
|||
// convert CSV file into raw lines (arrays)
|
|||
$this->csvFileProcessor->setSpecifics($this->configuration->getSpecifics()); |
|||
$this->csvFileProcessor->setHasHeaders($this->configuration->isHeaders()); |
|||
$this->csvFileProcessor->setDelimiter($this->configuration->getDelimiter()); |
|||
$CSVLines = $this->csvFileProcessor->processCSVFile(); |
|||
|
|||
// convert raw lines into arrays with individual ColumnValues
|
|||
$valueArrays = $this->lineProcessor->processCSVLines($CSVLines); |
|||
|
|||
// convert value arrays into (pseudo) transactions.
|
|||
$pseudo = $this->columnValueConverter->processValueArrays($valueArrays); |
|||
|
|||
// convert pseudo transactions into actual transactions.
|
|||
$transactions = $this->pseudoTransactionProcessor->processPseudo($pseudo); |
|||
|
|||
// save transactions to disk
|
|||
die('TODO save to disk!'); |
|||
|
|||
$count = count($CSVLines); |
|||
$this->mergeMessages($count); |
|||
$this->mergeWarnings($count); |
|||
$this->mergeErrors($count); |
|||
} |
|||
|
|||
/** |
|||
* @inheritDoc |
|||
*/ |
|||
public function getIdentifier(): string |
|||
{ |
|||
return $this->identifier; |
|||
} |
|||
|
|||
/** |
|||
* |
|||
*/ |
|||
private function generateIdentifier(): void |
|||
{ |
|||
Log::debug('Going to generate conversion routine identifier.'); |
|||
$disk = Storage::disk(self::DISK_NAME); |
|||
$count = 0; |
|||
do { |
|||
$generatedId = sprintf('conv-%s', Str::random(12)); |
|||
$count++; |
|||
Log::debug(sprintf('Attempt #%d results in "%s"', $count, $generatedId)); |
|||
} while ($count < 30 && $disk->exists($generatedId)); |
|||
$this->identifier = $generatedId; |
|||
Log::info(sprintf('Job identifier is "%s"', $generatedId)); |
|||
} |
|||
} |
|||
@ -0,0 +1,84 @@ |
|||
<?php |
|||
/* |
|||
* ImportJobStatus.php |
|||
* Copyright (c) 2021 james@firefly-iii.org |
|||
* |
|||
* This file is part of the Firefly III Data Importer |
|||
* (https://github.com/firefly-iii/data-importer). |
|||
* |
|||
* This program is free software: you can redistribute it and/or modify |
|||
* it under the terms of the GNU Affero General Public License as |
|||
* published by the Free Software Foundation, either version 3 of the |
|||
* License, or (at your option) any later version. |
|||
* |
|||
* This program is distributed in the hope that it will be useful, |
|||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|||
* GNU Affero General Public License for more details. |
|||
* |
|||
* You should have received a copy of the GNU Affero General Public License |
|||
* along with this program. If not, see <https://www.gnu.org/licenses/>. |
|||
*/ |
|||
|
|||
declare(strict_types=1); |
|||
|
|||
namespace App\Services\Shared\Conversion; |
|||
|
|||
/** |
|||
* Class ConversionStatus |
|||
*/ |
|||
class ConversionStatus |
|||
{ |
|||
/** @var string */ |
|||
public const CONVERSION_DONE = 'conv_done'; |
|||
/** @var string */ |
|||
public const CONVERSION_ERRORED = 'conv_errored'; |
|||
/** @var string */ |
|||
public const CONVERSION_RUNNING = 'conv_running'; |
|||
/** @var string */ |
|||
public const CONVERSION_WAITING = 'waiting_to_start'; |
|||
public string $status; |
|||
public array $errors; |
|||
public array $warnings; |
|||
public array $messages; |
|||
|
|||
/** |
|||
* ConversionStatus constructor. |
|||
*/ |
|||
public function __construct() |
|||
{ |
|||
$this->status = self::CONVERSION_WAITING; |
|||
$this->errors = []; |
|||
$this->warnings = []; |
|||
$this->messages = []; |
|||
} |
|||
|
|||
/** |
|||
* @param array $array |
|||
* |
|||
* @return static |
|||
*/ |
|||
public static function fromArray(array $array): self |
|||
{ |
|||
$config = new self; |
|||
$config->status = $array['status']; |
|||
$config->errors = $array['errors'] ?? []; |
|||
$config->warnings = $array['warnings'] ?? []; |
|||
$config->messages = $array['messages'] ?? []; |
|||
|
|||
return $config; |
|||
} |
|||
|
|||
/** |
|||
* @return array |
|||
*/ |
|||
public function toArray(): array |
|||
{ |
|||
return [ |
|||
'status' => $this->status, |
|||
'errors' => $this->errors, |
|||
'warnings' => $this->warnings, |
|||
'messages' => $this->messages, |
|||
]; |
|||
} |
|||
} |
|||
@ -0,0 +1,47 @@ |
|||
<?php |
|||
/* |
|||
* RoutineManagerInterface.php |
|||
* Copyright (c) 2021 james@firefly-iii.org |
|||
* |
|||
* This file is part of the Firefly III Data Importer |
|||
* (https://github.com/firefly-iii/data-importer). |
|||
* |
|||
* This program is free software: you can redistribute it and/or modify |
|||
* it under the terms of the GNU Affero General Public License as |
|||
* published by the Free Software Foundation, either version 3 of the |
|||
* License, or (at your option) any later version. |
|||
* |
|||
* This program is distributed in the hope that it will be useful, |
|||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|||
* GNU Affero General Public License for more details. |
|||
* |
|||
* You should have received a copy of the GNU Affero General Public License |
|||
* along with this program. If not, see <https://www.gnu.org/licenses/>. |
|||
*/ |
|||
|
|||
namespace App\Services\Shared\Conversion; |
|||
|
|||
use App\Services\CSV\Configuration\Configuration; |
|||
|
|||
/** |
|||
* Interface RoutineManagerInterface |
|||
*/ |
|||
interface RoutineManagerInterface |
|||
{ |
|||
/** |
|||
* @param Configuration $configuration |
|||
*/ |
|||
public function setConfiguration(Configuration $configuration): void; |
|||
|
|||
/** |
|||
* |
|||
*/ |
|||
public function start(): void; |
|||
|
|||
/** |
|||
* @return string |
|||
*/ |
|||
public function getIdentifier(): string; |
|||
|
|||
} |
|||
@ -0,0 +1,205 @@ |
|||
<?php |
|||
/* |
|||
* RoutineStatusManager.php |
|||
* Copyright (c) 2021 james@firefly-iii.org |
|||
* |
|||
* This file is part of the Firefly III Data Importer |
|||
* (https://github.com/firefly-iii/data-importer). |
|||
* |
|||
* This program is free software: you can redistribute it and/or modify |
|||
* it under the terms of the GNU Affero General Public License as |
|||
* published by the Free Software Foundation, either version 3 of the |
|||
* License, or (at your option) any later version. |
|||
* |
|||
* This program is distributed in the hope that it will be useful, |
|||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|||
* GNU Affero General Public License for more details. |
|||
* |
|||
* You should have received a copy of the GNU Affero General Public License |
|||
* along with this program. If not, see <https://www.gnu.org/licenses/>. |
|||
*/ |
|||
|
|||
declare(strict_types=1); |
|||
|
|||
namespace App\Services\Shared\Conversion; |
|||
|
|||
use App\Exceptions\ImporterErrorException; |
|||
use App\Services\Session\Constants; |
|||
use Illuminate\Contracts\Filesystem\FileNotFoundException; |
|||
use JsonException; |
|||
use Log; |
|||
use Psr\Container\ContainerExceptionInterface; |
|||
use Psr\Container\NotFoundExceptionInterface; |
|||
use Storage; |
|||
|
|||
/** |
|||
* Class RoutineStatusManager |
|||
*/ |
|||
class RoutineStatusManager |
|||
{ |
|||
private const DISK_NAME = 'conversion-routines'; |
|||
|
|||
/** |
|||
* @param string $identifier |
|||
* @param int $index |
|||
* @param string $error |
|||
*/ |
|||
public static function addError(string $identifier, int $index, string $error): void |
|||
{ |
|||
$lineNo = $index + 1; |
|||
Log::debug(sprintf('Add error on index #%d (line no. %d): %s', $index, $lineNo, $error)); |
|||
|
|||
$disk = Storage::disk(self::DISK_NAME); |
|||
try { |
|||
if ($disk->exists($identifier)) { |
|||
try { |
|||
$status = ConversionStatus::fromArray(json_decode($disk->get($identifier), true, 512, JSON_THROW_ON_ERROR)); |
|||
} catch (JsonException $e) { |
|||
Log::error($e->getMessage()); |
|||
$status = new ConversionStatus; |
|||
} |
|||
$status->errors[$index] = $status->errors[$index] ?? []; |
|||
$status->errors[$index][] = $error; |
|||
self::storeConversionStatus($identifier, $status); |
|||
} |
|||
} catch (FileNotFoundException $e) { |
|||
Log::error($e->getMessage()); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* @param string $identifier |
|||
* @param ConversionStatus $status |
|||
*/ |
|||
private static function storeConversionStatus(string $identifier, ConversionStatus $status): void |
|||
{ |
|||
Log::debug(sprintf('Now in storeConversionStatus(%s): %s', $identifier, $status->status)); |
|||
$disk = Storage::disk(self::DISK_NAME); |
|||
try { |
|||
$disk->put($identifier, json_encode($status->toArray(), JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT)); |
|||
} catch (JsonException $e) { |
|||
// do nothing
|
|||
Log::error($e->getMessage()); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* @param string $identifier |
|||
* @param int $index |
|||
* @param string $warning |
|||
* |
|||
*/ |
|||
public static function addWarning(string $identifier, int $index, string $warning): void |
|||
{ |
|||
$lineNo = $index + 1; |
|||
Log::debug(sprintf('Add warning on index #%d (line no. %d): %s', $index, $lineNo, $warning)); |
|||
|
|||
$disk = Storage::disk(self::DISK_NAME); |
|||
try { |
|||
if ($disk->exists($identifier)) { |
|||
try { |
|||
$status = ConversionStatus::fromArray(json_decode($disk->get($identifier), true, 512, JSON_THROW_ON_ERROR)); |
|||
} catch (JsonException $e) { |
|||
Log::error($e->getMessage()); |
|||
$status = new ConversionStatus; |
|||
} |
|||
$status->warnings[$index] = $status->warnings[$index] ?? []; |
|||
$status->warnings[$index][] = $warning; |
|||
self::storeConversionStatus($identifier, $status); |
|||
} |
|||
} catch (FileNotFoundException $e) { |
|||
Log::error($e->getMessage()); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* @param string $identifier |
|||
* @param int $index |
|||
* @param string $message |
|||
* |
|||
*/ |
|||
public static function addMessage(string $identifier, int $index, string $message): void |
|||
{ |
|||
$lineNo = $index + 1; |
|||
Log::debug(sprintf('Add message on index #%d (line no. %d): %s', $index, $lineNo, $message)); |
|||
|
|||
$disk = Storage::disk(self::DISK_NAME); |
|||
try { |
|||
if ($disk->exists($identifier)) { |
|||
try { |
|||
$status = ConversionStatus::fromArray(json_decode($disk->get($identifier), true, 512, JSON_THROW_ON_ERROR)); |
|||
} catch (JsonException $e) { |
|||
Log::error($e->getMessage()); |
|||
$status = new ConversionStatus; |
|||
} |
|||
$status->messages[$index] = $status->messages[$index] ?? []; |
|||
$status->messages[$index][] = $message; |
|||
self::storeConversionStatus($identifier, $status); |
|||
} |
|||
} catch (FileNotFoundException $e) { |
|||
Log::error($e->getMessage()); |
|||
} |
|||
} |
|||
|
|||
|
|||
/** |
|||
* @param string $status |
|||
* |
|||
* @return ConversionStatus |
|||
* @throws ImporterErrorException |
|||
*/ |
|||
public static function setConversionStatus(string $status): ConversionStatus |
|||
{ |
|||
try { |
|||
$identifier = session()->get(Constants::CONVERSION_JOB_IDENTIFIER); |
|||
} catch (ContainerExceptionInterface|NotFoundExceptionInterface $e) { |
|||
throw new ImporterErrorException('No identifier found'); |
|||
} |
|||
Log::debug(sprintf('Now in setConversionStatus(%s)', $status)); |
|||
Log::debug(sprintf('Found "%s" in the session', $identifier)); |
|||
|
|||
$jobStatus = self::startOrFindConversion($identifier); |
|||
$jobStatus->status = $status; |
|||
|
|||
self::storeConversionStatus($identifier, $jobStatus); |
|||
|
|||
return $jobStatus; |
|||
} |
|||
|
|||
/** |
|||
* @param string $identifier |
|||
* |
|||
* @return ConversionStatus |
|||
*/ |
|||
public static function startOrFindConversion(string $identifier): ConversionStatus |
|||
{ |
|||
Log::debug(sprintf('Now in startOrFindConversion(%s)', $identifier)); |
|||
$disk = Storage::disk(self::DISK_NAME); |
|||
Log::debug(sprintf('Try to see if file exists for conversion "%s".', $identifier)); |
|||
if ($disk->exists($identifier)) { |
|||
Log::debug(sprintf('Status file exists for conversion "%s".', $identifier)); |
|||
try { |
|||
$array = json_decode($disk->get($identifier), true, 512, JSON_THROW_ON_ERROR); |
|||
$status = ConversionStatus::fromArray($array); |
|||
} catch (FileNotFoundException | JsonException $e) { |
|||
Log::error($e->getMessage()); |
|||
$status = new ConversionStatus; |
|||
} |
|||
|
|||
return $status; |
|||
|
|||
} |
|||
Log::debug('File does not exist or error, create a new one.'); |
|||
$status = new ConversionStatus; |
|||
try { |
|||
$disk->put($identifier, json_encode($status->toArray(), JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT)); |
|||
} catch (JsonException $e) { |
|||
Log::error($e->getMessage()); |
|||
} |
|||
|
|||
Log::debug('Return status.', $status->toArray()); |
|||
|
|||
return $status; |
|||
} |
|||
} |
|||
@ -0,0 +1,2 @@ |
|||
* |
|||
!.gitignore |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue