You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
491 lines
18 KiB
491 lines
18 KiB
<?php
|
|
/*
|
|
* AutoImports.php
|
|
* Copyright (c) 2021 james@firefly-iii.org
|
|
*
|
|
* This file is part of the Firefly III Data Importer
|
|
* (https://github.com/firefly-iii/data-importer).
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU Affero General Public License as
|
|
* published by the Free Software Foundation, either version 3 of the
|
|
* License, or (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU Affero General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Console;
|
|
|
|
use App\Events\ImportedTransactions;
|
|
use App\Exceptions\ImporterErrorException;
|
|
use App\Services\Camt\Conversion\RoutineManager as CamtRoutineManager;
|
|
use App\Services\CSV\Conversion\RoutineManager as CSVRoutineManager;
|
|
use App\Services\Nordigen\Conversion\RoutineManager as NordigenRoutineManager;
|
|
use App\Services\Shared\Configuration\Configuration;
|
|
use App\Services\Shared\Conversion\ConversionStatus;
|
|
use App\Services\Shared\Conversion\RoutineStatusManager;
|
|
use App\Services\Shared\File\FileContentSherlock;
|
|
use App\Services\Shared\Import\Routine\RoutineManager;
|
|
use App\Services\Shared\Import\Status\SubmissionStatus;
|
|
use App\Services\Shared\Import\Status\SubmissionStatusManager;
|
|
use App\Services\Spectre\Conversion\RoutineManager as SpectreRoutineManager;
|
|
use Illuminate\Contracts\Filesystem\FileNotFoundException;
|
|
use JsonException;
|
|
use League\Flysystem\FilesystemException;
|
|
use Storage;
|
|
|
|
/**
|
|
* Trait AutoImports
|
|
*/
|
|
trait AutoImports
|
|
{
|
|
protected array $conversionErrors = [];
|
|
protected array $conversionMessages = [];
|
|
protected array $conversionWarnings = [];
|
|
protected string $identifier;
|
|
protected array $importErrors = [];
|
|
protected array $importMessages = [];
|
|
protected array $importWarnings = [];
|
|
|
|
/**
|
|
* @param string $directory
|
|
*
|
|
* @return array
|
|
*/
|
|
protected function getFiles(string $directory): array
|
|
{
|
|
$ignore = ['.', '..'];
|
|
|
|
if ('' === $directory) {
|
|
$this->error(sprintf('Directory "%s" is empty or invalid.', $directory));
|
|
|
|
return [];
|
|
}
|
|
$array = scandir($directory);
|
|
if (!is_array($array)) {
|
|
$this->error(sprintf('Directory "%s" is empty or invalid.', $directory));
|
|
|
|
return [];
|
|
}
|
|
$files = array_diff($array, $ignore);
|
|
$return = [];
|
|
foreach ($files as $file) {
|
|
// import importable file with JSON companion
|
|
// TODO may also need to be able to detect other file types. Or: detect any file with accompanying json file
|
|
if ('csv' === $this->getExtension($file) && $this->hasJsonConfiguration($directory, $file)) {
|
|
$return[] = $file;
|
|
}
|
|
// import JSON with no importable file.
|
|
// TODO must detect json files without accompanying camt/csv/whatever file.
|
|
if ('json' === $this->getExtension($file) && !$this->hasCsvFile($directory, $file)) {
|
|
$return[] = $file;
|
|
}
|
|
}
|
|
|
|
return $return;
|
|
}
|
|
|
|
/**
|
|
* @param string $directory
|
|
* @param array $files
|
|
*
|
|
* @throws ImporterErrorException
|
|
*/
|
|
protected function importFiles(string $directory, array $files): void
|
|
{
|
|
/** @var string $file */
|
|
foreach ($files as $file) {
|
|
$this->importFile($directory, $file);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param string $file
|
|
*
|
|
* @return string
|
|
*/
|
|
private function getExtension(string $file): string
|
|
{
|
|
$parts = explode('.', $file);
|
|
if (1 === count($parts)) {
|
|
return '';
|
|
}
|
|
|
|
return strtolower($parts[count($parts) - 1]);
|
|
}
|
|
|
|
/**
|
|
* TODO this function must be more universal.
|
|
*
|
|
* @param string $directory
|
|
* @param string $file
|
|
*
|
|
* @return bool
|
|
*/
|
|
private function hasCsvFile(string $directory, string $file): bool
|
|
{
|
|
$short = substr($file, 0, -5);
|
|
$csvFile = sprintf('%s.csv', $short);
|
|
$fullJson = sprintf('%s/%s', $directory, $csvFile);
|
|
if (!file_exists($fullJson)) {
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* @param string $directory
|
|
* @param string $file
|
|
*
|
|
* @return bool
|
|
*/
|
|
private function hasJsonConfiguration(string $directory, string $file): bool
|
|
{
|
|
$short = substr($file, 0, -4);
|
|
$jsonFile = sprintf('%s.json', $short);
|
|
$fullJson = sprintf('%s/%s', $directory, $jsonFile);
|
|
if (!file_exists($fullJson)) {
|
|
$this->warn(sprintf('Can\'t find JSON file "%s" expected to go with CSV file "%s". CSV file will be ignored.', $fullJson, $file));
|
|
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* @param string $file
|
|
* @param string $directory
|
|
*
|
|
* @throws ImporterErrorException
|
|
*/
|
|
private function importFile(string $directory, string $file): void
|
|
{
|
|
app('log')->debug(sprintf('ImportFile: directory "%s"', $directory));
|
|
app('log')->debug(sprintf('ImportFile: file "%s"', $file));
|
|
$importableFile = sprintf('%s/%s', $directory, $file);
|
|
$jsonFile = sprintf('%s/%s.json', $directory, substr($file, 0, -5));
|
|
|
|
// TODO not yet sure why the distinction is necessary.
|
|
// TODO this may also be necessary for camt files.
|
|
if ('csv' === $this->getExtension($file)) {
|
|
$jsonFile = sprintf('%s/%s.json', $directory, substr($file, 0, -4));
|
|
}
|
|
// same for XML files.
|
|
if ('xml' === $this->getExtension($file)) {
|
|
$jsonFile = sprintf('%s/%s.json', $directory, substr($file, 0, -4));
|
|
}
|
|
|
|
app('log')->debug(sprintf('ImportFile: importable "%s"', $importableFile));
|
|
app('log')->debug(sprintf('ImportFile: JSON "%s"', $jsonFile));
|
|
|
|
// do JSON check
|
|
$jsonResult = $this->verifyJSON($jsonFile);
|
|
if (false === $jsonResult) {
|
|
$message = sprintf('The importer can\'t import %s: could not decode the JSON in config file %s.', $importableFile, $jsonFile);
|
|
$this->error($message);
|
|
|
|
return;
|
|
}
|
|
$configuration = Configuration::fromArray(json_decode(file_get_contents($jsonFile), true));
|
|
|
|
// sanity check. If the importableFile is a .json file, and it parses as valid json, don't import it:
|
|
if ('file' === $configuration->getFlow() && str_ends_with(strtolower($importableFile), '.json') && $this->verifyJSON($importableFile)) {
|
|
app('log')->warning('Almost tried to import a JSON file as a file lol. Skip it.');
|
|
|
|
return;
|
|
}
|
|
|
|
$configuration->updateDateRange();
|
|
$this->line(sprintf('Going to convert from file %s using configuration %s and flow "%s".', $importableFile, $jsonFile, $configuration->getFlow()));
|
|
|
|
// this is it!
|
|
$this->startConversion($configuration, $importableFile);
|
|
$this->reportConversion();
|
|
|
|
$this->line(sprintf('Done converting from file %s using configuration %s.', $importableFile, $jsonFile));
|
|
$this->startImport($configuration);
|
|
$this->reportImport();
|
|
|
|
$this->line('Done!');
|
|
event(
|
|
new ImportedTransactions(
|
|
array_merge($this->conversionMessages, $this->importMessages),
|
|
array_merge($this->conversionWarnings, $this->importWarnings),
|
|
array_merge($this->conversionErrors, $this->importErrors)
|
|
)
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @param string $jsonFile
|
|
* @param null|string $importableFile
|
|
*
|
|
* @throws ImporterErrorException
|
|
*/
|
|
private function importUpload(string $jsonFile, ?string $importableFile): void
|
|
{
|
|
// do JSON check
|
|
$jsonResult = $this->verifyJSON($jsonFile);
|
|
if (false === $jsonResult) {
|
|
$message = sprintf('The importer can\'t import %s: could not decode the JSON in config file %s.', $importableFile, $jsonFile);
|
|
$this->error($message);
|
|
|
|
return;
|
|
}
|
|
$configuration = Configuration::fromArray(json_decode(file_get_contents($jsonFile), true));
|
|
$configuration->updateDateRange();
|
|
|
|
$this->line(sprintf('Going to convert from file "%s" using configuration "%s" and flow "%s".', $importableFile, $jsonFile, $configuration->getFlow()));
|
|
|
|
// this is it!
|
|
$this->startConversion($configuration, $importableFile);
|
|
$this->reportConversion();
|
|
|
|
$this->line(sprintf('Done converting from file %s using configuration %s.', $importableFile, $jsonFile));
|
|
$this->startImport($configuration);
|
|
$this->reportImport();
|
|
|
|
$this->line('Done!');
|
|
event(
|
|
new ImportedTransactions(
|
|
array_merge($this->conversionMessages, $this->importMessages),
|
|
array_merge($this->conversionWarnings, $this->importWarnings),
|
|
array_merge($this->conversionErrors, $this->importErrors)
|
|
)
|
|
);
|
|
}
|
|
|
|
/**
|
|
*
|
|
*/
|
|
private function reportConversion(): void
|
|
{
|
|
$list = [
|
|
'info' => $this->conversionMessages,
|
|
'warn' => $this->conversionWarnings,
|
|
'error' => $this->conversionErrors,
|
|
];
|
|
foreach ($list as $func => $set) {
|
|
/**
|
|
* @var int $index
|
|
* @var array $messages
|
|
*/
|
|
foreach ($set as $index => $messages) {
|
|
if (count($messages) > 0) {
|
|
foreach ($messages as $message) {
|
|
$this->$func(sprintf('Conversion index %d: %s', $index, $message));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
*
|
|
*/
|
|
private function reportImport(): void
|
|
{
|
|
$list = [
|
|
'info' => $this->importMessages,
|
|
'warn' => $this->importWarnings,
|
|
'error' => $this->importErrors,
|
|
];
|
|
foreach ($list as $func => $set) {
|
|
/**
|
|
* @var int $index
|
|
* @var array $messages
|
|
*/
|
|
foreach ($set as $index => $messages) {
|
|
if (count($messages) > 0) {
|
|
foreach ($messages as $message) {
|
|
$this->$func(sprintf('Import index %d: %s', $index, $message));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param Configuration $configuration
|
|
*
|
|
* @param string|null $importableFile
|
|
*
|
|
* @throws ImporterErrorException
|
|
*/
|
|
private function startConversion(Configuration $configuration, ?string $importableFile): void
|
|
{
|
|
$this->conversionMessages = [];
|
|
$this->conversionWarnings = [];
|
|
$this->conversionErrors = [];
|
|
|
|
app('log')->debug(sprintf('Now in %s', __METHOD__));
|
|
|
|
switch ($configuration->getFlow()) {
|
|
default:
|
|
$this->error(sprintf('There is no support for flow "%s"', $configuration->getFlow()));
|
|
exit();
|
|
case 'file':
|
|
$contentType = $configuration->getContentType();
|
|
if ('unknown' === $contentType) {
|
|
app('log')->debug('Content type is "unknown" in startConversion(), detect it.');
|
|
$detector = new FileContentSherlock();
|
|
$contentType = $detector->detectContentType($importableFile);
|
|
}
|
|
switch ($contentType) {
|
|
default:
|
|
case 'unknown':
|
|
case 'csv':
|
|
$manager = new CSVRoutineManager(null);
|
|
$this->identifier = $manager->getIdentifier();
|
|
$manager->setContent(file_get_contents($importableFile));
|
|
break;
|
|
case 'camt':
|
|
$manager = new CamtRoutineManager(null);
|
|
$this->identifier = $manager->getIdentifier();
|
|
$manager->setContent(file_get_contents($importableFile));
|
|
break;
|
|
}
|
|
|
|
break;
|
|
case 'nordigen':
|
|
$manager = new NordigenRoutineManager(null);
|
|
$this->identifier = $manager->getIdentifier();
|
|
break;
|
|
case 'spectre':
|
|
$manager = new SpectreRoutineManager(null);
|
|
$this->identifier = $manager->getIdentifier();
|
|
break;
|
|
}
|
|
|
|
RoutineStatusManager::startOrFindConversion($this->identifier);
|
|
RoutineStatusManager::setConversionStatus(ConversionStatus::CONVERSION_RUNNING, $this->identifier);
|
|
|
|
// then push stuff into the routine:
|
|
$manager->setConfiguration($configuration);
|
|
$transactions = [];
|
|
try {
|
|
$transactions = $manager->start();
|
|
} catch (ImporterErrorException $e) {
|
|
app('log')->error($e->getMessage());
|
|
RoutineStatusManager::setConversionStatus(ConversionStatus::CONVERSION_ERRORED, $this->identifier);
|
|
$this->conversionMessages = $manager->getAllMessages();
|
|
$this->conversionWarnings = $manager->getAllWarnings();
|
|
$this->conversionErrors = $manager->getAllErrors();
|
|
}
|
|
if (0 === count($transactions)) {
|
|
app('log')->error('Zero transactions!');
|
|
RoutineStatusManager::setConversionStatus(ConversionStatus::CONVERSION_ERRORED, $this->identifier);
|
|
$this->conversionMessages = $manager->getAllMessages();
|
|
$this->conversionWarnings = $manager->getAllWarnings();
|
|
$this->conversionErrors = $manager->getAllErrors();
|
|
}
|
|
// save transactions in 'jobs' directory under the same key as the conversion thing.
|
|
$disk = Storage::disk('jobs');
|
|
try {
|
|
$disk->put(sprintf('%s.json', $this->identifier), json_encode($transactions, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
|
} catch (JsonException $e) {
|
|
app('log')->error(sprintf('JSON exception: %s', $e->getMessage()));
|
|
RoutineStatusManager::setConversionStatus(ConversionStatus::CONVERSION_ERRORED, $this->identifier);
|
|
$this->conversionMessages = $manager->getAllMessages();
|
|
$this->conversionWarnings = $manager->getAllWarnings();
|
|
$this->conversionErrors = $manager->getAllErrors();
|
|
$transactions = [];
|
|
}
|
|
|
|
if (count($transactions) > 0) {
|
|
// set done:
|
|
RoutineStatusManager::setConversionStatus(ConversionStatus::CONVERSION_DONE, $this->identifier);
|
|
|
|
$this->conversionMessages = $manager->getAllMessages();
|
|
$this->conversionWarnings = $manager->getAllWarnings();
|
|
$this->conversionErrors = $manager->getAllErrors();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param Configuration $configuration
|
|
* @throws FilesystemException
|
|
*/
|
|
private function startImport(Configuration $configuration): void
|
|
{
|
|
app('log')->debug(sprintf('Now at %s', __METHOD__));
|
|
$routine = new RoutineManager($this->identifier);
|
|
SubmissionStatusManager::startOrFindSubmission($this->identifier);
|
|
$disk = Storage::disk('jobs');
|
|
$fileName = sprintf('%s.json', $this->identifier);
|
|
|
|
// get files from disk:
|
|
if (!$disk->has($fileName)) {
|
|
SubmissionStatusManager::setSubmissionStatus(SubmissionStatus::SUBMISSION_ERRORED, $this->identifier);
|
|
$message = sprintf('File "%s" not found, cannot continue.', $fileName);
|
|
$this->error($message);
|
|
SubmissionStatusManager::addError($this->identifier, 0, $message);
|
|
$this->importMessages = $routine->getAllMessages();
|
|
$this->importWarnings = $routine->getAllWarnings();
|
|
$this->importErrors = $routine->getAllErrors();
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
try {
|
|
$json = $disk->get($fileName);
|
|
$transactions = json_decode($json, true, 512, JSON_THROW_ON_ERROR);
|
|
app('log')->debug(sprintf('Found %d transactions on the drive.', count($transactions)));
|
|
} catch (FileNotFoundException|JsonException $e) {
|
|
SubmissionStatusManager::setSubmissionStatus(SubmissionStatus::SUBMISSION_ERRORED, $this->identifier);
|
|
$message = sprintf('File "%s" could not be decoded, cannot continue..', $fileName);
|
|
$this->error($message);
|
|
SubmissionStatusManager::addError($this->identifier, 0, $message);
|
|
$this->importMessages = $routine->getAllMessages();
|
|
$this->importWarnings = $routine->getAllWarnings();
|
|
$this->importErrors = $routine->getAllErrors();
|
|
|
|
return;
|
|
}
|
|
if (0 === count($transactions)) {
|
|
SubmissionStatusManager::setSubmissionStatus(SubmissionStatus::SUBMISSION_DONE, $this->identifier);
|
|
app('log')->error('No transactions in array, there is nothing to import.');
|
|
$this->importMessages = $routine->getAllMessages();
|
|
$this->importWarnings = $routine->getAllWarnings();
|
|
$this->importErrors = $routine->getAllErrors();
|
|
|
|
return;
|
|
}
|
|
|
|
$routine->setTransactions($transactions);
|
|
|
|
SubmissionStatusManager::setSubmissionStatus(SubmissionStatus::SUBMISSION_RUNNING, $this->identifier);
|
|
|
|
// then push stuff into the routine:
|
|
$routine->setConfiguration($configuration);
|
|
try {
|
|
$routine->start();
|
|
} catch (ImporterErrorException $e) {
|
|
app('log')->error($e->getMessage());
|
|
SubmissionStatusManager::setSubmissionStatus(SubmissionStatus::SUBMISSION_ERRORED, $this->identifier);
|
|
SubmissionStatusManager::addError($this->identifier, 0, $e->getMessage());
|
|
$this->importMessages = $routine->getAllMessages();
|
|
$this->importWarnings = $routine->getAllWarnings();
|
|
$this->importErrors = $routine->getAllErrors();
|
|
|
|
return;
|
|
}
|
|
|
|
// set done:
|
|
SubmissionStatusManager::setSubmissionStatus(SubmissionStatus::SUBMISSION_DONE, $this->identifier);
|
|
$this->importMessages = $routine->getAllMessages();
|
|
$this->importWarnings = $routine->getAllWarnings();
|
|
$this->importErrors = $routine->getAllErrors();
|
|
}
|
|
}
|