Mirrors
/
data-importer
mirror of https://github.com/firefly-iii/data-importer.git
3
0
Fork 0
Code Releases Activity
Browse Source

Catch errors better

pull/473/head
James Cole 2 years ago
parent
a1545df4fc
commit
d48a703eba
No known key found for this signature in database GPG Key ID: B49A324B7EAD6D80
5 changed files with 81 additions and 69 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 29
      app/Console/AutoImports.php
  2. 9
      app/Console/Commands/Import.php
  3. 2
      app/Services/CSV/Conversion/Routine/CSVFileProcessor.php
  4. 4
      app/Services/CSV/Conversion/Routine/PseudoTransactionProcessor.php
  5. 106
      app/Services/CSV/Conversion/RoutineManager.php

29
app/Console/AutoImports.php
View File

@ -225,28 +225,10 @@ trait AutoImports
app('log')->error('Conversion errors', $this->conversionErrors);
app('log')->error('Conversion warnings', $this->conversionWarnings);
app('log')->error('Conversion messages', $this->conversionMessages);
// log all conversion errors first.
foreach ($this->conversionErrors as $index => $errors) {
foreach ($errors as $error) {
app('log')->error(sprintf('Conversion error on line #%d: %s', $index, $error));
}
}
foreach ($this->conversionWarnings as $index => $warnings) {
foreach ($warnings as $warning) {
app('log')->warning(sprintf('Conversion warning on line #%d: %s', $index, $warning));
}
}
foreach ($this->conversionMessages as $index => $messages) {
foreach ($messages as $message) {
app('log')->info(sprintf('Conversion message on line #%d: %s', $index, $message));
}
}
$this->error(sprintf('Too many errors in the data conversion (%d), exit.', count($this->conversionErrors)));
throw new ImporterErrorException('Too many errors in the data conversion.');
}
$this->line(sprintf('Done converting from file %s using configuration %s.', $importableFile, $jsonFile));
$this->startImport($configuration);
$this->reportImport();
@ -286,6 +268,15 @@ trait AutoImports
$this->startConversion($configuration, $importableFile);
$this->reportConversion();
// crash here if the conversion failed.
if (0 !== count($this->conversionErrors)) {
app('log')->error('Conversion errors', $this->conversionErrors);
app('log')->error('Conversion warnings', $this->conversionWarnings);
app('log')->error('Conversion messages', $this->conversionMessages);
$this->error(sprintf('Too many errors in the data conversion (%d), exit.', count($this->conversionErrors)));
throw new ImporterErrorException('Too many errors in the data conversion.');
}
$this->line(sprintf('Done converting from file %s using configuration %s.', $importableFile, $jsonFile));
$this->startImport($configuration);
$this->reportImport();
@ -318,7 +309,7 @@ trait AutoImports
foreach ($set as $index => $messages) {
if (count($messages) > 0) {
foreach ($messages as $message) {
$this->$func(sprintf('Conversion index %d: %s', $index, $message)); // @phpstan-ignore-line
$this->$func(sprintf('Conversion index (%s) %d: %s', $func, $index, $message)); // @phpstan-ignore-line
}
}
}

9
app/Console/Commands/Import.php
View File

@ -132,6 +132,15 @@ final class Import extends Command
$this->startConversion($configuration, $file);
$this->reportConversion();
// crash here if the conversion failed.
if (0 !== count($this->conversionErrors)) {
app('log')->error('Conversion errors', $this->conversionErrors);
app('log')->error('Conversion warnings', $this->conversionWarnings);
app('log')->error('Conversion messages', $this->conversionMessages);
$this->error(sprintf('Too many errors in the data conversion (%d), exit.', count($this->conversionErrors)));
throw new ImporterErrorException('Too many errors in the data conversion.');
}
$this->line(sprintf('Done converting from file %s using configuration %s.', $file, $config));
$this->startImport($configuration);
$this->reportImport();

2
app/Services/CSV/Conversion/Routine/CSVFileProcessor.php
View File

@ -65,6 +65,7 @@ class CSVFileProcessor
app('log')->debug('Now in processCSVFile()');
$offset = $this->hasHeaders ? 1 : 0;
try {
$this->reader->setDelimiter($this->delimiter);
@ -149,6 +150,7 @@ class CSVFileProcessor
$line = $this->sanitize($line);
app('log')->debug(sprintf('Parsing line %d/%d', $currentIndex, $count));
$updatedRecords[] = $line;
$currentIndex++;
}
app('log')->info(sprintf('Parsed all %d lines.', $count));

4
app/Services/CSV/Conversion/Routine/PseudoTransactionProcessor.php
View File

@ -73,14 +73,14 @@ class PseudoTransactionProcessor
app('log')->debug(sprintf('Now in %s', __METHOD__));
$count = count($lines);
$processed = [];
app('log')->info(sprintf('Converting %d lines into transactions.', $count));
app('log')->info(sprintf('Converting %d line(s) into transactions.', $count));
/** @var array $line */
foreach ($lines as $index => $line) {
app('log')->info(sprintf('Now processing line %d/%d.', ($index + 1), $count));
$processed[] = $this->processPseudoLine($line);
// $this->addMessage($index, sprintf('Converted CSV line %d into a transaction.', $index + 1));
}
app('log')->info(sprintf('Done converting %d lines into transactions.', $count));
app('log')->info(sprintf('Done converting %d line(s) into transactions.', $count));
return $processed;
}

106
app/Services/CSV/Conversion/RoutineManager.php
View File

@ -121,7 +121,7 @@ class RoutineManager implements RoutineManagerInterface
}
/**
* @param string $content
* @param string $content
*/
public function setContent(string $content): void
{
@ -129,7 +129,7 @@ class RoutineManager implements RoutineManagerInterface
}
/**
* @param bool $forceCli
* @param bool $forceCli
*/
public function setForceCli(bool $forceCli): void
{
@ -155,7 +155,7 @@ class RoutineManager implements RoutineManagerInterface
if ('' === $this->content) {
try {
$this->csvFileProcessor->setReader(FileReader::getReaderFromSession($this->configuration->isConversion()));
} catch (ContainerExceptionInterface|NotFoundExceptionInterface $e) {
} catch (ContainerExceptionInterface | NotFoundExceptionInterface $e) {
throw new ImporterErrorException($e->getMessage(), 0, $e);
}
}
@ -171,6 +171,7 @@ class RoutineManager implements RoutineManagerInterface
// convert pseudo transactions into actual transactions.
$transactions = $this->pseudoTransactionProcessor->processPseudo($pseudo);
$count = count($CSVLines);
$this->mergeMessages($count);
$this->mergeWarnings($count);
@ -180,67 +181,76 @@ class RoutineManager implements RoutineManagerInterface
}
/**
* @param int $count
* @param int $count
*/
private function mergeErrors(int $count): void
{
$one = $this->csvFileProcessor->getErrors();
$two = $this->lineProcessor->getErrors();
$three = $this->columnValueConverter->getErrors();
$four = $this->pseudoTransactionProcessor->getErrors();
$total = [];
for ($i = 0; $i < $count; $i++) {
$total[$i] = array_merge(
$one[$i] ?? [],
$two[$i] ?? [],
$three[$i] ?? [],
$four[$i] ?? [],
);
}
$this->allErrors = $this->mergeArrays([
$this->csvFileProcessor->getErrors(),
$this->lineProcessor->getErrors(),
$this->columnValueConverter->getErrors(),
$this->pseudoTransactionProcessor->getErrors(),
], $count);
$this->allErrors = $total;
}
/**
* @param int $count
* @param int $count
*/
private function mergeMessages(int $count): void
{
$one = $this->csvFileProcessor->getMessages();
$two = $this->lineProcessor->getMessages();
$three = $this->columnValueConverter->getMessages();
$four = $this->pseudoTransactionProcessor->getMessages();
$total = [];
for ($i = 0; $i < $count; $i++) {
$total[$i] = array_merge(
$one[$i] ?? [],
$two[$i] ?? [],
$three[$i] ?? [],
$four[$i] ?? [],
);
}
$this->allMessages = $total;
$this->allMessages = $this->mergeArrays([
$this->csvFileProcessor->getMessages(),
$this->lineProcessor->getMessages(),
$this->columnValueConverter->getMessages(),
$this->pseudoTransactionProcessor->getMessages(),
], $count);
}
/**
* @param int $count
* @param int $count
*/
private function mergeWarnings(int $count): void
{
$one = $this->csvFileProcessor->getWarnings();
$two = $this->lineProcessor->getWarnings();
$three = $this->columnValueConverter->getWarnings();
$four = $this->pseudoTransactionProcessor->getWarnings();
$total = [];
for ($i = 0; $i < $count; $i++) {
$total[$i] = array_merge(
$one[$i] ?? [],
$two[$i] ?? [],
$three[$i] ?? [],
$four[$i] ?? [],
);
$this->allWarnings = $this->mergeArrays([
$this->csvFileProcessor->getWarnings(),
$this->lineProcessor->getWarnings(),
$this->columnValueConverter->getWarnings(),
$this->pseudoTransactionProcessor->getWarnings(),
], $count);
}
/**
* @param array $collection
* @param int $count
*
* @return array
*/
private function mergeArrays(array $collection, int $count): array
{
$return = [];
foreach ($collection as $set) {
if (0 === count($set)) {
continue;
}
for ($i = 0; $i < $count; $i++) {
if (array_key_exists($i, $set)) {
$return[$i] = array_key_exists($i, $return) ? $return[$i] : [];
$return[$i] = array_merge($return[$i], $set[$i]);
}
}
}
$this->allWarnings = $total;
// sanity check (should not be necessary)
foreach ($return as $index => $set) {
if (0 === count($set)) {
unset($return[$index]);
}
}
if (0 === count($return)) {
$return = [];
}
return $return;
}
}
Write Preview
Loading…
Cancel
Save