Mirrors
/
data-importer
mirror of https://github.com/firefly-iii/data-importer.git
3
0
Fork 0
Code Releases Activity
Browse Source

Happy flow works for Nordigen.

pull/1/head
James Cole 4 years ago
parent
b311d955d2
commit
c7c86469a7
7 changed files with 246 additions and 52 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      app/Http/Controllers/Import/ConfigurationController.php
  2. 181
      app/Http/Controllers/Import/MapController.php
  3. 3
      app/Http/Controllers/Import/SubmitController.php
  4. 44
      app/Http/Middleware/IsReadyForStep.php
  5. 1
      app/Services/Session/Constants.php
  6. 60
      app/Services/Shared/Import/Routine/ApiSubmitter.php
  7. 4
      resources/views/import/007-convert/index.twig

5
app/Http/Controllers/Import/ConfigurationController.php
View File

@ -195,7 +195,10 @@ class ConfigurationController extends Controller
// set config as complete.
session()->put(Constants::CONFIG_COMPLETE_INDICATOR, true);
if('nordigen' === $configuration->getFlow()) {
// at this point, nordigen is ready for data conversion.
session()->put(Constants::READY_FOR_CONVERSION, true);
}
// always redirect to roles, even if this isn't the step yet
// for nordigen and spectre, roles will be skipped right away.
return redirect(route('005-roles.index'));

181
app/Http/Controllers/Import/MapController.php
View File

@ -25,6 +25,7 @@ declare(strict_types=1);
namespace App\Http\Controllers\Import;
use App\Exceptions\ImporterErrorException;
use App\Http\Controllers\Controller;
use App\Http\Middleware\MapControllerMiddleware;
use App\Services\CSV\Configuration\Configuration;
@ -36,8 +37,10 @@ use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Contracts\View\Factory;
use Illuminate\Http\RedirectResponse;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Storage;
use Illuminate\View\View;
use InvalidArgumentException;
use JsonException;
use League\Csv\Exception;
use Log;
@ -46,6 +49,7 @@ use Log;
*/
class MapController extends Controller
{
protected const DISK_NAME = 'jobs';
/**
* RoleController constructor.
@ -66,11 +70,13 @@ class MapController extends Controller
public function index()
{
$mainTitle = 'Map data';
$subTitle = 'Map values in CSV file to actual data in Firefly III';
$subTitle = 'Map values in file to actual data in Firefly III';
$data = [];
$roles = [];
Log::debug('Now in mapController index');
// get configuration object.
$configuration = Configuration::fromArray(session()->get(Constants::CONFIGURATION));
$configuration = Configuration::fromArray(session()->get(Constants::CONFIGURATION) ?? []);
// the config in the session will miss important values, we must get those from disk:
// 'mapping', 'do_mapping', 'roles' are missing.
@ -84,63 +90,97 @@ class MapController extends Controller
$configuration->setRoles($diskConfig->getRoles());
}
// then we can use them:
$roles = $configuration->getRoles();
$existingMapping = $configuration->getMapping();
$doMapping = $configuration->getDoMapping();
$data = [];
foreach ($roles as $index => $role) {
$info = config('csv.import_roles')[$role] ?? null;
$mappable = $info['mappable'] ?? false;
if (null === $info) {
continue;
}
if (false === $mappable) {
continue;
}
$mapColumn = $doMapping[$index] ?? false;
if (false === $mapColumn) {
continue;
// depends on flow how to handle mapping
if ('csv' === $configuration->getFlow()) {
// then we can use them:
$roles = $configuration->getRoles();
$existingMapping = $configuration->getMapping();
$doMapping = $configuration->getDoMapping();
$data = [];
foreach ($roles as $index => $role) {
$info = config('csv.import_roles')[$role] ?? null;
$mappable = $info['mappable'] ?? false;
if (null === $info) {
continue;
}
if (false === $mappable) {
continue;
}
$mapColumn = $doMapping[$index] ?? false;
if (false === $mapColumn) {
continue;
}
Log::debug(sprintf('Mappable role is "%s"', $role));
$info['role'] = $role;
$info['values'] = [];
// create the "mapper" class which will get data from Firefly III.
$class = sprintf('App\\Services\\CSV\\Mapper\\%s', $info['mapper']);
if (!class_exists($class)) {
throw new InvalidArgumentException(sprintf('Class %s does not exist.', $class));
}
Log::debug(sprintf('Associated class is %s', $class));
/** @var MapperInterface $object */
$object = app($class);
$info['mapping_data'] = $object->getMap();
$info['mapped'] = $existingMapping[$index] ?? [];
Log::debug(sprintf('Mapping data length is %d', count($info['mapping_data'])));
$data[$index] = $info;
}
Log::debug(sprintf('Mappable role is "%s"', $role));
$info['role'] = $role;
$info['values'] = [];
// get columns from file
$content = StorageService::getContent(session()->get(Constants::UPLOAD_CSV_FILE));
$delimiter = (string) config(sprintf('csv.delimiters.%s', $configuration->getDelimiter()));
$data = MapperService::getMapData($content, $delimiter, $configuration->isHeaders(), $configuration->getSpecifics(), $data);
}
/*
* To map Nordigen, pretend the file has one "column" (this is based on the CSV importer after all)
* that contains:
* - opposing account names (this is preordained).
*/
if ('nordigen' === $configuration->getFlow()) {
$roles = [];
$data = [];
// index 0, opposing account name:
$index = 0;
$opposingName = config('csv.import_roles.opposing-name') ?? null;
$opposingName['role'] = 'opposing-name';
$opposingName['values'] = $this->getOpposingAccounts();
// create the "mapper" class which will get data from Firefly III.
$class = sprintf('App\\Services\\CSV\\Mapper\\%s', $info['mapper']);
$class = sprintf('App\\Services\\CSV\\Mapper\\%s', $opposingName['mapper']);
if (!class_exists($class)) {
throw new InvalidArgumentException(sprintf('Class %s does not exist.', $class));
}
Log::debug(sprintf('Associated class is %s', $class));
/** @var MapperInterface $object */
$object = app($class);
$info['mapping_data'] = $object->getMap();
$info['mapped'] = $existingMapping[$index] ?? [];
Log::debug(sprintf('Mapping data length is %d', count($info['mapping_data'])));
$data[$index] = $info;
$object = app($class);
$opposingName['mapping_data'] = $object->getMap();
$opposingName['mapped'] = $existingMapping[$index] ?? [];
$data[] = $opposingName;
}
if ('spectre' === $configuration->getFlow()) {
die('spectre');
}
// if nothing to map, just set mappable to true and go to the next step:
if (0 === count($data)) {
// set map config as complete.
//session()->put(Constants::MAPPING_COMPLETE_INDICATOR, true);
session()->put(Constants::READY_FOR_CONVERSION, true);
session()->put(Constants::MAPPING_COMPLETE_INDICATOR, true);
return redirect()->route('007-convert.index');
}
// get columns from file
$content = StorageService::getContent(session()->get(Constants::UPLOAD_CSV_FILE));
$delimiter = (string) config(sprintf('csv.delimiters.%s', $configuration->getDelimiter()));
$data = MapperService::getMapData($content, $delimiter, $configuration->isHeaders(), $configuration->getSpecifics(), $data);
return view('import.006-mapping.index', compact('mainTitle', 'subTitle', 'roles', 'data'));
}
@ -183,9 +223,14 @@ class MapController extends Controller
// at this point the $data array must be merged with the mapping as it is on the disk,
// and then saved to disk once again in a new config file.
$diskArray = json_decode(StorageService::getContent(session()->get(Constants::UPLOAD_CONFIG_FILE)), true, JSON_THROW_ON_ERROR);
$diskConfig = Configuration::fromArray($diskArray);
$originalMapping = $diskConfig->getMapping();
$configFileName = session()->get(Constants::UPLOAD_CONFIG_FILE);
$originalMapping = [];
$diskConfig = null;
if (null !== $configFileName) {
$diskArray = json_decode(StorageService::getContent($configFileName), true, JSON_THROW_ON_ERROR);
$diskConfig = Configuration::fromArray($diskArray);
$originalMapping = $diskConfig->getMapping();
}
// loop $data and save values:
$mergedMapping = $this->mergeMapping($originalMapping, $data);
@ -198,8 +243,10 @@ class MapController extends Controller
// since the configuration saved in the session will omit 'mapping', 'do_mapping' and 'roles'
// these must be set to the configuration file
// no need to do this sooner because toSessionArray would have dropped them anyway.
$configuration->setRoles($diskConfig->getRoles());
$configuration->setDoMapping($diskConfig->getDoMapping());
if (null !== $diskConfig) {
$configuration->setRoles($diskConfig->getRoles());
$configuration->setDoMapping($diskConfig->getDoMapping());
}
// then save entire thing to a new disk file:
$configFileName = StorageService::storeArray($configuration->toArray());
@ -212,6 +259,10 @@ class MapController extends Controller
// set map config as complete.
session()->put(Constants::MAPPING_COMPLETE_INDICATOR, true);
session()->put(Constants::READY_FOR_CONVERSION, true);
if('nordigen' === $configuration->getFlow()) {
// if nordigen, now ready for submission!
session()->put(Constants::READY_FOR_SUBMISSION, true);
}
return redirect()->route('007-convert.index');
}
@ -239,4 +290,46 @@ class MapController extends Controller
// original has been updated:
return $original;
}
/**
* @return array
* @throws FileNotFoundException
* @throws ImporterErrorException
* @throws \Psr\Container\ContainerExceptionInterface
* @throws \Psr\Container\NotFoundExceptionInterface
* TODO move to helper or something
*/
private function getOpposingAccounts(): array
{
Log::debug(sprintf('Now in %s', __METHOD__));
$downloadIdentifier = session()->get(Constants::CONVERSION_JOB_IDENTIFIER);
$disk = Storage::disk(self::DISK_NAME);
$json = $disk->get(sprintf('%s.json', $downloadIdentifier));
try {
$array = json_decode($json, true, 512, JSON_THROW_ON_ERROR);
} catch (JsonException $e) {
throw new ImporterErrorException(sprintf('Could not decode download: %s', $e->getMessage()), 0, $e);
}
$opposing = [];
$total = count($array);
/** @var array $transaction */
foreach ($array as $index => $transaction) {
Log::debug(sprintf('[%s/%s] Parsing transaction', ($index + 1), $total));
/** @var array $row */
foreach ($transaction['transactions'] as $row) {
$opposing[] = (string) array_key_exists('destination_name', $row) ? $row['destination_name'] : '';
$opposing[] = (string) array_key_exists('source_name', $row) ? $row['source_name'] : '';
}
}
$filtered = array_filter(
$opposing,
static function (string $value) {
return '' !== $value;
}
);
return array_unique($filtered);
}
}

3
app/Http/Controllers/Import/SubmitController.php
View File

@ -52,6 +52,7 @@ class SubmitController extends Controller
public function __construct()
{
parent::__construct();
view()->share('pageTitle','Submit data to Firefly III');
$this->middleware(SubmitControllerMiddleware::class);
}
@ -66,7 +67,7 @@ class SubmitController extends Controller
$mainTitle = 'Submit the data';
// get configuration object.
$configuration = Configuration::fromArray(session()->get(Constants::CONFIGURATION));
$configuration = Configuration::fromArray(session()->get(Constants::CONFIGURATION) ?? []);
// append info from the file on disk:
$configFileName = session()->get(Constants::UPLOAD_CONFIG_FILE);
if (null !== $configFileName) {

44
app/Http/Middleware/IsReadyForStep.php
View File

@ -123,9 +123,25 @@ trait IsReadyForStep
}
return false;
case 'map':
// mapping must be complete, or not ready for this step.
if (session()->has(Constants::MAPPING_COMPLETE_INDICATOR) && true === session()->get(Constants::MAPPING_COMPLETE_INDICATOR)) {
Log::debug('Return false, not ready for step [1].');
return false;
}
// conversion complete?
if (session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) && true === session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)) {
Log::debug('Return true, ready for step [4].');
return true;
}
// must already have the conversion, or not ready for this step:
if (session()->has(Constants::READY_FOR_CONVERSION) && true === session()->get(Constants::READY_FOR_CONVERSION)) {
Log::debug('Return false, not yet ready for step [2].');
return false;
}
// otherwise return false.
Log::debug('Return true, ready for step [3].');
return true;
case 'nordigen-link':
// must have upload, thats it
@ -134,10 +150,15 @@ trait IsReadyForStep
}
return false;
case 'conversion':
if (session()->has(Constants::READY_FOR_SUBMISSION) && true === session()->get(Constants::READY_FOR_SUBMISSION)) {
Log::debug('Return false, ready for submission.');
return false;
}
// if/else is in reverse!
if (session()->has(Constants::READY_FOR_CONVERSION) && true === session()->get(Constants::READY_FOR_CONVERSION)) {
return true;
}
// will probably never return false, but OK.
return false;
case 'configuration':
@ -263,9 +284,32 @@ trait IsReadyForStep
Log::debug(sprintf('Return redirect to "%s"', $route));
return redirect($route);
case 'define-roles':
// will always push to mapping, and mapping will send them to
// the right step.
$route = route('006-mapping.index');
Log::debug(sprintf('Return redirect to "%s"', $route));
return redirect($route);
case 'map':
// if no conversion yet, go there first
// must already have the conversion, or not ready for this step:
if (session()->has(Constants::READY_FOR_CONVERSION) && true === session()->get(Constants::READY_FOR_CONVERSION)) {
Log::debug('Is ready for conversion, so send to conversion.');
$route = route('007-convert.index');
Log::debug(sprintf('Return redirect to "%s"', $route));
return redirect($route);
}
Log::debug('Is ready for submit.');
// otherwise go to import right away
$route = route('008-submit.index');
Log::debug(sprintf('Return redirect to "%s"', $route));
return redirect($route);
case 'conversion':
if (session()->has(Constants::READY_FOR_SUBMISSION) && true === session()->get(Constants::READY_FOR_SUBMISSION)) {
$route = route('008-submit.index');
Log::debug(sprintf('Return redirect to "%s"', $route));
return redirect($route);
}
throw new ImporterErrorException(sprintf('redirectToCorrectNordigenStep: Cannot handle Nordigen step "%s" [1]', self::STEP));
}
}

1
app/Services/Session/Constants.php
View File

@ -60,6 +60,7 @@ class Constants
public const CONVERSION_COMPLETE_INDICATOR = 'conversion_complete';
public const SUBMISSION_COMPLETE_INDICATOR = 'submission_complete';
public const SELECTED_BANK_COUNTRY = 'selected_bank_country';
public const READY_FOR_SUBMISSION = 'ready_for_submission';
// nordigen specific constants
public const REQUISITION_REFERENCE = 'requisition_reference';

60
app/Services/Shared/Import/Routine/ApiSubmitter.php
View File

@ -52,6 +52,7 @@ class ApiSubmitter
private bool $addTag;
private string $vanityURL;
private Configuration $configuration;
private array $mapping;
/**
* @param array $lines
@ -76,20 +77,21 @@ class ApiSubmitter
* @var array $line
*/
foreach ($lines as $index => $line) {
Log::debug(sprintf('Now submitting transaction %d/%d', ($index+1), $count));
Log::debug(sprintf('Now submitting transaction %d/%d', ($index + 1), $count));
// first do local duplicate transaction check (the "cell" method):
$unique = $this->uniqueTransaction($index, $line);
if (true === $unique) {
Log::debug(sprintf('Transaction #%d is unique.', $index+1));
Log::debug(sprintf('Transaction #%d is unique.', $index + 1));
$groupInfo = $this->processTransaction($index, $line);
$this->addTagToGroups($groupInfo);
}
if(false === $unique) {
Log::debug(sprintf('Transaction #%d is NOT unique.', $index+1));
if (false === $unique) {
Log::debug(sprintf('Transaction #%d is NOT unique.', $index + 1));
}
}
Log::info(sprintf('Done submitting %d transactions to your Firefly III instance.', $count));
}
/**
*
*/
@ -225,6 +227,7 @@ class ApiSubmitter
*/
private function processTransaction(int $index, array $line): array
{
$line = $this->replaceMappings($line);
$return = [];
$url = Token::getURL();
$token = Token::getAccessToken();
@ -408,7 +411,17 @@ class ApiSubmitter
{
$this->configuration = $configuration;
$this->setAddTag($configuration->isAddImportTag());
$this->setMapping($configuration->getMapping());
}
/**
* @param array $mapping
*/
public function setMapping(array $mapping): void
{
$this->mapping = $mapping;
}
/**
* @param bool $addTag
*/
@ -416,4 +429,43 @@ class ApiSubmitter
{
$this->addTag = $addTag;
}
/**
* @param array $line
* @return array
*/
private function replaceMappings(array $line): array
{
Log::debug('Going to map data for this line.');
if (array_key_exists(0, $this->mapping)) {
Log::debug('Configuration has mapping for opposing account name!');
/**
* @var int $index
* @var array $transaction
*/
foreach ($line['transactions'] as $index => $transaction) {
if ('withdrawal' === $transaction['type']) {
// replace destination_name with destination_id
$destination = $transaction['destination_name'] ?? '';
if (array_key_exists($destination, $this->mapping[0])) {
unset($line['transactions'][$index]['destination_name']);
unset($line['transactions'][$index]['destination_iban']);
$line['transactions'][$index]['destination_id'] = $this->mapping[0][$destination];
Log::debug(sprintf('Replaced destination name "%s" with a reference to account id #%d', $destination, $this->mapping[0][$destination]));
}
}
if ('deposit' === $transaction['type']) {
// replace source_name with source_id
$source = $transaction['source_name'] ?? '';
if (array_key_exists($source, $this->mapping[0])) {
unset($line['transactions'][$index]['source_name']);
unset($line['transactions'][$index]['source_iban']);
$line['transactions'][$index]['source_id'] = $this->mapping[0][$source];
Log::debug(sprintf('Replaced source name "%s" with a reference to account id #%d', $source, $this->mapping[0][$source]));
}
}
}
}
return $line;
}
}

4
resources/views/import/007-convert/index.twig
View File

@ -6,8 +6,8 @@
 
</div>
</div>
<div class="row">
<div class="col-lg-12">
<div class="row mt-3">
<div class="col-lg-10 offset-lg-1">
<h1>{{ mainTitle }}</h1>
</div>
</div>

Write Preview
Loading…
Cancel
Save