Mirrors
/
data-importer
mirror of https://github.com/firefly-iii/data-importer.git
3
0
Fork 0
Code Releases Activity
Browse Source

Roles now come from config, also add mapping capability.

pull/373/head
James Cole 3 years ago
parent
9e84576798
commit
70d718f7c6
5 changed files with 456 additions and 30 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 51
      app/Http/Controllers/Import/File/RoleController.php
  2. 72
      app/Http/Controllers/Import/MapController.php
  3. 90
      app/Services/CSV/Mapper/MapperService.php
  4. 3
      app/Services/Camt/Transaction.php
  5. 270
      config/camt.php

51
app/Http/Controllers/Import/File/RoleController.php
View File

@ -249,6 +249,7 @@ class RoleController extends Controller
'title' => trans('camt.level_A'),
'explanation' => trans('camt.explain_A'),
'fields' => [
// TODO replace me with info from config.
[
'section' => false,
'title' => 'messageId',
@ -263,6 +264,7 @@ class RoleController extends Controller
'title' => trans('camt.level_B'),
'explanation' => trans('camt.explain_B'),
'fields' => [
// TODO replace me with info from config.
[
'section' => false,
'title' => 'statementCreationDate',
@ -295,6 +297,7 @@ class RoleController extends Controller
'title' => trans('camt.level_C'),
'explanation' => trans('camt.explain_C'),
'fields' => [
// TODO replace me with info from config.
[
'section' => false,
'title' => 'entryDate',
@ -333,6 +336,7 @@ class RoleController extends Controller
'title' => 'transaction',
],
// entryAmount
// TODO replace me with info from config.
[
'section' => false,
'title' => 'entryAmount',
@ -407,32 +411,32 @@ class RoleController extends Controller
// entryDetailOpposingAccountIban
[
'section' => false,
'title' => 'entryDetailOpposingAccountIban',
'selected_role' => $roles['entryDetailOpposingAccountIban'] ?? 'opposing-iban',
'title' => 'entryOpposingAccountIban',
'selected_role' => $roles['entryOpposingAccountIban'] ?? 'opposing-iban',
'roles' => config('camt.roles.iban'),
'mappable' => true,
'do_mapping' => $doMapping['entryDetailOpposingAccountIban'] ?? false,
'do_mapping' => $doMapping['entryOpposingAccountIban'] ?? false,
'example_data' => $examples['entryDetailOpposingAccountIban'],
],
// entryDetailOpposingAccountNumber
[
'section' => false,
'title' => 'entryDetailOpposingAccountNumber',
'selected_role' => $roles['entryDetailOpposingAccountNumber'] ?? 'opposing-number',
'title' => 'entryOpposingAccountNumber',
'selected_role' => $roles['entryOpposingAccountNumber'] ?? 'opposing-number',
'roles' => config('camt.roles.account_number'),
'mappable' => true,
'do_mapping' => $doMapping['entryDetailOpposingAccountNumber'] ?? false,
'example_data' => $examples['entryDetailOpposingAccountNumber'],
'do_mapping' => $doMapping['entryOpposingAccountNumber'] ?? false,
'example_data' => $examples['entryOpposingAccountNumber'],
],
// entryDetailOpposingName
[
'section' => false,
'title' => 'entryDetailOpposingName',
'selected_role' => $roles['entryDetailOpposingName'] ?? 'opposing-name',
'title' => 'entryOpposingName',
'selected_role' => $roles['entryOpposingName'] ?? 'opposing-name',
'roles' => config('camt.roles.account_name'),
'mappable' => true,
'do_mapping' => $doMapping['entryDetailOpposingName'] ?? false,
'example_data' => $examples['entryDetailOpposingName'],
'do_mapping' => $doMapping['entryOpposingName'] ?? false,
'example_data' => $examples['entryOpposingName'],
],
@ -442,6 +446,7 @@ class RoleController extends Controller
'title' => trans('camt.level_D'),
'explanation' => trans('camt.explain_D'),
'fields' => [
// TODO replace me with info from config.
// entryDetailAccountServicerReference
[
'section' => false,
@ -500,32 +505,32 @@ class RoleController extends Controller
'section' => true,
'title' => 'Btc',
],
// entryBtcDomainCode
// entryDetailBtcDomainCode
[
'section' => false,
'title' => 'entryBtcDomainCode',
'selected_role' => $roles['entryBtcDomainCode'] ?? 'note',
'title' => 'entryDetailBtcDomainCode',
'selected_role' => $roles['entryDetailBtcDomainCode'] ?? 'note',
'roles' => config('camt.roles.meta'),
'mappable' => false,
'example_data' => $examples['entryBtcDomainCode'],
'example_data' => $examples['entryDetailBtcDomainCode'],
],
// entryBtcFamilyCode
// entryDetailBtcFamilyCode
[
'section' => false,
'title' => 'entryBtcFamilyCode',
'selected_role' => $roles['entryBtcFamilyCode'] ?? 'note',
'title' => 'entryDetailBtcFamilyCode',
'selected_role' => $roles['entryDetailBtcFamilyCode'] ?? 'note',
'roles' => config('camt.roles.meta'),
'mappable' => false,
'example_data' => $examples['entryBtcFamilyCode'],
'example_data' => $examples['entryDetailBtcFamilyCode'],
],
// entryBtcSubFamilyCode
// entryDetailBtcSubFamilyCode
[
'section' => false,
'title' => 'entryBtcSubFamilyCode',
'selected_role' => $roles['entryBtcSubFamilyCode'] ?? 'note',
'title' => 'entryDetailBtcSubFamilyCode',
'selected_role' => $roles['entryDetailBtcSubFamilyCode'] ?? 'note',
'roles' => config('camt.roles.meta'),
'mappable' => false,
'example_data' => $examples['entryBtcSubFamilyCode'],
'example_data' => $examples['entryDetailBtcSubFamilyCode'],
],
// section_opposingPart

72
app/Http/Controllers/Import/MapController.php
View File

@ -77,11 +77,16 @@ class MapController extends Controller
$data = [];
$roles = [];
if ('file' === $configuration->getFlow()) {
app('log')->debug('Get mapping data for importable file');
if ('file' === $configuration->getFlow() && 'csv' === $configuration->getContentType()) {
app('log')->debug('Get mapping data for CSV file');
$roles = $configuration->getRoles();
$data = $this->getCSVMapInformation();
}
if ('file' === $configuration->getFlow() && 'camt' === $configuration->getContentType()) {
app('log')->debug('Get mapping data for CAMT file');
$roles = $configuration->getRoles();
$data = $this->getCamtMapInformation();
}
// nordigen, spectre and others:
if ('file' !== $configuration->getFlow()) {
@ -170,6 +175,69 @@ class MapController extends Controller
return MapperService::getMapData($content, $delimiter, $configuration->isHeaders(), $configuration->getSpecifics(), $data);
}
/**
* Return the map data necessary for the importable file mapping based on some weird helpers.
* TODO needs refactoring and proper splitting into helpers.
*
* @return array
* @throws ContainerExceptionInterface
* @throws ImporterErrorException
* @throws NotFoundExceptionInterface
*/
private function getCamtMapInformation(): array
{
$configuration = $this->restoreConfiguration();
$roles = $configuration->getRoles();
$existingMapping = $configuration->getMapping();
$doMapping = $configuration->getDoMapping();
$data = [];
foreach ($roles as $index => $role) {
$info = config('camt.all_roles')[$role] ?? null;
$mappable = $info['mappable'] ?? false;
if (null === $info) {
app('log')->warning(sprintf('Role "%s" does not exist.', $role));
continue;
}
if (false === $mappable) {
app('log')->warning(sprintf('Role "%s" cannot be mapped.', $role));
continue;
}
$mapColumn = $doMapping[$index] ?? false;
if (false === $mapColumn) {
app('log')->warning(sprintf('Role "%s" does not have to be mapped.', $role));
continue;
}
app('log')->debug(sprintf('Mappable role is "%s"', $role));
$info['role'] = $role;
$info['values'] = [];
// create the "mapper" class which will get data from Firefly III.
$class = sprintf('App\\Services\\CSV\\Mapper\\%s', $info['mapper']);
if (!class_exists($class)) {
throw new InvalidArgumentException(sprintf('Class %s does not exist.', $class));
}
app('log')->debug(sprintf('Associated class is %s', $class));
/** @var MapperInterface $object */
$object = app($class);
$info['mapping_data'] = $object->getMap();
$info['mapped'] = $existingMapping[$index] ?? [];
app('log')->debug(sprintf('Mapping data length is %d', count($info['mapping_data'])));
$data[$index] = $info;
}
// get columns from file
$content = StorageService::getContent(session()->get(Constants::UPLOAD_DATA_FILE), $configuration->isConversion());
return MapperService::getMapDataForCamt($configuration, $content, $data);
}
/**
* Weird bunch of code to return info on Spectre and Nordigen.
*

90
app/Services/CSV/Mapper/MapperService.php
View File

@ -25,6 +25,14 @@ declare(strict_types=1);
namespace App\Services\CSV\Mapper;
use App\Exceptions\ImporterErrorException;
use App\Services\Camt\Transaction;
use App\Services\Session\Constants;
use App\Services\Shared\Configuration\Configuration;
use App\Services\Storage\StorageService;
use Genkgo\Camt\Camt053\DTO\Statement as CamtStatement;
use Genkgo\Camt\Config;
use Genkgo\Camt\DTO\Entry;
use Genkgo\Camt\Reader as CamtReader;
use League\Csv\Exception;
use League\Csv\Reader;
use League\Csv\Statement;
@ -36,6 +44,7 @@ class MapperService
{
/**
* Appends the given array with data from the CSV file in the config.
* TODO remove reference to specifics.
*
* @param string $content
* @param string $delimiter
@ -115,4 +124,85 @@ class MapperService
return $data;
}
/**
* Appends the given array with data from the CAMT file in the config.
*
* @param string $content
* @param string $delimiter
* @param bool $hasHeaders
* @param array $specifics
* @param array $data
*
* @return array
* @throws ImporterErrorException
*/
public static function getMapDataForCamt(Configuration $configuration, string $content, array $data): array
{
app('log')->debug('Now in getMapDataForCamt');
// make file reader first.
$camtReader = new CamtReader(Config::getDefault());
$camtMessage = $camtReader->readString($content);
$transactions = [];
// loop over records.
$statements = $camtMessage->getRecords();
/** @var CamtStatement $statement */
foreach ($statements as $statement) { // -> Level B
$entries = $statement->getEntries();
/** @var Entry $entry */
foreach ($entries as $entry) { // -> Level C
$count = count($entry->getTransactionDetails()); // count level D entries.
if (0 === $count) {
// TODO Create a single transaction, I guess?
$transactions[] = new Transaction($configuration, $camtMessage, $statement, $entry);
}
if (0 !== $count) {
foreach ($entry->getTransactionDetails() as $detail) {
$transactions[] = new Transaction($configuration, $camtMessage, $statement, $entry, $detail);
}
}
}
}
$mappableFields = self::getMappableFieldsForCamt();
/** @var Transaction $transaction */
foreach ($transactions as $transaction) {
// take all mappable fields from this transaction, and add to $values in the data thing
foreach(array_keys($mappableFields) as $title) {
$data[$title]['values'][] = $transaction->getField($title);
}
}
// make all values unique for mapping and remove empty vars.
foreach($data as $title => $info) {
$filtered = array_filter(
$info['values'],
static function (string $value) {
return '' !== $value;
}
);
$info['values'] = array_unique($filtered);
sort($info['values']);
$data[$title]['values'] = $info['values'];
}
return $data;
}
/**
* @return array
*/
private static function getMappableFieldsForCamt(): array
{
$fields = config('camt.fields');
$return = [];
/** @var array $field */
foreach($fields as $name => $field) {
if($field['mappable']) {
$return[$name] = $field;
}
}
return $return;
}
}

3
app/Services/Camt/Transaction.php
View File

@ -107,6 +107,7 @@ class Transaction
return (string)$this->levelC->getBankTransactionCode()->getDomain()->getFamily()->getCode();
case 'entryBtcSubFamilyCode':
return (string)$this->levelC->getBankTransactionCode()->getDomain()->getFamily()->getSubFamilyCode();
case 'entryOpposingAccountIban':
case 'entryDetailOpposingAccountIban':
$ret = '';
if ($account = $this->relatedOppositeParty?->getAccount()) {
@ -116,6 +117,7 @@ class Transaction
}
return $ret;
case 'entryOpposingAccountNumber':
case 'entryDetailOpposingAccountNumber':
$ret = '';
if ($account = $this->relatedOppositeParty?->getAccount()) {
@ -130,6 +132,7 @@ class Transaction
}
return $ret;
case 'entryOpposingName':
case 'entryDetailOpposingName':
return '';
// TODO this code doesnt work yet.

270
config/camt.php
View File

@ -22,10 +22,7 @@
declare(strict_types=1);
// TODO -> clean up, remove CSV parts... add missing parts
// to prevent duplicates first define some roles?
// all roles available for all CAMT data fields.
$availableRoles = [
'_ignore' => [
'mappable' => false,
@ -239,7 +236,19 @@ $availableRoles = [
return [
// roles for the header:
/*
* Roles are divided into a number of groups,
* i.e. the "level_a" group and the "dates" group etc.
*
* For each field in the CAMT file, it has been specified which of these groups apply.
* This particular config can be found further ahead.
*
* Extra groups of roles can be created here, or existing groups extended.
* For example, if you wish extract a field called "due date" from the CAMT file,
* you could use existing group "dates" but you are free to make a new group called "due_date" with only one option.
*
* Make sure all groups also have the _ignore role as first option.
*/
'roles' => [
'level_a' => [
'_ignore' => $availableRoles['_ignore'],
@ -298,6 +307,257 @@ return [
'foreign-currency-code' => $availableRoles['foreign-currency-code'],
],
],
/*
* This particular config variable holds all possible roles.
*/
'all_roles' => $availableRoles,
/*
* This array denotes all fields that can be extracted from a CAMT file and the necessary
* configuration:
* TODO this array needs to be expanded based on the content of RoleController.php:247
*/
'fields' => [
// level A
'messageId' => [
'title' => 'messageId',
'roles' => 'level_a', // this is a reference to the role groups above.
'mappable' => false,
'default_role' => 'note',
'level' => 'A',
],
// level B, Statement
'statementCreationDate' => [
'title' => 'statementCreationDate',
'roles' => 'dates',
'mappable' => false,
'default_role' => 'date_process',
'level' => 'B',
],
'statementAccountIban' => [
'title' => 'statementAccountIban',
'roles' => 'iban',
'mappable' => true,
'default_role' => 'account-iban',
'level' => 'B',
],
'statementAccountNumber' => [
'title' => 'statementAccountNumber',
'roles' => 'account_number',
'mappable' => true,
'default_role' => 'account-number',
'level' => 'B',
],
'entryDate' => [
'section' => false,
'title' => 'entryDate',
'default_role' => 'date_transaction',
'roles' => 'dates',
'mappable' => false,
'level' => 'C',
],
// level C, Entry
'entryAccountServicerReference' => [
'section' => false,
'title' => 'entryAccountServicerReference',
'default_role' => 'external-id',
'roles' => 'meta',
'mappable' => false,
'level' => 'C',
],
'entryReference' => [
'section' => false,
'title' => 'entryReference',
'default_role' => 'internal_reference',
'roles' => 'meta',
'mappable' => false,
'level' => 'C',
],
'entryAdditionalInfo' => [
'section' => false,
'title' => 'entryAdditionalInfo',
'default_role' => 'note',
'roles' => 'meta',
'mappable' => false,
'level' => 'C',
],
'entryAmount' => [
'section' => false,
'title' => 'entryAmount',
'default_role' => 'amount',
'roles' => 'amount',
'mappable' => false,
'level' => 'C',
],
'entryAmountCurrency' =>
[
'title' => 'entryAmountCurrency',
'default_role' => 'currency-code',
'roles' => 'currency',
'mappable' => true,
'level' => 'C',
],
'entryValueDate' =>
[
'title' => 'entryValueDate',
'default_role' => 'date_payment',
'roles' => 'dates',
'mappable' => false,
'level' => 'C',
],
'entryBookingDate' =>
[
'title' => 'entryBookingDate',
'default_role' => 'date_book',
'roles' => 'dates',
'mappable' => false,
'level' => 'C',
],
'entryBtcDomainCode' =>
[
'title' => 'entryBtcDomainCode',
'default_role' => 'note',
'roles' => 'meta',
'mappable' => false,
'level' => 'C',
],
'entryBtcFamilyCode' =>
[
'title' => 'entryBtcFamilyCode',
'default_role' => 'note',
'roles' => 'meta',
'mappable' => false,
'level' => 'C',
],
'entryBtcSubFamilyCode' =>
[
'title' => 'entryBtcSubFamilyCode',
'default_role' => 'note',
'roles' => 'meta',
'mappable' => false,
'level' => 'C',
],
'entryOpposingAccountIban' =>
[
'title' => 'entryDetailOpposingAccountIban',
'default_role' => 'opposing-iban',
'roles' => 'iban',
'mappable' => true,
'level' => 'C',
],
'entryOpposingAccountNumber' =>
[
'title' => 'entryDetailOpposingAccountNumber',
'default_role' => 'opposing-number',
'roles' => 'account_number',
'mappable' => true,
'level' => 'C',
],
'entryOpposingName' =>
[
'title' => 'entryDetailOpposingName',
'default_role' => 'opposing-name',
'roles' => 'account_name',
'mappable' => true,
'level' => 'C',
],
// level D, entry detail
'entryDetailAccountServicerReference' =>
[
'title' => 'entryDetailAccountServicerReference',
'default_role' => 'note',
'roles' => 'meta',
'mappable' => false,
'level' => 'D',
],
'entryDetailRemittanceInformationUnstructuredBlockMessage' =>
[
'title' => 'entryDetailRemittanceInformationUnstructuredBlockMessage',
'default_role' => 'note',
'roles' => 'meta',
'mappable' => false,
'level' => 'D',
],
'entryDetailRemittanceInformationStructuredBlockAdditionalRemittanceInformation' =>
[
'title' => 'entryDetailRemittanceInformationStructuredBlockAdditionalRemittanceInformation',
'default_role' => 'note',
'roles' => 'meta',
'mappable' => false,
'level' => 'D',
],
'entryDetailAmount' =>
[
'title' => 'entryDetailAmount',
'default_role' => 'amount',
'roles' => 'amount',
'mappable' => false,
'level' => 'D',
],
'entryDetailAmountCurrency' =>
[
'title' => 'entryDetailAmountCurrency',
'default_role' => 'currency-code',
'roles' => 'currency',
'mappable' => true,
'level' => 'D',
],
'entryDetailBtcDomainCode' =>
[
'title' => 'entryBtcDomainCode',
'default_role' => 'note',
'roles' => 'meta',
'mappable' => false,
'level' => 'D',
],
'entryDetailBtcFamilyCode' =>
[
'title' => 'entryDetailBtcFamilyCode',
'default_role' => 'note',
'roles' => 'meta',
'mappable' => false,
'level' => 'D',
],
'entryDetailBtcSubFamilyCode' =>
[
'title' => 'entryDetailBtcSubFamilyCode',
'default_role' => 'note',
'roles' => 'meta',
'mappable' => false,
'level' => 'D',
],
'entryDetailOpposingAccountIban' =>
[
'title' => 'entryDetailOpposingAccountIban',
'default_role' => 'opposing-iban',
'roles' => 'iban',
'mappable' => true,
'level' => 'D',
],
'entryDetailOpposingAccountNumber' =>
[
'title' => 'entryDetailOpposingAccountNumber',
'default_role' => 'opposing-number',
'roles' => 'account_number',
'mappable' => true,
'level' => 'D',
],
'entryDetailOpposingName' =>
[
'title' => 'entryDetailOpposingName',
'default_role' => 'opposing-name',
'roles' => 'account_name',
'mappable' => true,
'level' => 'D',
],
],
// TODO remove me
'import_roles' => [

Write Preview
Loading…
Cancel
Save