Browse Source

First commit of SimpleFIN import by user 'skell'.

pull/838/head
James Cole 4 months ago
parent
commit
55aea07c59
  1. 6
      .env.example
  2. 139
      .linuxdev/development.md
  3. 14
      .linuxdev/laravel-dev.service
  4. 17
      .linuxdev/laravel-queue.service
  5. 28
      .linuxdev/start
  6. 27
      .linuxdev/start-queue
  7. 13
      .linuxdev/stop
  8. 21
      .linuxdev/stop-queue
  9. 8
      app/Http/Controllers/Import/AuthenticateController.php
  10. 355
      app/Http/Controllers/Import/ConfigurationController.php
  11. 109
      app/Http/Controllers/Import/ConversionController.php
  12. 128
      app/Http/Controllers/Import/DuplicateCheckController.php
  13. 113
      app/Http/Controllers/Import/MapController.php
  14. 60
      app/Http/Controllers/Import/SubmitController.php
  15. 261
      app/Http/Controllers/Import/UploadController.php
  16. 82
      app/Http/Controllers/IndexController.php
  17. 12
      app/Http/Controllers/NavController.php
  18. 316
      app/Http/Controllers/TokenController.php
  19. 38
      app/Http/Middleware/ConversionControllerMiddleware.php
  20. 479
      app/Http/Middleware/IsReadyForStep.php
  21. 236
      app/Http/Request/ConfigurationPostRequest.php
  22. 232
      app/Jobs/ProcessImportSubmissionJob.php
  23. 143
      app/Services/CSV/Mapper/ExpenseRevenueAccounts.php
  24. 6
      app/Services/Session/Constants.php
  25. 13
      app/Services/Shared/Authentication/SecretManager.php
  26. 41
      app/Services/Shared/Configuration/Configuration.php
  27. 372
      app/Services/Shared/Import/Routine/ApiSubmitter.php
  28. 38
      app/Services/Shared/Import/Status/SubmissionStatus.php
  29. 29
      app/Services/Shared/Import/Status/SubmissionStatusManager.php
  30. 42
      app/Services/Shared/Response/ResponseInterface.php
  31. 2
      app/Services/SimpleFIN/AuthenticationValidator.php
  32. 485
      app/Services/SimpleFIN/Conversion/AccountMapper.php
  33. 352
      app/Services/SimpleFIN/Conversion/RoutineManager.php
  34. 700
      app/Services/SimpleFIN/Conversion/TransactionTransformer.php
  35. 208
      app/Services/SimpleFIN/Model/Account.php
  36. 199
      app/Services/SimpleFIN/Model/Transaction.php
  37. 48
      app/Services/SimpleFIN/Request/AccountsRequest.php
  38. 72
      app/Services/SimpleFIN/Request/PostAccountRequest.php
  39. 159
      app/Services/SimpleFIN/Request/SimpleFINRequest.php
  40. 48
      app/Services/SimpleFIN/Request/TransactionsRequest.php
  41. 76
      app/Services/SimpleFIN/Response/AccountsResponse.php
  42. 47
      app/Services/SimpleFIN/Response/PostAccountResponse.php
  43. 107
      app/Services/SimpleFIN/Response/SimpleFINResponse.php
  44. 82
      app/Services/SimpleFIN/Response/TransactionsResponse.php
  45. 341
      app/Services/SimpleFIN/SimpleFINService.php
  46. 511
      app/Services/SimpleFIN/Validation/ConfigurationContractValidator.php
  47. 16
      app/Services/Spectre/Request/Request.php.rej
  48. 10
      app/Support/Http/CollectsAccounts.php
  49. 12
      app/Support/Http/CollectsAccounts.php.rej
  50. 86
      app/Support/Internal/CollectsAccounts.php
  51. 8
      config/importer.php
  52. 113
      config/simplefin.php
  53. 28
      resources/js/v2/src/pages/conversion/index.js
  54. 7
      resources/js/v2/src/pages/index/index.js
  55. 36
      resources/js/v2/src/pages/submit/index.js
  56. 612
      resources/views/v2/components/create-account-widget.blade.php
  57. 246
      resources/views/v2/components/firefly-iii-account-generic.blade.php
  58. 134
      resources/views/v2/components/importer-account-title.blade.php
  59. 25
      resources/views/v2/components/importer-account.blade.php
  60. 103
      resources/views/v2/import/003-upload/index.blade.php
  61. 353
      resources/views/v2/import/004-configure/index.blade.php
  62. 90
      resources/views/v2/import/007-convert/index.blade.php
  63. 28
      resources/views/v2/import/008-submit/index.blade.php
  64. 382
      resources/views/v2/layout/v2.blade.php
  65. 3
      routes/web.php
  66. 114
      tests/Feature/SimpleFIN/DemoModeTest.php
  67. 384
      tests/Unit/Services/SimpleFIN/Validation/ConfigurationContractValidatorTest.php

6
.env.example

@ -91,6 +91,12 @@ GOCARDLESS_GET_BALANCE_DETAILS=false
SPECTRE_APP_ID=
SPECTRE_SECRET=
# SimpleFIN Bridge Configuration
# Optional sfin key
SIMPLEFIN_BRIDGE_KEY=
# Optional CORS domain
SIMPLEFIN_BRIDGE_ORIGIN=
#
# Use cache. No need to do this.
#

139
.linuxdev/development.md

@ -0,0 +1,139 @@
# Data Importer Development Environment
## Quick Start
The data-importer uses Laravel's built-in development server. No Apache, no PHP-FPM, no SELinux complexity.
### Prerequisites
- PHP 8.4+ (already installed on trashcan)
- Composer (already installed)
- Git access to lil-debian deployment repository
### Start Development Server
#### Option 1: Systemd Service (Recommended)
```bash
./.linuxdev/start
```
#### Option 2: Manual
```bash
php artisan serve --host=127.0.0.1 --port=3000
```
Access at: `http://localhost:3000`
#### Service Management
```bash
# Stop the service
./.linuxdev/stop
# Follow Logs
journalctl --user -u laravel-dev-data-importer -f
# View last 50 Logs
journalctl --user -u laravel-dev-data-importer -n 50
# Check service status
systemctl --user status laravel-dev-data-importer
```
### Initial Setup (if needed)
```bash
# Install dependencies
composer install
# Generate application key (if missing)
php artisan key:generate
```
### Cache Control
```bash
# Clear all caches
php artisan config:clear
php artisan cache:clear
php artisan view:clear
php artisan route:clear
```
## Development Workflow
### Local Changes
1. Edit files in `~/Code/skellnet/src/data-importer/`
2. Changes are immediately reflected (no restart needed)
3. Check Laravel logs: `storage/logs/laravel.log`
### Deploy to Production
```bash
# Commit changes locally
git add .
git commit -m "Description of changes"
# Deploy to lil-debian
ssh root@lil-debian "cd /opt/data-importer && git pull"
ssh root@lil-debian "systemctl restart php8.4-fpm"
```
## Environment Configuration
### Local (.env)
- `APP_ENV=local`
- `APP_DEBUG=true` (for detailed error messages)
- Database connections point to development instances
### Production (lil-debian)
- `APP_ENV=production`
- `APP_DEBUG=false`
- Real database connections
- Nginx + PHP-FPM stack
## Debugging
### Application Errors
- Check `storage/logs/laravel.log` for detailed error traces
- Set `APP_DEBUG=true` in `.env` for browser error display
- Use `app('log')->debug('message', $data)` for custom logging
### Common Issues
- **500 errors**: Check Laravel log first, not web server logs
- **Permission issues**: Ensure `storage/` and `bootstrap/cache/` are writable
- **Missing dependencies**: Run `composer install`
- **Cache problems**: Clear all Laravel caches with artisan commands
## Frontend Assets
If frontend compilation is required:
```bash
npm install
npm run dev # for development
npm run build # for production
```
## Database
For local database development, configure connections in `.env`:
- Use local MySQL/PostgreSQL instances
- Or connect directly to development databases on the network
- Run migrations: `php artisan migrate`
## Testing
```bash
# Run PHP tests
php artisan test
# Run specific test file
php artisan test --filter SpecificTestClass
```
## Production Comparison
| Aspect | Development (trashcan) | Production (lil-debian) |
|--------|------------------------|-------------------------|
| Web Server | Laravel dev server | Nginx + PHP-FPM |
| Port | 3000 | 443 (HTTPS) |
| Debug | Enabled | Disabled |
| Logs | `storage/logs/` | `storage/logs/` + syslog |
| SSL | None | Full chain via acme.sh |
The development server handles PHP execution directly, eliminating the web server complexity needed in production.

14
.linuxdev/laravel-dev.service

@ -0,0 +1,14 @@
[Unit]
Description=Laravel Development Server
After=network.target
[Service]
Type=simple
ExecStart=/usr/bin/php artisan serve --host=127.0.0.1 --port=3000
WorkingDirectory=%i
Restart=on-failure
RestartSec=5
Environment=APP_ENV=local
Environment=APP_DEBUG=true
StandardOutput=journal
StandardError=journal

17
.linuxdev/laravel-queue.service

@ -0,0 +1,17 @@
[Unit]
Description=Laravel Queue Worker
After=network.target
[Service]
Type=simple
ExecStart=/usr/bin/php artisan queue:work --timeout=1800 --tries=1 --daemon
WorkingDirectory=%i
Restart=on-failure
RestartSec=10
Environment=APP_ENV=local
Environment=APP_DEBUG=true
StandardOutput=journal
StandardError=journal
[Install]
WantedBy=multi-user.target

28
.linuxdev/start

@ -0,0 +1,28 @@
#!/bin/bash
set -e
PWD=$(pwd)
SERVICE_NAME="laravel-dev-$(basename "$PWD")"
# Check if Laravel project
if [[ ! -f "artisan" ]]; then
echo "Error: No artisan file found. Run from Laravel project root."
exit 1
fi
# Stop existing service if running
systemctl --user stop "$SERVICE_NAME" 2>/dev/null || true
# Start service with current directory
systemd-run --user \
--unit="$SERVICE_NAME" \
--working-directory="$PWD" \
--setenv=APP_ENV=local \
--setenv=APP_DEBUG=true \
php artisan serve --host=127.0.0.1 --port=3000
echo "Laravel development server started as user service: $SERVICE_NAME"
echo "Access at: http://localhost:3000"
echo "Stop with: ./.linuxdev/stop"
echo "(Re)start with: ./.linuxdev/start"
echo "Logs with: journalctl --user -u $SERVICE_NAME -f"

27
.linuxdev/start-queue

@ -0,0 +1,27 @@
#!/bin/bash
set -e
PWD=$(pwd)
SERVICE_NAME="laravel-queue-$(basename "$PWD")"
# Check if Laravel project
if [[ ! -f "artisan" ]]; then
echo "Error: No artisan file found. Run from Laravel project root."
exit 1
fi
# Stop existing service if running
systemctl --user stop "$SERVICE_NAME" 2>/dev/null || true
# Start queue worker service with current directory
systemd-run --user \
--unit="$SERVICE_NAME" \
--working-directory="$PWD" \
--setenv=APP_ENV=local \
--setenv=APP_DEBUG=true \
php artisan queue:work --timeout=1800 --tries=1 --daemon
echo "Laravel queue worker started as user service: $SERVICE_NAME"
echo "Stop with: systemctl --user stop $SERVICE_NAME"
echo "Status with: systemctl --user status $SERVICE_NAME"
echo "Logs with: journalctl --user -u $SERVICE_NAME -f"

13
.linuxdev/stop

@ -0,0 +1,13 @@
#!/bin/bash
set -e
PWD=$(pwd)
SERVICE_NAME="laravel-dev-$(basename "$PWD")"
# Stop the service
if systemctl --user is-active --quiet "$SERVICE_NAME" 2>/dev/null; then
systemctl --user stop "$SERVICE_NAME"
echo "Stopped Laravel development server: $SERVICE_NAME"
else
echo "Laravel development server not running: $SERVICE_NAME"
fi

21
.linuxdev/stop-queue

@ -0,0 +1,21 @@
#!/bin/bash
set -e
PWD=$(pwd)
SERVICE_NAME="laravel-queue-$(basename "$PWD")"
# Check if Laravel project
if [[ ! -f "artisan" ]]; then
echo "Error: No artisan file found. Run from Laravel project root."
exit 1
fi
# Stop queue worker service
if systemctl --user stop "$SERVICE_NAME" 2>/dev/null; then
echo "Laravel queue worker stopped: $SERVICE_NAME"
else
echo "No queue worker service running or already stopped: $SERVICE_NAME"
fi
# Show final status
systemctl --user status "$SERVICE_NAME" --no-pager -l || true

8
app/Http/Controllers/Import/AuthenticateController.php

@ -96,8 +96,14 @@ class AuthenticateController extends Controller
return redirect(route('003-upload.index'));
}
}
if ('simplefin' === $flow) {
// This case should ideally be handled by middleware redirecting to upload.
// Adding explicit redirect here as a safeguard if middleware fails or is bypassed.
app('log')->warning('AuthenticateController reached for simplefin flow; middleware redirect might have failed. Redirecting to upload.');
return redirect(route('003-upload.index'));
}
throw new ImporterErrorException('Impossible flow exception [a].');
throw new ImporterErrorException(sprintf('Impossible flow exception. Unexpected flow "%s" encountered.', $flow ?? 'NULL'));
}
/**

355
app/Http/Controllers/Import/ConfigurationController.php

@ -28,12 +28,15 @@ namespace App\Http\Controllers\Import;
use App\Exceptions\AgreementExpiredException;
use App\Exceptions\ImporterErrorException;
use App\Http\Controllers\Controller;
use App\Services\SimpleFIN\Validation\ConfigurationContractValidator;
use App\Http\Middleware\ConfigurationControllerMiddleware;
use App\Http\Request\ConfigurationPostRequest;
use App\Services\CSV\Converter\Date;
use App\Services\Session\Constants;
use App\Services\Shared\Configuration\Configuration;
use App\Services\Shared\File\FileContentSherlock;
use App\Services\SimpleFIN\Conversion\AccountMapper;
use App\Services\CSV\Mapper\TransactionCurrencies;
use App\Services\Storage\StorageService;
use App\Support\Http\RestoresConfiguration;
use App\Support\Internal\CollectsAccounts;
@ -70,18 +73,24 @@ class ConfigurationController extends Controller
public function index(Request $request)
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
$mainTitle = 'Configuration';
$subTitle = 'Configure your import';
$flow = $request->cookie(Constants::FLOW_COOKIE); // TODO should be from configuration right
$configuration = $this->restoreConfiguration();
$mainTitle = 'Configuration';
$subTitle = 'Configure your import';
$flow = $request->cookie(Constants::FLOW_COOKIE); // TODO should be from configuration right
$configuration = $this->restoreConfiguration();
// if config says to skip it, skip it:
$overruleSkip = 'true' === $request->get('overruleskip');
$overruleSkip = 'true' === $request->get('overruleskip');
if (true === $configuration->isSkipForm() && false === $overruleSkip) {
app('log')->debug('Skip configuration, go straight to the next step.');
app('log')->debug(
'Skip configuration, go straight to the next step.'
);
// set config as complete.
session()->put(Constants::CONFIG_COMPLETE_INDICATOR, true);
if ('nordigen' === $configuration->getFlow() || 'spectre' === $configuration->getFlow()) {
if (
'nordigen' === $configuration->getFlow() ||
'spectre' === $configuration->getFlow()
) {
// at this point, nordigen is ready for data conversion.
session()->put(Constants::READY_FOR_CONVERSION, true);
}
@ -96,8 +105,8 @@ class ConfigurationController extends Controller
// possibilities for duplicate detection (unique columns)
// also get the nordigen / spectre accounts
$importerAccounts = [];
$uniqueColumns = config('csv.unique_column_options');
$importerAccounts = [];
$uniqueColumns = config('csv.unique_column_options');
if ('nordigen' === $flow) {
// TODO here we need to redirect to Nordigen.
try {
@ -109,44 +118,253 @@ class ConfigurationController extends Controller
$configuration->clearRequisitions();
// save configuration in session and on disk:
session()->put(Constants::CONFIGURATION, $configuration->toSessionArray());
$configFileName = StorageService::storeContent(json_encode($configuration->toArray(), JSON_PRETTY_PRINT));
session()->put(
Constants::CONFIGURATION,
$configuration->toSessionArray()
);
$configFileName = StorageService::storeContent(
json_encode($configuration->toArray(), JSON_PRETTY_PRINT)
);
session()->put(Constants::UPLOAD_CONFIG_FILE, $configFileName);
// redirect to selection.
return redirect()->route('009-selection.index');
}
$uniqueColumns = config('nordigen.unique_column_options');
$importerAccounts = $this->mergeNordigenAccountLists($importerAccounts, $fireflyIIIaccounts);
$uniqueColumns = config('nordigen.unique_column_options');
$importerAccounts = $this->mergeNordigenAccountLists(
$importerAccounts,
$fireflyIIIaccounts
);
}
if ('spectre' === $flow) {
$importerAccounts = $this->getSpectreAccounts($configuration);
$uniqueColumns = config('spectre.unique_column_options');
$importerAccounts = $this->mergeSpectreAccountLists($importerAccounts, $fireflyIIIaccounts);
$uniqueColumns = config('spectre.unique_column_options');
$importerAccounts = $this->mergeSpectreAccountLists(
$importerAccounts,
$fireflyIIIaccounts
);
}
if ('simplefin' === $flow) {
$importerAccounts = $this->getSimpleFINAccounts();
$uniqueColumns = config('simplefin.unique_column_options', ['id']);
$importerAccounts = $this->mergeSimpleFINAccountLists(
$importerAccounts,
$fireflyIIIaccounts
);
}
if ('file' === $flow) {
// detect content type and save to config object.
$detector = new FileContentSherlock();
$content = StorageService::getContent(session()->get(Constants::UPLOAD_DATA_FILE), $configuration->isConversion());
$content = StorageService::getContent(
session()->get(Constants::UPLOAD_DATA_FILE),
$configuration->isConversion()
);
$fileType = $detector->detectContentTypeFromContent($content);
$configuration->setContentType($fileType);
}
// Get currency data for SimpleFIN account creation widget
$currencies = $this->getCurrencies();
return view(
'import.004-configure.index',
compact('mainTitle', 'subTitle', 'fireflyIIIaccounts', 'configuration', 'flow', 'importerAccounts', 'uniqueColumns')
compact(
'mainTitle',
'subTitle',
'fireflyIIIaccounts',
'configuration',
'flow',
'importerAccounts',
'uniqueColumns',
'currencies'
)
);
}
/**
* Get SimpleFIN accounts from session data
*/
private function getSimpleFINAccounts(): array
{
$accountsData = session()->get(Constants::SIMPLEFIN_ACCOUNTS_DATA, []);
$accounts = [];
foreach ($accountsData ?? [] as $account) {
// Ensure the account has required SimpleFIN protocol fields
if (!isset($account['id']) || empty($account['id'])) {
app('log')->warning(
'SimpleFIN account data is missing a valid ID, skipping.',
['account_data' => $account]
);
continue;
}
if (!isset($account['name'])) {
app('log')->warning(
'SimpleFIN account data is missing name field, adding default.',
['account_id' => $account['id']]
);
$account['name'] =
'Unknown Account (ID: ' . $account['id'] . ')';
}
if (!isset($account['currency'])) {
app('log')->warning(
'SimpleFIN account data is missing currency field, this may cause issues.',
['account_id' => $account['id']]
);
}
if (!isset($account['balance'])) {
app('log')->warning(
'SimpleFIN account data is missing balance field, this may cause issues.',
['account_id' => $account['id']]
);
}
// Preserve raw SimpleFIN protocol data structure
$accounts[] = $account;
}
return $accounts;
}
/**
* Merge SimpleFIN accounts with Firefly III accounts
*/
private function mergeSimpleFINAccountLists(
array $simplefinAccounts,
array $fireflyAccounts
): array {
$return = [];
foreach ($simplefinAccounts as $sfinAccountData) {
// $sfinAccountData is raw SimpleFIN protocol data with fields:
// ['id', 'name', 'currency', 'balance', 'balance-date', 'org', etc.]
$importAccountRepresentation = (object) [
'id' => $sfinAccountData['id'], // Expected by component for form elements, and by getMappedTo (as 'identifier')
'name' => $sfinAccountData['name'], // Expected by getMappedTo, display in component
'status' => 'active', // Expected by view for status checks
'currency' => $sfinAccountData['currency'] ?? null, // SimpleFIN currency field
'balance' => $sfinAccountData['balance'] ?? null, // SimpleFIN balance (numeric string)
'balance_date' => $sfinAccountData['balance-date'] ?? null, // SimpleFIN balance timestamp
'org' => $sfinAccountData['org'] ?? null, // SimpleFIN organization data
'iban' => null, // Placeholder for consistency if component expects it
'extra' => $sfinAccountData['extra'] ?? [], // SimpleFIN extra data
'bic' => null, // Placeholder
'product' => null, // Placeholder
'cashAccountType' => null, // Placeholder
'usage' => null, // Placeholder
'resourceId' => null, // Placeholder
'bban' => null, // Placeholder
'ownerName' => null, // Placeholder
];
$return[] = [
'import_account' => $importAccountRepresentation, // The DTO-like object for the component
'name' => $sfinAccountData['name'], // SimpleFIN account name
'id' => $sfinAccountData['id'], // ID for form fields (do_import[ID], accounts[ID])
'mapped_to' => $this->getMappedTo(
(object) [
'identifier' => $importAccountRepresentation->id,
'name' => $importAccountRepresentation->name,
],
$fireflyAccounts
), // getMappedTo needs 'identifier'
'type' => 'source', // Indicates it's an account from the import source
'firefly_iii_accounts' => $fireflyAccounts, // Required by x-importer-account component
];
}
return $return;
}
/**
* Get available currencies from Firefly III for account creation
*/
private function getCurrencies(): array
{
try {
/** @var TransactionCurrencies $mapper */
$mapper = app(TransactionCurrencies::class);
return $mapper->getMap();
} catch (\Exception $e) {
app('log')->error('Failed to load currencies: ' . $e->getMessage());
return [];
}
}
/**
* Stub for determining if an imported account is mapped to a Firefly III account.
* TODO: Implement actual mapping logic.
*
* @param object $importAccount An object representing the account from the import source.
* Expected to have at least 'identifier' and 'name' properties.
* @param array $fireflyAccounts Array of existing Firefly III accounts.
* @return ?string The ID of the mapped Firefly III account, or null if not mapped.
*/
private function getMappedTo(
object $importAccount,
array $fireflyAccounts
): ?string {
$importAccountName = $importAccount->name ?? null;
if (empty($importAccountName)) {
return null;
}
// Check assets accounts for name match
if (
isset($fireflyAccounts['assets']) &&
is_array($fireflyAccounts['assets'])
) {
foreach ($fireflyAccounts['assets'] as $fireflyAccount) {
$fireflyAccountName = $fireflyAccount->name ?? null;
if (
!empty($fireflyAccountName) &&
trim(strtolower($fireflyAccountName)) ===
trim(strtolower($importAccountName))
) {
return (string) $fireflyAccount->id;
}
}
}
// Check liability accounts for name match
if (
isset($fireflyAccounts['liabilities']) &&
is_array($fireflyAccounts['liabilities'])
) {
foreach ($fireflyAccounts['liabilities'] as $fireflyAccount) {
$fireflyAccountName = $fireflyAccount->name ?? null;
if (
!empty($fireflyAccountName) &&
trim(strtolower($fireflyAccountName)) ===
trim(strtolower($importAccountName))
) {
return (string) $fireflyAccount->id;
}
}
}
return null;
}
public function phpDate(Request $request): JsonResponse
{
app('log')->debug(sprintf('Method %s', __METHOD__));
$dateObj = new Date();
[$locale, $format] = $dateObj->splitLocaleFormat((string) $request->get('format'));
$date = today()->locale($locale);
$dateObj = new Date();
[$locale, $format] = $dateObj->splitLocaleFormat(
(string) $request->get('format')
);
$date = today()->locale($locale);
return response()->json(['result' => $date->translatedFormat($format)]);
}
@ -154,30 +372,97 @@ class ConfigurationController extends Controller
/**
* @throws ImporterErrorException
*/
public function postIndex(ConfigurationPostRequest $request): RedirectResponse
{
public function postIndex(
ConfigurationPostRequest $request
): RedirectResponse {
app('log')->debug(sprintf('Now running %s', __METHOD__));
// store config on drive.v
$fromRequest = $request->getAll();
$fromRequest = $request->getAll();
$configuration = Configuration::fromRequest($fromRequest);
$configuration->setFlow($request->cookie(Constants::FLOW_COOKIE));
// TODO are all fields actually in the config?
// loop accounts:
$accounts = [];
$accounts = [];
foreach (array_keys($fromRequest['do_import']) as $identifier) {
if (array_key_exists($identifier, $fromRequest['accounts'])) {
$accounts[$identifier] = (int) $fromRequest['accounts'][$identifier];
$accountValue = (int)$fromRequest['accounts'][$identifier];
$accounts[$identifier] = $accountValue;
} else {
app('log')->warning(
sprintf(
'Account identifier %s in do_import but not in accounts array',
$identifier
)
);
}
}
$configuration->setAccounts($accounts);
// Store new account creation data
$newAccounts = $fromRequest['new_account'] ?? [];
$configuration->setNewAccounts($newAccounts);
// Store do_import selections in session for validation
session()->put('do_import', $fromRequest['do_import'] ?? []);
// Validate configuration contract for SimpleFIN
if ('simplefin' === $configuration->getFlow()) {
$validator = new ConfigurationContractValidator();
// Validate form structure first
$formValidation = $validator->validateFormFieldStructure(
$fromRequest
);
if (!$formValidation->isValid()) {
app('log')->error(
'SimpleFIN form validation failed',
$formValidation->getErrors()
);
return redirect()
->back()
->withErrors($formValidation->getErrorMessages())
->withInput();
}
// Validate complete configuration contract
$contractValidation = $validator->validateConfigurationContract(
$configuration
);
if (!$contractValidation->isValid()) {
app('log')->error(
'SimpleFIN configuration contract validation failed',
$contractValidation->getErrors()
);
return redirect()
->back()
->withErrors($contractValidation->getErrorMessages())
->withInput();
}
if ($contractValidation->hasWarnings()) {
app('log')->warning(
'SimpleFIN configuration contract warnings',
$contractValidation->getWarnings()
);
}
}
$configuration->updateDateRange();
$json = '{}';
// Map data option is now user-selectable for SimpleFIN via checkbox
$json = '{}';
try {
$json = json_encode($configuration->toArray(), JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT);
$json = json_encode(
$configuration->toArray(),
JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT
);
} catch (\JsonException $e) {
app('log')->error($e->getMessage());
@ -185,18 +470,24 @@ class ConfigurationController extends Controller
}
StorageService::storeContent($json);
session()->put(Constants::CONFIGURATION, $configuration->toSessionArray());
session()->put(
Constants::CONFIGURATION,
$configuration->toSessionArray()
);
app('log')->debug(sprintf('Configuration debug: Connection ID is "%s"', $configuration->getConnection()));
// set config as complete.
session()->put(Constants::CONFIG_COMPLETE_INDICATOR, true);
if ('nordigen' === $configuration->getFlow() || 'spectre' === $configuration->getFlow()) {
// at this point, nordigen is ready for data conversion.
if (
'nordigen' === $configuration->getFlow() ||
'spectre' === $configuration->getFlow() ||
'simplefin' === $configuration->getFlow()
) {
// at this point, nordigen, spectre, and simplefin are ready for data conversion.
session()->put(Constants::READY_FOR_CONVERSION, true);
}
// always redirect to roles, even if this isn't the step yet
// for nordigen and spectre, roles will be skipped right away.
// for nordigen, spectre, and simplefin, roles will be skipped right away.
return redirect(route('005-roles.index'));
}
}

109
app/Http/Controllers/Import/ConversionController.php

@ -34,6 +34,8 @@ use App\Services\Nordigen\Conversion\RoutineManager as NordigenRoutineManager;
use App\Services\Session\Constants;
use App\Services\Shared\Conversion\ConversionStatus;
use App\Services\Shared\Conversion\RoutineStatusManager;
use App\Services\SimpleFIN\Conversion\RoutineManager as SimpleFINRoutineManager;
use App\Services\SimpleFIN\Validation\ConfigurationContractValidator;
use App\Services\Spectre\Conversion\RoutineManager as SpectreRoutineManager;
use App\Support\Http\RestoresConfiguration;
use Illuminate\Http\JsonResponse;
@ -70,16 +72,24 @@ class ConversionController extends Controller
$configuration = $this->restoreConfiguration();
app('log')->debug('Will now verify configuration content.');
$jobBackUrl = route('back.mapping');
if (0 === count($configuration->getDoMapping()) && 'file' === $configuration->getFlow()) {
$flow = $configuration->getFlow();
// Set appropriate back URL based on flow
if ('simplefin' === $flow) {
// SimpleFIN always goes back to configuration
$jobBackUrl = route('back.config');
app('log')->debug('SimpleFIN: Pressing "back" will send you to configure.');
} elseif (0 === count($configuration->getDoMapping()) && 'file' === $flow) {
// no mapping, back to roles
app('log')->debug('Pressing "back" will send you to roles.');
$jobBackUrl = route('back.roles');
}
if (0 === count($configuration->getMapping())) {
} elseif (0 === count($configuration->getMapping())) {
// back to mapping
app('log')->debug('Pressing "back" will send you to mapping.');
$jobBackUrl = route('back.mapping');
} else {
// default back to mapping
$jobBackUrl = route('back.mapping');
}
// TODO option is not used atm.
@ -119,7 +129,20 @@ class ConversionController extends Controller
app('log')->debug('Create Spectre routine manager.');
$routine = new SpectreRoutineManager($identifier);
}
if ($configuration->isMapAllData() && in_array($flow, ['spectre', 'nordigen'], true)) {
if ('simplefin' === $flow) {
app('log')->debug('Create SimpleFIN routine manager.');
try {
$routine = new SimpleFINRoutineManager($identifier);
app('log')->debug('SimpleFIN routine manager created successfully.');
} catch (\Throwable $e) {
app('log')->error('Failed to create SimpleFIN routine manager: ' . $e->getMessage());
app('log')->error('Error class: ' . get_class($e));
app('log')->error('Error file: ' . $e->getFile() . ':' . $e->getLine());
app('log')->error('Stack trace: ' . $e->getTraceAsString());
throw $e;
}
}
if ($configuration->isMapAllData() && in_array($flow, ['spectre', 'nordigen', 'simplefin'], true)) {
app('log')->debug('Will redirect to mapping after conversion.');
$nextUrl = route('006-mapping.index');
}
@ -136,7 +159,20 @@ class ConversionController extends Controller
session()->put(Constants::CONVERSION_JOB_IDENTIFIER, $identifier);
app('log')->debug(sprintf('Stored "%s" under "%s"', $identifier, Constants::CONVERSION_JOB_IDENTIFIER));
return view('import.007-convert.index', compact('mainTitle', 'identifier', 'jobBackUrl', 'flow', 'nextUrl'));
// Prepare new account creation data for SimpleFIN
$newAccountsToCreate = [];
if ('simplefin' === $flow) {
$accounts = $configuration->getAccounts();
$newAccounts = $configuration->getNewAccounts();
foreach ($accounts as $simplefinAccountId => $fireflyAccountId) {
if ('create_new' === $fireflyAccountId && isset($newAccounts[$simplefinAccountId])) {
$newAccountsToCreate[$simplefinAccountId] = $newAccounts[$simplefinAccountId];
}
}
}
return view('import.007-convert.index', compact('mainTitle', 'identifier', 'jobBackUrl', 'flow', 'nextUrl', 'newAccountsToCreate'));
}
public function start(Request $request): JsonResponse
@ -146,6 +182,55 @@ class ConversionController extends Controller
$configuration = $this->restoreConfiguration();
$routine = null;
// Validate configuration contract for SimpleFIN before proceeding
if ('simplefin' === $configuration->getFlow()) {
$validator = new ConfigurationContractValidator();
$contractValidation = $validator->validateConfigurationContract($configuration);
if (!$contractValidation->isValid()) {
app('log')->error('SimpleFIN configuration contract validation failed during conversion start', $contractValidation->getErrors());
RoutineStatusManager::setConversionStatus(ConversionStatus::CONVERSION_ERRORED);
$importJobStatus = RoutineStatusManager::startOrFindConversion($identifier);
return response()->json($importJobStatus->toArray());
}
if ($contractValidation->hasWarnings()) {
app('log')->warning('SimpleFIN configuration contract warnings during conversion start', $contractValidation->getWarnings());
}
app('log')->debug('SimpleFIN configuration contract validation successful for conversion start');
}
// Handle new account data for SimpleFIN
if ('simplefin' === $configuration->getFlow()) {
$newAccountData = $request->get('new_account_data', []);
if (!empty($newAccountData)) {
app('log')->debug('Updating configuration with detailed new account data', $newAccountData);
// Update the configuration with the detailed account creation data
$existingNewAccounts = $configuration->getNewAccounts();
foreach ($newAccountData as $accountId => $accountDetails) {
if (isset($existingNewAccounts[$accountId])) {
// Merge the detailed data with existing data
$existingNewAccounts[$accountId] = array_merge(
$existingNewAccounts[$accountId],
[
'name' => $accountDetails['name'],
'type' => $accountDetails['type'],
'currency' => $accountDetails['currency'],
'opening_balance' => $accountDetails['opening_balance']
]
);
}
}
$configuration->setNewAccounts($existingNewAccounts);
// Update session with new configuration
session()->put(Constants::CONFIGURATION, $configuration->toSessionArray());
}
}
// now create the right class:
$flow = $configuration->getFlow();
if (!in_array($flow, config('importer.flows'), true)) {
@ -167,6 +252,18 @@ class ConversionController extends Controller
if ('spectre' === $flow) {
$routine = new SpectreRoutineManager($identifier);
}
if ('simplefin' === $flow) {
try {
$routine = new SimpleFINRoutineManager($identifier);
app('log')->debug('SimpleFIN routine manager created successfully in start method.');
} catch (\Throwable $e) {
app('log')->error('Failed to create SimpleFIN routine manager in start method: ' . $e->getMessage());
app('log')->error('Error class: ' . get_class($e));
app('log')->error('Error file: ' . $e->getFile() . ':' . $e->getLine());
app('log')->error('Stack trace: ' . $e->getTraceAsString());
throw $e;
}
}
if (null === $routine) {
throw new ImporterErrorException(sprintf('Could not create routine manager for flow "%s"', $flow));

128
app/Http/Controllers/Import/DuplicateCheckController.php

@ -0,0 +1,128 @@
<?php
/*
* DuplicateCheckController.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Http\Controllers\Import;
use App\Http\Controllers\Controller;
use App\Http\Middleware\ConfigurationControllerMiddleware;
use App\Services\SimpleFIN\Validation\ConfigurationContractValidator;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Log;
/**
* Class DuplicateCheckController
*
* Provides AJAX endpoint for real-time duplicate account validation
*/
class DuplicateCheckController extends Controller
{
/**
* Create a new controller instance.
*/
public function __construct()
{
parent::__construct();
$this->middleware(ConfigurationControllerMiddleware::class);
}
/**
* Check if an account name and type combination already exists
*
* @param Request $request
* @return JsonResponse
*/
public function checkDuplicate(Request $request): JsonResponse
{
try {
$name = trim($request->input('name', ''));
$type = trim($request->input('type', ''));
Log::debug('DUPLICATE_CHECK: Received request', [
'name' => $name,
'type' => $type,
'name_length' => strlen($name)
]);
// Empty name or type means no duplicate possible
if ('' === $name || '' === $type) {
Log::debug('DUPLICATE_CHECK: Empty name or type, returning no duplicate');
return response()->json([
'isDuplicate' => false,
'message' => null
]);
}
// Validate account type
$validTypes = ['asset', 'liability', 'expense', 'revenue'];
if (!in_array($type, $validTypes, true)) {
Log::warning('DUPLICATE_CHECK: Invalid account type provided', [
'type' => $type,
'valid_types' => $validTypes
]);
return response()->json([
'isDuplicate' => false,
'message' => null
]);
}
// Create validator instance and check for duplicates
$validator = new ConfigurationContractValidator();
$isDuplicate = $validator->checkSingleAccountDuplicate($name, $type);
$message = null;
if ($isDuplicate) {
$message = sprintf('%s <em>%s</em> already exists!', ucfirst($type), $name);
}
Log::debug('DUPLICATE_CHECK: Validation result', [
'name' => $name,
'type' => $type,
'isDuplicate' => $isDuplicate,
'message' => $message
]);
return response()->json([
'isDuplicate' => $isDuplicate,
'message' => $message
]);
} catch (\Exception $e) {
Log::error('DUPLICATE_CHECK: Exception during duplicate check', [
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString(),
'name' => $request->input('name', ''),
'type' => $request->input('type', '')
]);
// Return safe response on error - assume no duplicate to avoid blocking user
return response()->json([
'isDuplicate' => false,
'message' => null,
'error' => 'Unable to check for duplicates at this time'
]);
}
}
}

113
app/Http/Controllers/Import/MapController.php

@ -84,9 +84,9 @@ class MapController extends Controller
$data = $this->getCamtMapInformation();
}
// nordigen, spectre and others:
// nordigen, spectre, simplefin and others:
if ('file' !== $configuration->getFlow()) {
app('log')->debug('Get mapping data for GoCardless and spectre');
app('log')->debug('Get mapping data for GoCardless, Spectre, and SimpleFIN');
$roles = [];
$data = $this->getImporterMapInformation();
}
@ -247,7 +247,7 @@ class MapController extends Controller
$configuration = $this->restoreConfiguration();
$existingMapping = $configuration->getMapping();
/*
* To map Nordigen, pretend the file has one "column" (this is based on the CSV importer after all)
* To map Nordigen and SimpleFIN, pretend the file has one "column" (this is based on the CSV importer after all)
* that contains:
* - opposing account names (this is preordained).
*/
@ -293,6 +293,26 @@ class MapController extends Controller
$category['mapped'] = $existingMapping[$index] ?? [];
$data[] = $category;
}
if ('simplefin' === $configuration->getFlow()) {
// index 0: expense/revenue account mapping
$index = 0;
$expenseRevenue = config('csv.import_roles.opposing-name') ?? null;
$expenseRevenue['role'] = 'opposing-name';
$expenseRevenue['values'] = $this->getExpenseRevenueAccounts();
// Use ExpenseRevenueAccounts mapper for SimpleFIN
$class = 'App\\Services\\CSV\\Mapper\\ExpenseRevenueAccounts';
if (!class_exists($class)) {
throw new \InvalidArgumentException(sprintf('Class %s does not exist.', $class));
}
app('log')->debug(sprintf('Associated class is %s', $class));
/** @var MapperInterface $object */
$object = app($class);
$expenseRevenue['mapping_data'] = $object->getMap();
$expenseRevenue['mapped'] = $existingMapping[$index] ?? [];
$data[] = $expenseRevenue;
}
return $data;
}
@ -301,8 +321,25 @@ class MapController extends Controller
{
app('log')->debug(sprintf('Now in %s', __METHOD__));
$downloadIdentifier = session()->get(Constants::CONVERSION_JOB_IDENTIFIER);
$disk = Storage::disk(self::DISK_NAME);
$json = $disk->get(sprintf('%s.json', $downloadIdentifier));
if (null === $downloadIdentifier) {
app('log')->warning('No conversion job identifier found in session - mapping called before conversion');
return [];
}
$disk = Storage::disk(self::DISK_NAME);
if (!$disk->exists(sprintf('%s.json', $downloadIdentifier))) {
app('log')->warning(sprintf('Conversion file %s.json does not exist - mapping called before conversion', $downloadIdentifier));
return [];
}
$json = $disk->get(sprintf('%s.json', $downloadIdentifier));
if (null === $json) {
app('log')->warning(sprintf('Conversion file %s.json is empty', $downloadIdentifier));
return [];
}
try {
$array = json_decode($json, true, 512, JSON_THROW_ON_ERROR);
@ -332,6 +369,67 @@ class MapController extends Controller
return array_unique($filtered);
}
private function getExpenseRevenueAccounts(): array
{
app('log')->debug(sprintf('Now in %s', __METHOD__));
$downloadIdentifier = session()->get(Constants::CONVERSION_JOB_IDENTIFIER);
if (null === $downloadIdentifier) {
app('log')->warning('No conversion job identifier found in session - mapping called before conversion');
return [];
}
$disk = Storage::disk(self::DISK_NAME);
if (!$disk->exists(sprintf('%s.json', $downloadIdentifier))) {
app('log')->warning(sprintf('Conversion file %s.json does not exist - mapping called before conversion', $downloadIdentifier));
return [];
}
$json = $disk->get(sprintf('%s.json', $downloadIdentifier));
if (null === $json) {
app('log')->warning(sprintf('Conversion file %s.json is empty', $downloadIdentifier));
return [];
}
try {
$array = json_decode($json, true, 512, JSON_THROW_ON_ERROR);
} catch (\JsonException $e) {
throw new ImporterErrorException(sprintf('Could not decode download: %s', $e->getMessage()), 0, $e);
}
$expenseRevenue = [];
$total = count($array);
/** @var array $transaction */
foreach ($array as $index => $transaction) {
app('log')->debug(sprintf('[%s/%s] Parsing transaction for expense/revenue accounts', $index + 1, $total));
/** @var array $row */
foreach ($transaction['transactions'] as $row) {
// Extract expense/revenue destination names from SimpleFIN transactions
$destinationName = (string) (array_key_exists('destination_name', $row) ? $row['destination_name'] : '');
$sourceName = (string) (array_key_exists('source_name', $row) ? $row['source_name'] : '');
// Add both source and destination names as potential expense/revenue accounts
if (!empty($destinationName)) {
$expenseRevenue[] = $destinationName;
}
if (!empty($sourceName)) {
$expenseRevenue[] = $sourceName;
}
}
}
$filtered = array_filter(
$expenseRevenue,
static function (string $value) {
return '' !== $value;
}
);
return array_unique($filtered);
}
private function getCategories(): array
{
app('log')->debug(sprintf('Now in %s', __METHOD__));
@ -436,9 +534,10 @@ class MapController extends Controller
// set map config as complete.
session()->put(Constants::MAPPING_COMPLETE_INDICATOR, true);
session()->put(Constants::READY_FOR_CONVERSION, true);
if ('nordigen' === $configuration->getFlow() || 'spectre' === $configuration->getFlow()) {
// if nordigen, now ready for submission!
if ('nordigen' === $configuration->getFlow() || 'spectre' === $configuration->getFlow() || 'simplefin' === $configuration->getFlow()) {
// if nordigen, spectre, or simplefin, now ready for submission!
session()->put(Constants::READY_FOR_SUBMISSION, true);
return redirect()->route('008-submit.index');
}
return redirect()->route('007-convert.index');

60
app/Http/Controllers/Import/SubmitController.php

@ -29,7 +29,9 @@ use App\Events\ImportedTransactions;
use App\Exceptions\ImporterErrorException;
use App\Http\Controllers\Controller;
use App\Http\Middleware\SubmitControllerMiddleware;
use App\Jobs\ProcessImportSubmissionJob;
use App\Services\Session\Constants;
use App\Services\Shared\Authentication\SecretManager;
use App\Services\Shared\Import\Routine\RoutineManager;
use App\Services\Shared\Import\Status\SubmissionStatus;
use App\Services\Shared\Import\Status\SubmissionStatusManager;
@ -71,7 +73,8 @@ class SubmitController extends Controller
$statusManager = new SubmissionStatusManager();
$configuration = $this->restoreConfiguration();
$flow = $configuration->getFlow();
$jobBackUrl = route('back.conversion');
// The step immediately preceding submit (008) is always convert (007)
$jobBackUrl = route('007-convert.index');
// submission job ID may be in session:
$identifier = session()->get(Constants::IMPORT_JOB_IDENTIFIER);
@ -136,38 +139,37 @@ class SubmitController extends Controller
return response()->json($importJobStatus->toArray());
}
$routine->setTransactions($transactions);
// Retrieve authentication credentials for job
$accessToken = SecretManager::getAccessToken();
$baseUrl = SecretManager::getBaseUrl();
$vanityUrl = SecretManager::getVanityUrl();
SubmissionStatusManager::setSubmissionStatus(SubmissionStatus::SUBMISSION_RUNNING);
// then push stuff into the routine:
$routine->setConfiguration($configuration);
try {
$routine->start();
} catch (ImporterErrorException $e) {
app('log')->error($e->getMessage());
SubmissionStatusManager::setSubmissionStatus(SubmissionStatus::SUBMISSION_ERRORED);
return response()->json($importJobStatus->toArray());
}
// set done:
SubmissionStatusManager::setSubmissionStatus(SubmissionStatus::SUBMISSION_DONE);
// set config as complete.
session()->put(Constants::SUBMISSION_COMPLETE_INDICATOR, true);
// Set initial running status before dispatching job
SubmissionStatusManager::setSubmissionStatus(
SubmissionStatus::SUBMISSION_RUNNING,
$identifier
);
event(
new ImportedTransactions(
array_merge($routine->getAllMessages()),
array_merge($routine->getAllWarnings()),
array_merge($routine->getAllErrors()),
[]
)
// Dispatch asynchronous job for processing
ProcessImportSubmissionJob::dispatch(
$identifier,
$configuration,
$transactions,
$accessToken,
$baseUrl,
$vanityUrl
);
return response()->json($importJobStatus->toArray());
app('log')->debug('ProcessImportSubmissionJob dispatched', [
'identifier' => $identifier,
'transaction_count' => count($transactions)
]);
// Return immediate response indicating job was dispatched
return response()->json([
'status' => 'job_dispatched',
'identifier' => $identifier
]);
}
public function status(Request $request): JsonResponse

261
app/Http/Controllers/Import/UploadController.php

@ -31,6 +31,7 @@ use App\Http\Middleware\UploadControllerMiddleware;
use App\Services\CSV\Configuration\ConfigFileProcessor;
use App\Services\Session\Constants;
use App\Services\Shared\File\FileContentSherlock;
use App\Services\SimpleFIN\SimpleFINService;
use App\Services\Storage\StorageService;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Contracts\View\Factory;
@ -59,7 +60,7 @@ class UploadController extends Controller
app('view')->share('pageTitle', 'Upload files');
$this->middleware(UploadControllerMiddleware::class);
// This variable is used to make sure the configuration object also knows the file type.
$this->contentType = 'unknown';
$this->contentType = 'unknown';
$this->configFileName = '';
}
@ -70,22 +71,22 @@ class UploadController extends Controller
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
$mainTitle = 'Upload your file(s)';
$subTitle = 'Start page and instructions';
$flow = $request->cookie(Constants::FLOW_COOKIE);
$subTitle = 'Start page and instructions';
$flow = $request->cookie(Constants::FLOW_COOKIE);
// get existing configs.
$disk = \Storage::disk('configurations');
$disk = \Storage::disk('configurations');
app('log')->debug(
sprintf(
'Going to check directory for config files: %s',
config('filesystems.disks.configurations.root'),
config('filesystems.disks.configurations.root')
)
);
$all = $disk->files();
$all = $disk->files();
// remove files from list
$list = [];
$ignored = config('importer.ignored_files');
$list = [];
$ignored = config('importer.ignored_files');
foreach ($all as $entry) {
if (!in_array($entry, $ignored, true)) {
$list[] = $entry;
@ -94,7 +95,10 @@ class UploadController extends Controller
app('log')->debug('List of files:', $list);
return view('import.003-upload.index', compact('mainTitle', 'subTitle', 'list', 'flow'));
return view(
'import.003-upload.index',
compact('mainTitle', 'subTitle', 'list', 'flow')
);
}
/**
@ -106,15 +110,20 @@ class UploadController extends Controller
*/
public function upload(Request $request)
{
app('log')->debug('DEBUG_ENTRY: UploadController::upload() INVOKED'); // Unique entry marker
app('log')->debug(sprintf('Now at %s', __METHOD__));
$importedFile = $request->file('importable_file');
$configFile = $request->file('config_file');
app('log')->debug(
'UploadController::upload() - Request All:',
$request->all()
);
$importedFile = $request->file('importable_file');
$configFile = $request->file('config_file');
$simpleFINtoken = $request->get('simplefin_token');
$flow = $request->cookie(Constants::FLOW_COOKIE);
$errors = new MessageBag();
$flow = $request->cookie(Constants::FLOW_COOKIE);
$errors = new MessageBag();
// process uploaded file (if present)
$errors = $this->processUploadedFile($flow, $errors, $importedFile);
$errors = $this->processUploadedFile($flow, $errors, $importedFile);
// process config file (if present)
if (0 === count($errors) && null !== $configFile) {
@ -122,21 +131,18 @@ class UploadController extends Controller
}
// process pre-selected file (if present):
$errors = $this->processSelection($errors, (string) $request->get('existing_config'), $configFile);
$errors = $this->processSelection(
$errors,
(string) $request->get('existing_config'),
$configFile
);
if ($errors->count() > 0) {
return redirect(route('003-upload.index'))->withErrors($errors);
}
if ('simplefin' === $flow) {
// at this point we have no configuration file where we can overwrite things, so collect it first.
// session()->put(Constants::UPLOAD_CONFIG_FILE, $configFileName);
if ('' === $this->configFileName) {
// user has not uploaded any configuration.
}
var_dump($this->configFileName);
exit;
return $this->handleSimpleFINFlow($request, new MessageBag());
}
if ('nordigen' === $flow) {
@ -159,8 +165,11 @@ class UploadController extends Controller
* @throws FilesystemException
* @throws ImporterErrorException
*/
private function processUploadedFile(string $flow, MessageBag $errors, ?UploadedFile $file): MessageBag
{
private function processUploadedFile(
string $flow,
MessageBag $errors,
?UploadedFile $file
): MessageBag {
if (null === $file && 'file' === $flow) {
$errors->add('importable_file', 'No file was uploaded.');
@ -174,18 +183,22 @@ class UploadController extends Controller
// upload the file to a temp directory and use it from there.
if (0 === $errorNumber) {
$detector = new FileContentSherlock();
$this->contentType = $detector->detectContentType($file->getPathname());
$content = '';
$detector = new FileContentSherlock();
$this->contentType = $detector->detectContentType(
$file->getPathname()
);
$content = '';
if ('csv' === $this->contentType) {
$content = file_get_contents($file->getPathname());
// https://stackoverflow.com/questions/11066857/detect-eol-type-using-php
// because apparently there are banks that use "\r" as newline. Looking at the morons of KBC Bank, Belgium.
// This one is for you: 🤦‍♀️
$eol = $this->detectEOL($content);
$eol = $this->detectEOL($content);
if ("\r" === $eol) {
app('log')->error('Your bank is dumb. Tell them to fix their CSV files.');
app('log')->error(
'Your bank is dumb. Tell them to fix their CSV files.'
);
$content = str_replace("\r", "\n", $content);
}
}
@ -193,7 +206,7 @@ class UploadController extends Controller
if ('camt' === $this->contentType) {
$content = file_get_contents($file->getPathname());
}
$fileName = StorageService::storeContent($content);
$fileName = StorageService::storeContent($content);
session()->put(Constants::UPLOAD_DATA_FILE, $fileName);
session()->put(Constants::HAS_UPLOAD, true);
}
@ -206,14 +219,14 @@ class UploadController extends Controller
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
$errors = [
UPLOAD_ERR_OK => 'There is no error, the file uploaded with success.',
UPLOAD_ERR_INI_SIZE => 'The uploaded file exceeds the upload_max_filesize directive in php.ini.',
UPLOAD_ERR_FORM_SIZE => 'The uploaded file exceeds the MAX_FILE_SIZE directive that was specified in the HTML form.',
UPLOAD_ERR_PARTIAL => 'The uploaded file was only partially uploaded.',
UPLOAD_ERR_NO_FILE => 'No file was uploaded.',
UPLOAD_ERR_OK => 'There is no error, the file uploaded with success.',
UPLOAD_ERR_INI_SIZE => 'The uploaded file exceeds the upload_max_filesize directive in php.ini.',
UPLOAD_ERR_FORM_SIZE => 'The uploaded file exceeds the MAX_FILE_SIZE directive that was specified in the HTML form.',
UPLOAD_ERR_PARTIAL => 'The uploaded file was only partially uploaded.',
UPLOAD_ERR_NO_FILE => 'No file was uploaded.',
UPLOAD_ERR_NO_TMP_DIR => 'Missing a temporary folder.',
UPLOAD_ERR_CANT_WRITE => 'Failed to write file to disk. Introduced in PHP 5.1.0.',
UPLOAD_ERR_EXTENSION => 'A PHP extension stopped the file upload.',
UPLOAD_ERR_EXTENSION => 'A PHP extension stopped the file upload.',
];
return $errors[$error] ?? 'Unknown error';
@ -221,21 +234,28 @@ class UploadController extends Controller
private function detectEOL(string $string): string
{
$eols = [
'\n\r' => "\n\r", // 0x0A - 0x0D - acorn BBC
'\r\n' => "\r\n", // 0x0D - 0x0A - Windows, DOS OS/2
'\n' => "\n", // 0x0A - - Unix, OSX
'\r' => "\r", // 0x0D - - Apple ][, TRS80
$eols = [
'\n\r' => "\n\r", // 0x0A - 0x0D - acorn BBC
'\r\n' => "\r\n", // 0x0D - 0x0A - Windows, DOS OS/2
'\n' => "\n", // 0x0A - - Unix, OSX
'\r' => "\r", // 0x0D - - Apple ][, TRS80
];
$curCount = 0;
$curEol = '';
$curEol = '';
foreach ($eols as $eolKey => $eol) {
$count = substr_count($string, $eol);
app('log')->debug(sprintf('Counted %dx "%s" EOL in upload.', $count, $eolKey));
app('log')->debug(
sprintf('Counted %dx "%s" EOL in upload.', $count, $eolKey)
);
if ($count > $curCount) {
$curCount = $count;
$curEol = $eol;
app('log')->debug(sprintf('Conclusion: "%s" is the EOL in this file.', $eolKey));
$curEol = $eol;
app('log')->debug(
sprintf(
'Conclusion: "%s" is the EOL in this file.',
$eolKey
)
);
}
}
@ -247,8 +267,10 @@ class UploadController extends Controller
*
* @throws ImporterErrorException
*/
private function processConfigFile(MessageBag $errors, UploadedFile $file): MessageBag
{
private function processConfigFile(
MessageBag $errors,
UploadedFile $file
): MessageBag {
app('log')->debug('Config file is present.');
$errorNumber = $file->getError();
if (0 !== $errorNumber) {
@ -257,27 +279,42 @@ class UploadController extends Controller
// upload the file to a temp directory and use it from there.
if (0 === $errorNumber) {
app('log')->debug('Config file uploaded.');
$this->configFileName = StorageService::storeContent(file_get_contents($file->getPathname()));
$this->configFileName = StorageService::storeContent(
file_get_contents($file->getPathname())
);
session()->put(Constants::UPLOAD_CONFIG_FILE, $this->configFileName);
session()->put(
Constants::UPLOAD_CONFIG_FILE,
$this->configFileName
);
// process the config file
$success = false;
$configuration = null;
$success = false;
$configuration = null;
try {
$configuration = ConfigFileProcessor::convertConfigFile($this->configFileName);
$configuration = ConfigFileProcessor::convertConfigFile(
$this->configFileName
);
$configuration->setContentType($this->contentType);
session()->put(Constants::CONFIGURATION, $configuration->toSessionArray());
$success = true;
session()->put(
Constants::CONFIGURATION,
$configuration->toSessionArray()
);
$success = true;
} catch (ImporterErrorException $e) {
$errors->add('config_file', $e->getMessage());
}
// if conversion of the config file was a success, store the new version again:
if (true === $success) {
$configuration->updateDateRange();
$this->configFileName = StorageService::storeContent(json_encode($configuration->toArray(), JSON_PRETTY_PRINT));
session()->put(Constants::UPLOAD_CONFIG_FILE, $this->configFileName);
$this->configFileName = StorageService::storeContent(
json_encode($configuration->toArray(), JSON_PRETTY_PRINT)
);
session()->put(
Constants::UPLOAD_CONFIG_FILE,
$this->configFileName
);
}
}
@ -287,19 +324,29 @@ class UploadController extends Controller
/**
* @throws ImporterErrorException
*/
private function processSelection(MessageBag $errors, string $selection, ?UploadedFile $file): MessageBag
{
private function processSelection(
MessageBag $errors,
string $selection,
?UploadedFile $file
): MessageBag {
if (null === $file && '' !== $selection) {
app('log')->debug('User selected a config file from the store.');
$disk = \Storage::disk('configurations');
$configFileName = StorageService::storeContent($disk->get($selection));
$disk = \Storage::disk('configurations');
$configFileName = StorageService::storeContent(
$disk->get($selection)
);
session()->put(Constants::UPLOAD_CONFIG_FILE, $configFileName);
// process the config file
try {
$configuration = ConfigFileProcessor::convertConfigFile($configFileName);
session()->put(Constants::CONFIGURATION, $configuration->toSessionArray());
$configuration = ConfigFileProcessor::convertConfigFile(
$configFileName
);
session()->put(
Constants::CONFIGURATION,
$configuration->toSessionArray()
);
} catch (ImporterErrorException $e) {
$errors->add('config_file', $e->getMessage());
}
@ -307,4 +354,92 @@ class UploadController extends Controller
return $errors;
}
/**
* Handle SimpleFIN flow integration
*/
private function handleSimpleFINFlow(
Request $request,
MessageBag $errors
): RedirectResponse {
app('log')->debug(
'DEBUG_ENTRY: UploadController::handleSimpleFINFlow() INVOKED'
); // Unique entry marker
app('log')->debug('Processing SimpleFIN flow in unified controller');
app('log')->debug(
'handleSimpleFINFlow() - Request All:',
$request->all()
);
app('log')->debug('handleSimpleFINFlow() - Raw use_demo input:', [
$request->input('use_demo'),
]);
$simpleFINToken = $request->get('simplefin_token');
$bridgeUrl = $request->get('simplefin_bridge_url');
$isDemo = $request->boolean('use_demo');
app('log')->debug('handleSimpleFINFlow() - Evaluated $isDemo:', [
$isDemo,
]);
app('log')->debug('handleSimpleFINFlow() - Bridge URL:', [$bridgeUrl]);
if ($isDemo) {
$simpleFINToken = config('importer.simplefin.demo_token');
$bridgeUrl = 'https://sfin.bridge.which.is'; // Demo mode uses known working Origin
} else {
if (empty($simpleFINToken)) {
$errors->add('simplefin_token', 'SimpleFIN token is required.');
}
if (empty($bridgeUrl)) {
$errors->add(
'simplefin_bridge_url',
'Bridge URL is required for CORS Origin header.'
);
} elseif (!filter_var($bridgeUrl, FILTER_VALIDATE_URL)) {
$errors->add(
'simplefin_bridge_url',
'Bridge URL must be a valid URL.'
);
}
}
if ($errors->count() > 0) {
return redirect(route('003-upload.index'))->withErrors($errors);
}
// Store bridge URL in session BEFORE SimpleFIN service call (service needs it for Origin header)
session()->put(Constants::SIMPLEFIN_BRIDGE_URL, $bridgeUrl);
try {
$simpleFINService = app(SimpleFINService::class);
// Use demo URL for demo mode, otherwise use empty string for claim URL token processing
$apiUrl = $isDemo ? config('importer.simplefin.demo_url') : '';
$accountsData = $simpleFINService->fetchAccountsAndInitialData(
$simpleFINToken,
$apiUrl
);
// Store SimpleFIN data in session for configuration step
session()->put(Constants::SIMPLEFIN_TOKEN, $simpleFINToken);
session()->put(Constants::SIMPLEFIN_ACCOUNTS_DATA, $accountsData);
session()->put(Constants::SIMPLEFIN_IS_DEMO, $isDemo);
session()->put(Constants::HAS_UPLOAD, true);
app('log')->info('SimpleFIN connection established', [
'account_count' => count($accountsData ?? []),
'is_demo' => $isDemo,
]);
return redirect(route('004-configure.index'));
} catch (ImporterErrorException $e) {
app('log')->error('SimpleFIN connection failed', [
'error' => $e->getMessage(),
]);
$errors->add(
'connection',
'Failed to connect to SimpleFIN: ' . $e->getMessage()
);
return redirect(route('003-upload.index'))->withErrors($errors);
}
}
}

82
app/Http/Controllers/IndexController.php

@ -47,12 +47,14 @@ class IndexController extends Controller
public function flush(): mixed
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
session()->forget([Constants::UPLOAD_DATA_FILE, Constants::UPLOAD_CONFIG_FILE, Constants::IMPORT_JOB_IDENTIFIER]);
session()->forget([
Constants::UPLOAD_DATA_FILE,
Constants::UPLOAD_CONFIG_FILE,
Constants::IMPORT_JOB_IDENTIFIER,
]);
session()->flush();
session()->regenerate(true);
$cookies = [
cookie(Constants::FLOW_COOKIE, ''),
];
$cookies = [cookie(Constants::FLOW_COOKIE, '')];
Artisan::call('cache:clear');
// Artisan::call('config:clear'); // disable command to try and fix
@ -66,7 +68,7 @@ class IndexController extends Controller
// global methods to get these values, from cookies or configuration.
// it's up to the manager to provide them.
// if invalid values, redirect to token index.
$validInfo = SecretManager::hasValidSecrets();
$validInfo = SecretManager::hasValidSecrets();
if (!$validInfo) {
app('log')->debug('No valid secrets, redirect to token.index');
@ -74,31 +76,57 @@ class IndexController extends Controller
}
// display to user the method of authentication
$clientId = (string) config('importer.client_id');
$url = (string) config('importer.url');
$pat = false;
if ('' !== (string) config('importer.access_token')) {
$clientId = (string) config('importer.client_id');
$url = (string) config('importer.url');
$accessTokenConfig = (string) config('importer.access_token');
app('log')->debug('DEBUG: IndexController authentication detection', [
'client_id' => $clientId,
'url' => $url,
'access_token_config' => $accessTokenConfig,
'access_token_empty' => '' === $accessTokenConfig,
]);
$pat = false;
if ('' !== $accessTokenConfig) {
$pat = true;
}
$clientIdWithURL = false;
if ('' !== $url && '' !== $clientId) {
$clientIdWithURL = true;
}
$URLonly = false;
if ('' !== $url && '' === $clientId && '' === (string) config('importer.access_token')
) {
$URLonly = false;
if ('' !== $url && '' === $clientId && '' === $accessTokenConfig) {
$URLonly = true;
}
$flexible = false;
$flexible = false;
if ('' === $url && '' === $clientId) {
$flexible = true;
}
$isDocker = env('IS_DOCKER', false);
$identifier = substr(session()->getId(), 0, 10);
$enabled = config('importer.enabled_flows');
return view('index', compact('pat', 'clientIdWithURL', 'URLonly', 'flexible', 'identifier', 'isDocker', 'enabled'));
app('log')->debug('DEBUG: IndexController authentication type flags', [
'pat' => $pat,
'clientIdWithURL' => $clientIdWithURL,
'URLonly' => $URLonly,
'flexible' => $flexible,
]);
$isDocker = env('IS_DOCKER', false);
$identifier = substr(session()->getId(), 0, 10);
$enabled = config('importer.enabled_flows');
return view(
'index',
compact(
'pat',
'clientIdWithURL',
'URLonly',
'flexible',
'identifier',
'isDocker',
'enabled'
)
);
}
public function postIndex(Request $request): mixed
@ -107,14 +135,18 @@ class IndexController extends Controller
// set cookie with flow:
$flow = $request->get('flow');
if (in_array($flow, config('importer.flows'), true)) {
app('log')->debug(sprintf('%s is a valid flow, redirect to authenticate.', $flow));
$cookies = [
cookie(Constants::FLOW_COOKIE, $flow),
];
return redirect(route('002-authenticate.index'))->withCookies($cookies);
app('log')->debug(
sprintf('%s is a valid flow, redirect to authenticate.', $flow)
);
$cookies = [cookie(Constants::FLOW_COOKIE, $flow)];
return redirect(route('002-authenticate.index'))->withCookies(
$cookies
);
}
app('log')->debug(sprintf('"%s" is not a valid flow, redirect to index.', $flow));
app('log')->debug(
sprintf('"%s" is not a valid flow, redirect to index.', $flow)
);
return redirect(route('index'));
}

12
app/Http/Controllers/NavController.php

@ -41,7 +41,17 @@ class NavController extends Controller
public function toConfig()
{
app('log')->debug(__METHOD__);
session()->forget(Constants::CONFIG_COMPLETE_INDICATOR);
// For SimpleFIN flow, don't forget CONFIG_COMPLETE_INDICATOR to preserve form state
$sessionConfig = session()->get(Constants::CONFIGURATION);
$flow = null;
if (is_array($sessionConfig) && isset($sessionConfig['flow'])) {
$flow = $sessionConfig['flow'];
}
if ('simplefin' !== $flow) {
session()->forget(Constants::CONFIG_COMPLETE_INDICATOR);
}
return redirect(route('004-configure.index').'?overruleskip=true');
}

316
app/Http/Controllers/TokenController.php

@ -26,6 +26,7 @@ declare(strict_types=1);
namespace App\Http\Controllers;
use App\Exceptions\ImporterErrorException;
use App\Services\Session\Constants;
use App\Services\Shared\Authentication\SecretManager;
use GrumpyDictator\FFIIIApiSupport\Exceptions\ApiHttpException;
use GrumpyDictator\FFIIIApiSupport\Request\SystemInformationRequest;
@ -58,22 +59,28 @@ class TokenController extends Controller
public function callback(Request $request)
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
$state = (string) session()->pull('state');
$state = (string) session()->pull('state');
$codeVerifier = (string) $request->session()->pull('code_verifier');
$clientId = (int) $request->session()->pull('form_client_id');
$baseURL = (string) $request->session()->pull('form_base_url');
$vanityURL = (string) $request->session()->pull('form_vanity_url');
$code = $request->get('code');
$clientId = (int) $request->session()->pull('form_client_id');
$baseURL = (string) $request->session()->pull('form_base_url');
$vanityURL = (string) $request->session()->pull('form_vanity_url');
$code = $request->get('code');
if ($state !== (string) $request->state) {
app('log')->error(sprintf('State according to session: "%s"', $state));
app('log')->error(sprintf('State returned in request : "%s"', $request->state));
app('log')->error(
sprintf('State according to session: "%s"', $state)
);
app('log')->error(
sprintf('State returned in request : "%s"', $request->state)
);
throw new ImporterErrorException('The "state" returned from your server doesn\'t match the state that was sent.');
throw new ImporterErrorException(
'The "state" returned from your server doesn\'t match the state that was sent.'
);
}
// always POST to the base URL, never the vanity URL.
$finalURL = sprintf('%s/oauth/token', $baseURL);
$params = [
$finalURL = sprintf('%s/oauth/token', $baseURL);
$params = [
'form_params' => [
'grant_type' => 'authorization_code',
'client_id' => $clientId,
@ -96,33 +103,60 @@ class TokenController extends Controller
try {
$response = (new Client($opts))->post($finalURL, $params);
} catch (ClientException|RequestException $e) {
} catch (ClientException | RequestException $e) {
$body = $e->getMessage();
if ($e->hasResponse()) {
$body = (string) $e->getResponse()->getBody();
app('log')->error(sprintf('Client exception when decoding response: %s', $e->getMessage()));
app('log')->error(
sprintf(
'Client exception when decoding response: %s',
$e->getMessage()
)
);
app('log')->error(sprintf('Response from server: "%s"', $body));
// app('log')->error($e->getTraceAsString());
}
return view('error')->with('message', $e->getMessage())->with('body', $body);
return view('error')
->with('message', $e->getMessage())
->with('body', $body);
}
try {
$data = json_decode((string) $response->getBody(), true, 512, JSON_THROW_ON_ERROR);
$data = json_decode(
(string) $response->getBody(),
true,
512,
JSON_THROW_ON_ERROR
);
} catch (\JsonException $e) {
app('log')->error(sprintf('JSON exception when decoding response: %s', $e->getMessage()));
app('log')->error(sprintf('Response from server: "%s"', (string) $response->getBody()));
app('log')->error(
sprintf(
'JSON exception when decoding response: %s',
$e->getMessage()
)
);
app('log')->error(
sprintf(
'Response from server: "%s"',
(string) $response->getBody()
)
);
// app('log')->error($e->getTraceAsString());
throw new ImporterErrorException(sprintf('JSON exception when decoding response: %s', $e->getMessage()));
throw new ImporterErrorException(
sprintf(
'JSON exception when decoding response: %s',
$e->getMessage()
)
);
}
app('log')->debug('Response', $data);
SecretManager::saveAccessToken((string) $data['access_token']);
SecretManager::saveAccessToken((string)$data['access_token']);
SecretManager::saveBaseUrl($baseURL);
SecretManager::saveVanityUrl($vanityURL);
SecretManager::saveRefreshToken((string) $data['refresh_token']);
SecretManager::saveRefreshToken((string)$data['refresh_token']);
app('log')->debug(sprintf('Return redirect to "%s"', route('index')));
return redirect(route('index'));
@ -135,47 +169,92 @@ class TokenController extends Controller
public function doValidate(): JsonResponse
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
$response = ['result' => 'OK', 'message' => null];
$response = ['result' => 'OK', 'message' => null];
// Check if OAuth is configured but no session token exists
$clientId = (string) config('importer.client_id');
$configToken = (string) config('importer.access_token');
// Corrected: Use the constant value directly with session helper
$sessionHasToken =
session()->has(Constants::SESSION_ACCESS_TOKEN) &&
'' !== session()->get(Constants::SESSION_ACCESS_TOKEN);
if ('' !== $clientId && '' === $configToken && !$sessionHasToken) {
app('log')->debug(
'OAuth configured but no session token - needs authentication'
);
return response()->json([
'result' => 'NEEDS_OAUTH',
'message' => 'OAuth authentication required',
]);
}
// get values from secret manager:
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$infoRequest = new SystemInformationRequest($url, $token);
$infoRequest->setVerify(config('importer.connection.verify'));
$infoRequest->setTimeOut(config('importer.connection.timeout'));
app('log')->debug(sprintf('Now trying to authenticate with Firefly III at %s', $url));
app('log')->debug(
sprintf('Now trying to authenticate with Firefly III at %s', $url)
);
try {
$result = $infoRequest->get();
} catch (ApiHttpException $e) {
app('log')->notice(sprintf('Could NOT authenticate with Firefly III at %s', $url));
app('log')->error(sprintf('Could not connect to Firefly III: %s', $e->getMessage()));
app('log')->notice(
sprintf('Could NOT authenticate with Firefly III at %s', $url)
);
app('log')->error(
sprintf(
'Could not connect to Firefly III: %s',
$e->getMessage()
)
);
return response()->json(['result' => 'NOK', 'message' => $e->getMessage()]);
return response()->json([
'result' => 'NOK',
'message' => $e->getMessage(),
]);
}
// -1 = OK (minimum is smaller)
// 0 = OK (same version)
// 1 = NOK (too low a version)
$minimum = (string) config('importer.minimum_version');
$compare = version_compare($minimum, $result->version);
$minimum = (string) config('importer.minimum_version');
$compare = version_compare($minimum, $result->version);
if (str_starts_with($result->version, 'develop')) {
// overrule compare, because the user is running a develop version
app('log')->warning(sprintf('You are connecting to a development version of Firefly III (%s). This may not work as expected.', $result->version));
app('log')->warning(
sprintf(
'You are connecting to a development version of Firefly III (%s). This may not work as expected.',
$result->version
)
);
$compare = -1;
}
if (str_starts_with($result->version, 'branch')) {
// overrule compare, because the user is running a branch version
app('log')->warning(sprintf('You are connecting to a branch version of Firefly III (%s). This may not work as expected.', $result->version));
app('log')->warning(
sprintf(
'You are connecting to a branch version of Firefly III (%s). This may not work as expected.',
$result->version
)
);
$compare = -1;
}
if (str_starts_with($result->version, 'branch')) {
// overrule compare, because the user is running a develop version
app('log')->warning(sprintf('You are connecting to a branch version of Firefly III (%s). This may not work as expected.', $result->version));
app('log')->warning(
sprintf(
'You are connecting to a branch version of Firefly III (%s). This may not work as expected.',
$result->version
)
);
$compare = -1;
}
@ -185,8 +264,10 @@ class TokenController extends Controller
$result->version,
$minimum
);
app('log')->error(sprintf('Could not link to Firefly III: %s', $errorMessage));
$response = ['result' => 'NOK', 'message' => $errorMessage];
app('log')->error(
sprintf('Could not link to Firefly III: %s', $errorMessage)
);
$response = ['result' => 'NOK', 'message' => $errorMessage];
}
app('log')->debug('Result is', $response);
@ -204,23 +285,33 @@ class TokenController extends Controller
*/
public function index(Request $request)
{
$pageTitle = 'Data importer';
$pageTitle = 'Data importer';
app('log')->debug(sprintf('Now at %s', __METHOD__));
$accessToken = SecretManager::getAccessToken();
$clientId = SecretManager::getClientId();
$baseUrl = SecretManager::getBaseUrl();
$vanityUrl = SecretManager::getVanityUrl();
$clientId = SecretManager::getClientId();
$baseUrl = SecretManager::getBaseUrl();
$vanityUrl = SecretManager::getVanityUrl();
app('log')->info('The following configuration information was found:');
app('log')->info(sprintf('Personal Access Token: "%s" (limited to 25 chars if present)', substr($accessToken, 0, 25)));
app('log')->info(
sprintf(
'Personal Access Token: "%s" (limited to 25 chars if present)',
substr($accessToken, 0, 25)
)
);
app('log')->info(sprintf('Client ID : "%s"', $clientId));
app('log')->info(sprintf('Base URL : "%s"', $baseUrl));
app('log')->info(sprintf('Vanity URL : "%s"', $vanityUrl));
// Option 1: access token and url are present:
if ('' !== $accessToken && '' !== $baseUrl) {
app('log')->debug(sprintf('Found personal access token + URL "%s" in config, set cookie and return to index.', $baseUrl));
app('log')->debug(
sprintf(
'Found personal access token + URL "%s" in config, set cookie and return to index.',
$baseUrl
)
);
SecretManager::saveAccessToken($accessToken);
SecretManager::saveBaseUrl($baseUrl);
@ -232,41 +323,71 @@ class TokenController extends Controller
// Option 2: client ID + base URL.
if (0 !== $clientId && '' !== $baseUrl) {
app('log')->debug(sprintf('Found client ID "%d" + URL "%s" in config, redirect to Firefly III for permission.', $clientId, $baseUrl));
app('log')->debug(
sprintf(
'Found client ID "%d" + URL "%s" in config, redirect to Firefly III for permission.',
$clientId,
$baseUrl
)
);
return $this->redirectForPermission($request, $baseUrl, $vanityUrl, $clientId);
return $this->redirectForPermission(
$request,
$baseUrl,
$vanityUrl,
$clientId
);
}
// Option 3: either is empty, ask for client ID and/or base URL:
$clientId = 0 === $clientId ? '' : $clientId;
$clientId = 0 === $clientId ? '' : $clientId;
// if the vanity url is the same as the base url, just give this view an empty string
if ($vanityUrl === $baseUrl) {
$vanityUrl = '';
}
return view('token.client_id', compact('baseUrl', 'vanityUrl', 'clientId', 'pageTitle'));
return view(
'token.client_id',
compact('baseUrl', 'vanityUrl', 'clientId', 'pageTitle')
);
}
/**
* This method forwards the user to Firefly III. Some parameters are stored in the user's session.
*/
private function redirectForPermission(Request $request, string $baseURL, string $vanityURL, int $clientId): RedirectResponse
{
$baseURL = rtrim($baseURL, '/');
$vanityURL = rtrim($vanityURL, '/');
app('log')->debug(sprintf('Now in %s(request, "%s", "%s", %d)', __METHOD__, $baseURL, $vanityURL, $clientId));
$state = \Str::random(40);
$codeVerifier = \Str::random(128);
private function redirectForPermission(
Request $request,
string $baseURL,
string $vanityURL,
int $clientId
): RedirectResponse {
$baseURL = rtrim($baseURL, '/');
$vanityURL = rtrim($vanityURL, '/');
app('log')->debug(
sprintf(
'Now in %s(request, "%s", "%s", %d)',
__METHOD__,
$baseURL,
$vanityURL,
$clientId
)
);
$state = \Str::random(40);
$codeVerifier = \Str::random(128);
$request->session()->put('state', $state);
$request->session()->put('code_verifier', $codeVerifier);
$request->session()->put('form_client_id', $clientId);
$request->session()->put('form_base_url', $baseURL);
$request->session()->put('form_vanity_url', $vanityURL);
$codeChallenge = strtr(rtrim(base64_encode(hash('sha256', $codeVerifier, true)), '='), '+/', '-_');
$params = [
$codeChallenge = strtr(
rtrim(base64_encode(hash('sha256', $codeVerifier, true)), '='),
'+/',
'-_'
);
$params = [
'client_id' => $clientId,
'redirect_uri' => route('token.callback'),
'response_type' => 'code',
@ -275,13 +396,27 @@ class TokenController extends Controller
'code_challenge' => $codeChallenge,
'code_challenge_method' => 'S256',
];
$query = http_build_query($params);
// we redirect the user to the vanity URL, which is the same as the base_url, unless the user actually set a vanity URL.
$finalURL = sprintf('%s/oauth/authorize?', $vanityURL);
$query = http_build_query($params);
$oauthAuthorizeBaseUrl = $vanityURL;
// Ensure $oauthAuthorizeBaseUrl is not empty before sprintf.
// This is a fallback in case vanity_url (derived from VANITY_URL) is empty,
// which would indicate a configuration problem.
if (empty($oauthAuthorizeBaseUrl)) {
$oauthAuthorizeBaseUrl = rtrim(
(string) config('importer.url'),
'/'
);
}
$finalURL = sprintf('%s/oauth/authorize?', $oauthAuthorizeBaseUrl);
app('log')->debug('Query parameters are', $params);
app('log')->debug(sprintf('Now redirecting to "%s" (params omitted)', $finalURL));
app('log')->debug(
sprintf('Now redirecting to "%s" (params omitted)', $finalURL)
);
return redirect($finalURL.$query);
return redirect($finalURL . $query);
}
/**
@ -293,46 +428,77 @@ class TokenController extends Controller
public function submitClientId(Request $request)
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
$data = $request->validate(
[
'client_id' => 'required|numeric|min:1|max:65536',
'base_url' => 'url',
]
);
$data = $request->validate([
'client_id' => 'required|numeric|min:1|max:65536',
'base_url' => 'url',
]);
app('log')->debug('Submitted data: ', $data);
if (true === config('importer.expect_secure_url') && array_key_exists('base_url', $data) && !str_starts_with($data['base_url'], 'https://')) {
$request->session()->flash('secure_url', 'URL must start with https://');
if (
true === config('importer.expect_secure_url') &&
array_key_exists('base_url', $data) &&
!str_starts_with($data['base_url'], 'https://')
) {
$request
->session()
->flash('secure_url', 'URL must start with https://');
return redirect(route('token.index'));
}
$data['client_id'] = (int) $data['client_id'];
$data['client_id'] = (int)$data['client_id'];
// grab base URL from config first, otherwise from submitted data:
$baseURL = config('importer.url');
app('log')->debug(sprintf('[a] Base URL is "%s" (based on "FIREFLY_III_URL")', $baseURL));
$vanityURL = $baseURL;
$baseURL = config('importer.url');
app('log')->debug(
sprintf(
'[a] Base URL is "%s" (based on "FIREFLY_III_URL")',
$baseURL
)
);
$vanityURL = $baseURL;
app('log')->debug(sprintf('[b] Vanity URL is now "%s" (based on "FIREFLY_III_URL")', $vanityURL));
app('log')->debug(
sprintf(
'[b] Vanity URL is now "%s" (based on "FIREFLY_III_URL")',
$vanityURL
)
);
// if the config has a vanity URL it will always overrule.
if ('' !== (string) config('importer.vanity_url')) {
$vanityURL = config('importer.vanity_url');
app('log')->debug(sprintf('[c] Vanity URL is now "%s" (based on "VANITY_URL")', $vanityURL));
app('log')->debug(
sprintf(
'[c] Vanity URL is now "%s" (based on "VANITY_URL")',
$vanityURL
)
);
}
// otherwise take base URL from the submitted data:
if (array_key_exists('base_url', $data) && '' !== $data['base_url']) {
$baseURL = $data['base_url'];
app('log')->debug(sprintf('[d] Base URL is now "%s" (from POST data)', $baseURL));
app('log')->debug(
sprintf('[d] Base URL is now "%s" (from POST data)', $baseURL)
);
}
if ('' === (string) $vanityURL) {
$vanityURL = $baseURL;
app('log')->debug(sprintf('[e] Vanity URL is now "%s" (from base URL)', $vanityURL));
app('log')->debug(
sprintf(
'[e] Vanity URL is now "%s" (from base URL)',
$vanityURL
)
);
}
// return request for permission:
return $this->redirectForPermission($request, $baseURL, $vanityURL, $data['client_id']);
return $this->redirectForPermission(
$request,
$baseURL,
$vanityURL,
$data['client_id']
);
}
}

38
app/Http/Middleware/ConversionControllerMiddleware.php

@ -25,6 +25,9 @@ declare(strict_types=1);
namespace App\Http\Middleware;
use App\Services\Session\Constants;
use Illuminate\Http\Request;
/**
* Class ConversionControllerMiddleware
*/
@ -33,4 +36,39 @@ class ConversionControllerMiddleware
use IsReadyForStep;
protected const STEP = 'conversion';
protected function isReadyForStep(Request $request): bool
{
$flow = $request->cookie(Constants::FLOW_COOKIE);
// Call trait logic directly since we can't use parent:: with traits
if (null === $flow) {
app('log')->debug(
'isReadyForStep returns true because $flow is null'
);
return true;
}
if ('file' === $flow) {
$result = $this->isReadyForFileStep();
app('log')->debug(
sprintf(
'isReadyForFileStep: Return %s',
var_export($result, true)
)
);
return $result;
}
if ('nordigen' === $flow) {
return $this->isReadyForNordigenStep();
}
if ('spectre' === $flow) {
return $this->isReadyForSpectreStep();
}
if ('simplefin' === $flow) {
return $this->isReadyForSimpleFINStep();
}
return $this->isReadyForBasicStep();
}
}

479
app/Http/Middleware/IsReadyForStep.php

@ -39,7 +39,7 @@ trait IsReadyForStep
public function handle(Request $request, \Closure $next): mixed
{
$result = $this->isReadyForStep($request);
$result = $this->isReadyForStep($request);
if (true === $result) {
return $next($request);
}
@ -49,21 +49,30 @@ trait IsReadyForStep
return $redirect;
}
throw new ImporterErrorException(sprintf('Cannot handle middleware: %s', self::STEP));
throw new ImporterErrorException(
sprintf('Cannot handle middleware: %s', self::STEP)
);
}
protected function isReadyForStep(Request $request): bool
{
$flow = $request->cookie(Constants::FLOW_COOKIE);
if (null === $flow) {
app('log')->debug('isReadyForStep returns true because $flow is null');
app('log')->debug(
'isReadyForStep returns true because $flow is null'
);
return true;
}
// TODO this flow is weird.
if ('file' === $flow) {
$result = $this->isReadyForFileStep();
app('log')->debug(sprintf('isReadyForFileStep: Return %s', var_export($result, true)));
app('log')->debug(
sprintf(
'isReadyForFileStep: Return %s',
var_export($result, true)
)
);
return $result;
}
@ -86,13 +95,21 @@ trait IsReadyForStep
switch (self::STEP) {
default:
throw new ImporterErrorException(sprintf('isReadyForFileStep: Cannot handle file step "%s"', self::STEP));
throw new ImporterErrorException(
sprintf(
'isReadyForFileStep: Cannot handle file step "%s"',
self::STEP
)
);
case 'service-validation':
return true;
case 'upload-files':
if (session()->has(Constants::HAS_UPLOAD) && true === session()->get(Constants::HAS_UPLOAD)) {
if (
session()->has(Constants::HAS_UPLOAD) &&
true === session()->get(Constants::HAS_UPLOAD)
) {
return false;
}
@ -103,29 +120,53 @@ trait IsReadyForStep
return false;
case 'define-roles':
if (session()->has(Constants::ROLES_COMPLETE_INDICATOR) && true === session()->get(Constants::ROLES_COMPLETE_INDICATOR)) {
if (
session()->has(Constants::ROLES_COMPLETE_INDICATOR) &&
true === session()->get(Constants::ROLES_COMPLETE_INDICATOR)
) {
return false;
}
return true;
case 'configuration':
if (session()->has(Constants::CONFIG_COMPLETE_INDICATOR) && true === session()->get(Constants::CONFIG_COMPLETE_INDICATOR)) {
if (
session()->has(Constants::CONFIG_COMPLETE_INDICATOR) &&
true ===
session()->get(Constants::CONFIG_COMPLETE_INDICATOR)
) {
return false;
}
return true;
case 'map':
if (session()->has(Constants::MAPPING_COMPLETE_INDICATOR) && true === session()->get(Constants::MAPPING_COMPLETE_INDICATOR)) {
if (
session()->has(Constants::MAPPING_COMPLETE_INDICATOR) &&
true ===
session()->get(Constants::MAPPING_COMPLETE_INDICATOR)
) {
return false;
}
return true;
case 'conversion':
$hasReadyFlag = session()->has(Constants::READY_FOR_CONVERSION);
$readyValue = session()->get(Constants::READY_FOR_CONVERSION);
$hasConfigComplete = session()->has(
Constants::CONFIG_COMPLETE_INDICATOR
);
$configCompleteValue = session()->get(
Constants::CONFIG_COMPLETE_INDICATOR
);
$hasSimpleFINData = session()->has(
Constants::SIMPLEFIN_ACCOUNTS_DATA
);
// if/else is in reverse!
if (session()->has(Constants::READY_FOR_CONVERSION) && true === session()->get(Constants::READY_FOR_CONVERSION)) {
if ($hasReadyFlag && true === $readyValue) {
return true;
}
@ -134,7 +175,11 @@ trait IsReadyForStep
case 'submit':
// if/else is in reverse!
if (session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) && true === session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)) {
if (
session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) &&
true ===
session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)
) {
return true;
}
@ -144,10 +189,15 @@ trait IsReadyForStep
private function isReadyForSimpleFINStep(): bool
{
// app('log')->debug(sprintf('isReadyForNordigenStep("%s")', self::STEP));
// app('log')->debug(sprintf('isReadyForSimpleFINStep("%s")', self::STEP));
switch (self::STEP) {
default:
throw new ImporterErrorException(sprintf('isReadyForSimpleFINStep: Cannot handle SimpleFIN step "%s"', self::STEP));
throw new ImporterErrorException(
sprintf(
'isReadyForSimpleFINStep: Cannot handle SimpleFIN step "%s"',
self::STEP
)
);
case 'authenticate':
// simpleFIN needs no authentication.
@ -156,6 +206,73 @@ trait IsReadyForStep
case 'upload-files':
// you can always upload SimpleFIN things
return true;
case 'configuration':
return session()->has(Constants::HAS_UPLOAD) &&
session()->has(Constants::SIMPLEFIN_ACCOUNTS_DATA);
case 'define-roles':
// SimpleFIN should bypass roles if ready for conversion
if (
session()->has(Constants::READY_FOR_CONVERSION) &&
true === session()->get(Constants::READY_FOR_CONVERSION)
) {
return false;
}
if (
session()->has(Constants::ROLES_COMPLETE_INDICATOR) &&
true === session()->get(Constants::ROLES_COMPLETE_INDICATOR)
) {
return false;
}
return true;
case 'map':
if (
session()->has(Constants::MAPPING_COMPLETE_INDICATOR) &&
true ===
session()->get(Constants::MAPPING_COMPLETE_INDICATOR)
) {
return false;
}
// Ready for mapping if conversion is complete
if (
session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) &&
true === session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)
) {
app('log')->debug('SimpleFIN: Conversion complete, ready for mapping');
return true;
}
app('log')->debug('SimpleFIN: Conversion not complete, not ready for mapping');
return false;
case 'conversion':
// if/else is in reverse!
if (
session()->has(Constants::READY_FOR_CONVERSION) &&
true === session()->get(Constants::READY_FOR_CONVERSION)
) {
return true;
}
// will probably never return false, but OK.
return false;
case 'submit':
// if/else is in reverse!
if (
session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) &&
true ===
session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)
) {
return true;
}
return false;
}
return false;
@ -166,7 +283,12 @@ trait IsReadyForStep
// app('log')->debug(sprintf('isReadyForNordigenStep("%s")', self::STEP));
switch (self::STEP) {
default:
throw new ImporterErrorException(sprintf('isReadyForNordigenStep: Cannot handle Nordigen step "%s"', self::STEP));
throw new ImporterErrorException(
sprintf(
'isReadyForNordigenStep: Cannot handle Nordigen step "%s"',
self::STEP
)
);
case 'authenticate':
case 'service-validation':
@ -176,7 +298,10 @@ trait IsReadyForStep
return false;
case 'upload-files':
if (session()->has(Constants::HAS_UPLOAD) && true === session()->get(Constants::HAS_UPLOAD)) {
if (
session()->has(Constants::HAS_UPLOAD) &&
true === session()->get(Constants::HAS_UPLOAD)
) {
return false;
}
@ -184,7 +309,10 @@ trait IsReadyForStep
case 'nordigen-selection':
// must have upload, that's it
if (session()->has(Constants::HAS_UPLOAD) && true === session()->get(Constants::HAS_UPLOAD)) {
if (
session()->has(Constants::HAS_UPLOAD) &&
true === session()->get(Constants::HAS_UPLOAD)
) {
return true;
}
@ -192,22 +320,35 @@ trait IsReadyForStep
case 'map':
// mapping must be complete, or not ready for this step.
if (session()->has(Constants::MAPPING_COMPLETE_INDICATOR) && true === session()->get(Constants::MAPPING_COMPLETE_INDICATOR)) {
if (
session()->has(Constants::MAPPING_COMPLETE_INDICATOR) &&
true ===
session()->get(Constants::MAPPING_COMPLETE_INDICATOR)
) {
app('log')->debug('Return false, not ready for step [1].');
return false;
}
// conversion complete?
if (session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) && true === session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)) {
if (
session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) &&
true ===
session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)
) {
app('log')->debug('Return true, ready for step [4].');
return true;
}
// must already have the conversion, or not ready for this step:
if (session()->has(Constants::READY_FOR_CONVERSION) && true === session()->get(Constants::READY_FOR_CONVERSION)) {
app('log')->debug('GoCardless: return false, not yet ready for step [2].');
if (
session()->has(Constants::READY_FOR_CONVERSION) &&
true === session()->get(Constants::READY_FOR_CONVERSION)
) {
app('log')->debug(
'GoCardless: return false, not yet ready for step [2].'
);
return false;
}
@ -218,20 +359,29 @@ trait IsReadyForStep
case 'nordigen-link':
// must have upload, thats it
if (session()->has(Constants::SELECTED_BANK_COUNTRY) && true === session()->get(Constants::SELECTED_BANK_COUNTRY)) {
if (
session()->has(Constants::SELECTED_BANK_COUNTRY) &&
true === session()->get(Constants::SELECTED_BANK_COUNTRY)
) {
return true;
}
return false;
case 'conversion':
if (session()->has(Constants::READY_FOR_SUBMISSION) && true === session()->get(Constants::READY_FOR_SUBMISSION)) {
if (
session()->has(Constants::READY_FOR_SUBMISSION) &&
true === session()->get(Constants::READY_FOR_SUBMISSION)
) {
app('log')->debug('Return false, ready for submission.');
return false;
}
// if/else is in reverse!
if (session()->has(Constants::READY_FOR_CONVERSION) && true === session()->get(Constants::READY_FOR_CONVERSION)) {
if (
session()->has(Constants::READY_FOR_CONVERSION) &&
true === session()->get(Constants::READY_FOR_CONVERSION)
) {
return true;
}
@ -239,7 +389,10 @@ trait IsReadyForStep
return false;
case 'configuration':
if (session()->has(Constants::SELECTED_BANK_COUNTRY) && true === session()->get(Constants::SELECTED_BANK_COUNTRY)) {
if (
session()->has(Constants::SELECTED_BANK_COUNTRY) &&
true === session()->get(Constants::SELECTED_BANK_COUNTRY)
) {
return true;
}
@ -247,7 +400,11 @@ trait IsReadyForStep
case 'submit':
// if/else is in reverse!
if (session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) && true === session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)) {
if (
session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) &&
true ===
session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)
) {
return true;
}
@ -261,20 +418,33 @@ trait IsReadyForStep
switch (self::STEP) {
default:
throw new ImporterErrorException(sprintf('isReadyForSpectreStep: Cannot handle Spectre step "%s"', self::STEP));
throw new ImporterErrorException(
sprintf(
'isReadyForSpectreStep: Cannot handle Spectre step "%s"',
self::STEP
)
);
case 'service-validation':
case 'authenticate':
return true;
case 'conversion':
if (session()->has(Constants::READY_FOR_SUBMISSION) && true === session()->get(Constants::READY_FOR_SUBMISSION)) {
app('log')->debug('Spectre: Return false, ready for submission.');
if (
session()->has(Constants::READY_FOR_SUBMISSION) &&
true === session()->get(Constants::READY_FOR_SUBMISSION)
) {
app('log')->debug(
'Spectre: Return false, ready for submission.'
);
return false;
}
// if/else is in reverse!
if (session()->has(Constants::READY_FOR_CONVERSION) && true === session()->get(Constants::READY_FOR_CONVERSION)) {
if (
session()->has(Constants::READY_FOR_CONVERSION) &&
true === session()->get(Constants::READY_FOR_CONVERSION)
) {
return true;
}
@ -282,21 +452,31 @@ trait IsReadyForStep
return false;
case 'upload-files':
if (session()->has(Constants::HAS_UPLOAD) && true === session()->get(Constants::HAS_UPLOAD)) {
if (
session()->has(Constants::HAS_UPLOAD) &&
true === session()->get(Constants::HAS_UPLOAD)
) {
return false;
}
return true;
case 'select-connection':
if (session()->has(Constants::HAS_UPLOAD) && true === session()->get(Constants::HAS_UPLOAD)) {
if (
session()->has(Constants::HAS_UPLOAD) &&
true === session()->get(Constants::HAS_UPLOAD)
) {
return true;
}
return false;
case 'configuration':
if (session()->has(Constants::CONNECTION_SELECTED_INDICATOR) && true === session()->get(Constants::CONNECTION_SELECTED_INDICATOR)) {
if (
session()->has(Constants::CONNECTION_SELECTED_INDICATOR) &&
true ===
session()->get(Constants::CONNECTION_SELECTED_INDICATOR)
) {
return true;
}
@ -307,22 +487,39 @@ trait IsReadyForStep
case 'map':
// mapping must be complete, or not ready for this step.
if (session()->has(Constants::MAPPING_COMPLETE_INDICATOR) && true === session()->get(Constants::MAPPING_COMPLETE_INDICATOR)) {
app('log')->debug('Spectre: Return false, not ready for step [1].');
if (
session()->has(Constants::MAPPING_COMPLETE_INDICATOR) &&
true ===
session()->get(Constants::MAPPING_COMPLETE_INDICATOR)
) {
app('log')->debug(
'Spectre: Return false, not ready for step [1].'
);
return false;
}
// conversion complete?
if (session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) && true === session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)) {
app('log')->debug('Spectre: Return true, ready for step [4].');
if (
session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) &&
true ===
session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)
) {
app('log')->debug(
'Spectre: Return true, ready for step [4].'
);
return true;
}
// must already have the conversion, or not ready for this step:
if (session()->has(Constants::READY_FOR_CONVERSION) && true === session()->get(Constants::READY_FOR_CONVERSION)) {
app('log')->debug('Spectre: Return false, not yet ready for step [2].');
if (
session()->has(Constants::READY_FOR_CONVERSION) &&
true === session()->get(Constants::READY_FOR_CONVERSION)
) {
app('log')->debug(
'Spectre: Return false, not yet ready for step [2].'
);
return false;
}
@ -333,7 +530,11 @@ trait IsReadyForStep
case 'submit':
// if/else is in reverse!
if (session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) && true === session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)) {
if (
session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) &&
true ===
session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)
) {
return true;
}
@ -352,14 +553,22 @@ trait IsReadyForStep
return true;
}
throw new ImporterErrorException(sprintf('isReadyForBasicStep: Cannot handle basic step "%s"', self::STEP));
throw new ImporterErrorException(
sprintf(
'isReadyForBasicStep: Cannot handle basic step "%s"',
self::STEP
)
);
}
protected function redirectToCorrectStep(Request $request): ?RedirectResponse
{
protected function redirectToCorrectStep(
Request $request
): ?RedirectResponse {
$flow = $request->cookie(Constants::FLOW_COOKIE);
if (null === $flow) {
app('log')->debug('redirectToCorrectStep returns NULL because $flow is null');
app('log')->debug(
'redirectToCorrectStep returns NULL because $flow is null'
);
return null;
}
@ -384,11 +593,18 @@ trait IsReadyForStep
*/
private function redirectToCorrectFileStep(): RedirectResponse
{
app('log')->debug(sprintf('redirectToCorrectFileStep("%s")', self::STEP));
app('log')->debug(
sprintf('redirectToCorrectFileStep("%s")', self::STEP)
);
switch (self::STEP) {
default:
throw new ImporterErrorException(sprintf('redirectToCorrectFileStep: Cannot handle file step "%s"', self::STEP));
throw new ImporterErrorException(
sprintf(
'redirectToCorrectFileStep: Cannot handle file step "%s"',
self::STEP
)
);
case 'upload-files':
$route = route('004-configure.index');
@ -415,7 +631,20 @@ trait IsReadyForStep
return redirect($route);
case 'conversion':
// redirect to mapping
// Check authoritative session flow before defaulting to file-based redirection
$authoritativeFlow = null;
$sessionConfig = session()->get(Constants::CONFIGURATION);
if (is_array($sessionConfig) && isset($sessionConfig['flow'])) {
$authoritativeFlow = $sessionConfig['flow'];
}
// If authoritative flow is SimpleFIN, redirect to configure instead of mapping
if ('simplefin' === $authoritativeFlow) {
$route = route('004-configure.index');
return redirect($route);
}
// Default file-based behavior: redirect to mapping
$route = route('006-mapping.index');
app('log')->debug(sprintf('Return redirect to "%s"', $route));
@ -438,11 +667,18 @@ trait IsReadyForStep
private function redirectToCorrectNordigenStep(): RedirectResponse
{
app('log')->debug(sprintf('redirectToCorrectNordigenStep("%s")', self::STEP));
app('log')->debug(
sprintf('redirectToCorrectNordigenStep("%s")', self::STEP)
);
switch (self::STEP) {
default:
throw new ImporterErrorException(sprintf('redirectToCorrectNordigenStep: Cannot handle Nordigen step "%s"', self::STEP));
throw new ImporterErrorException(
sprintf(
'redirectToCorrectNordigenStep: Cannot handle Nordigen step "%s"',
self::STEP
)
);
case 'nordigen-selection':
// back to upload
@ -471,10 +707,17 @@ trait IsReadyForStep
case 'map':
// if no conversion yet, go there first
// must already have the conversion, or not ready for this step:
if (session()->has(Constants::READY_FOR_CONVERSION) && true === session()->get(Constants::READY_FOR_CONVERSION)) {
app('log')->debug('Is ready for conversion, so send to conversion.');
if (
session()->has(Constants::READY_FOR_CONVERSION) &&
true === session()->get(Constants::READY_FOR_CONVERSION)
) {
app('log')->debug(
'Is ready for conversion, so send to conversion.'
);
$route = route('007-convert.index');
app('log')->debug(sprintf('Return redirect to "%s"', $route));
app('log')->debug(
sprintf('Return redirect to "%s"', $route)
);
return redirect($route);
}
@ -486,14 +729,24 @@ trait IsReadyForStep
return redirect($route);
case 'conversion':
if (session()->has(Constants::READY_FOR_SUBMISSION) && true === session()->get(Constants::READY_FOR_SUBMISSION)) {
if (
session()->has(Constants::READY_FOR_SUBMISSION) &&
true === session()->get(Constants::READY_FOR_SUBMISSION)
) {
$route = route('008-submit.index');
app('log')->debug(sprintf('Return redirect to "%s"', $route));
app('log')->debug(
sprintf('Return redirect to "%s"', $route)
);
return redirect($route);
}
throw new ImporterErrorException(sprintf('redirectToCorrectNordigenStep: Cannot handle Nordigen step "%s" [1]', self::STEP));
throw new ImporterErrorException(
sprintf(
'redirectToCorrectNordigenStep: Cannot handle Nordigen step "%s" [1]',
self::STEP
)
);
case 'submit':
$route = route('007-convert.index');
@ -505,11 +758,18 @@ trait IsReadyForStep
private function redirectToCorrectSpectreStep(): RedirectResponse
{
app('log')->debug(sprintf('redirectToCorrectSpectreStep("%s")', self::STEP));
app('log')->debug(
sprintf('redirectToCorrectSpectreStep("%s")', self::STEP)
);
switch (self::STEP) {
default:
throw new ImporterErrorException(sprintf('redirectToCorrectSpectreStep: Cannot handle basic step "%s"', self::STEP));
throw new ImporterErrorException(
sprintf(
'redirectToCorrectSpectreStep: Cannot handle basic step "%s"',
self::STEP
)
);
case 'upload-files':
// assume files are uploaded, go to step 11 (connection selection)
@ -530,24 +790,38 @@ trait IsReadyForStep
case 'map':
// if no conversion yet, go there first
// must already have the conversion, or not ready for this step:
if (session()->has(Constants::READY_FOR_CONVERSION) && true === session()->get(Constants::READY_FOR_CONVERSION)) {
app('log')->debug('Spectre: Is ready for conversion, so send to conversion.');
if (
session()->has(Constants::READY_FOR_CONVERSION) &&
true === session()->get(Constants::READY_FOR_CONVERSION)
) {
app('log')->debug(
'Spectre: Is ready for conversion, so send to conversion.'
);
$route = route('007-convert.index');
app('log')->debug(sprintf('Spectre: Return redirect to "%s"', $route));
app('log')->debug(
sprintf('Spectre: Return redirect to "%s"', $route)
);
return redirect($route);
}
app('log')->debug('Spectre: Is ready for submit.');
// otherwise go to import right away
$route = route('008-submit.index');
app('log')->debug(sprintf('Spectre: Return redirect to "%s"', $route));
app('log')->debug(
sprintf('Spectre: Return redirect to "%s"', $route)
);
return redirect($route);
case 'conversion':
if (session()->has(Constants::READY_FOR_SUBMISSION) && true === session()->get(Constants::READY_FOR_SUBMISSION)) {
if (
session()->has(Constants::READY_FOR_SUBMISSION) &&
true === session()->get(Constants::READY_FOR_SUBMISSION)
) {
$route = route('008-submit.index');
app('log')->debug(sprintf('Return redirect to "%s"', $route));
app('log')->debug(
sprintf('Return redirect to "%s"', $route)
);
return redirect($route);
}
@ -556,17 +830,85 @@ trait IsReadyForStep
private function redirectToCorrectSimpleFINStep(): RedirectResponse
{
app('log')->debug(sprintf('redirectToCorrectSimpleFINStep("%s")', self::STEP));
app('log')->debug(
sprintf('redirectToCorrectSimpleFINStep("%s")', self::STEP)
);
switch (self::STEP) {
default:
throw new ImporterErrorException(sprintf('redirectToCorrectSpectreStep: Cannot handle basic step "%s"', self::STEP));
throw new ImporterErrorException(
sprintf(
'redirectToCorrectSimpleFINStep: Cannot handle SimpleFIN step "%s"',
self::STEP
)
);
case 'authenticate':
// simple fin does not authenticate, redirect to upload step.
$route = route('003-upload.index');
app('log')->debug(sprintf('SimpleFIN: Return redirect to "%s"', $route));
app('log')->debug(
sprintf('SimpleFIN: Return redirect to "%s"', $route)
);
return redirect($route);
case 'configuration':
app('log')->debug(
sprintf(
'SimpleFIN: Not ready for configuration (STEP: "%s"), redirecting to upload.',
self::STEP
)
);
return redirect(route('003-upload.index'));
case 'define-roles':
$route = route('007-convert.index');
app('log')->debug(
sprintf('SimpleFIN: Return redirect to "%s"', $route)
);
return redirect($route);
case 'map':
// if conversion not complete yet, go there first
if (
!session()->has(Constants::CONVERSION_COMPLETE_INDICATOR) ||
true !== session()->get(Constants::CONVERSION_COMPLETE_INDICATOR)
) {
app('log')->debug(
'SimpleFIN: Conversion not complete, redirecting to conversion.'
);
$route = route('007-convert.index');
app('log')->debug(
sprintf('SimpleFIN: Return redirect to "%s"', $route)
);
return redirect($route);
}
app('log')->debug('SimpleFIN: Conversion complete, redirecting to configuration.');
// conversion complete but no mapping data yet, redirect to configuration
$route = route('004-configure.index');
app('log')->debug(
sprintf('SimpleFIN: Return redirect to "%s"', $route)
);
return redirect($route);
case 'conversion':
// This case is reached if isReadyForSimpleFINStep() returned false for 'conversion',
// meaning Constants::READY_FOR_CONVERSION was not true.
// The user should be redirected to the configuration step, which is the prerequisite.
app('log')->debug(
sprintf(
'SimpleFIN: Not ready for conversion (STEP: "%s"), redirecting to configuration.',
self::STEP
)
);
$route = route('004-configure.index');
return redirect($route);
case 'submit':
$route = route('007-convert.index');
app('log')->debug(
sprintf('SimpleFIN: Return redirect to "%s"', $route)
);
return redirect($route);
}
}
@ -580,7 +922,12 @@ trait IsReadyForStep
switch (self::STEP) {
default:
throw new ImporterErrorException(sprintf('redirectToBasicStep: Cannot handle basic step "%s"', self::STEP));
throw new ImporterErrorException(
sprintf(
'redirectToBasicStep: Cannot handle basic step "%s"',
self::STEP
)
);
}
}
}

236
app/Http/Request/ConfigurationPostRequest.php

@ -43,16 +43,71 @@ class ConfigurationPostRequest extends Request
public function getAll(): array
{
// Debug: Log raw form data before processing
app('log')->debug('DEBUG: ConfigurationPostRequest raw form data', [
'do_import_raw' => $this->get('do_import') ?? [],
'accounts_raw' => $this->get('accounts') ?? [],
'new_account_raw' => $this->get('new_account') ?? [],
]);
// Decode underscore-encoded account IDs back to original IDs with spaces
$doImport = $this->get('do_import') ?? [];
$accounts = $this->get('accounts') ?? [];
$newAccount = $this->get('new_account') ?? [];
$decodedDoImport = [];
$decodedAccounts = [];
$decodedNewAccount = [];
// Decode do_import array keys
foreach ($doImport as $encodedId => $value) {
$originalId = str_replace('_', ' ', $encodedId);
$decodedDoImport[$originalId] = $value;
app('log')->debug('DEBUG: Decoded do_import', [
'encoded' => $encodedId,
'decoded' => $originalId,
'value' => $value,
]);
}
// Decode accounts array keys
foreach ($accounts as $encodedId => $value) {
$originalId = str_replace('_', ' ', $encodedId);
$decodedAccounts[$originalId] = $value;
app('log')->debug('DEBUG: Decoded accounts', [
'encoded' => $encodedId,
'decoded' => $originalId,
'value' => $value,
]);
}
// Decode new_account array keys
foreach ($newAccount as $encodedId => $accountData) {
$originalId = str_replace('_', ' ', $encodedId);
$decodedNewAccount[$originalId] = $accountData;
app('log')->debug('DEBUG: Decoded new_account', [
'encoded' => $encodedId,
'decoded' => $originalId,
'data' => $accountData,
]);
}
return [
'headers' => $this->convertBoolean($this->get('headers')),
'delimiter' => $this->convertToString('delimiter'),
'date' => $this->convertToString('date'),
'default_account' => $this->convertToInteger('default_account'),
'rules' => $this->convertBoolean($this->get('rules')),
'ignore_duplicate_lines' => $this->convertBoolean($this->get('ignore_duplicate_lines')),
'ignore_duplicate_transactions' => $this->convertBoolean($this->get('ignore_duplicate_transactions')),
'ignore_duplicate_lines' => $this->convertBoolean(
$this->get('ignore_duplicate_lines')
),
'ignore_duplicate_transactions' => $this->convertBoolean(
$this->get('ignore_duplicate_transactions')
),
'skip_form' => $this->convertBoolean($this->get('skip_form')),
'add_import_tag' => $this->convertBoolean($this->get('add_import_tag')),
'add_import_tag' => $this->convertBoolean(
$this->get('add_import_tag')
),
'specifics' => [],
'roles' => [],
'mapping' => [],
@ -62,53 +117,85 @@ class ConfigurationPostRequest extends Request
'custom_tag' => $this->convertToString('custom_tag'),
// duplicate detection:
'duplicate_detection_method' => $this->convertToString('duplicate_detection_method'),
'unique_column_index' => $this->convertToInteger('unique_column_index'),
'unique_column_type' => $this->convertToString('unique_column_type'),
'duplicate_detection_method' => $this->convertToString(
'duplicate_detection_method'
),
'unique_column_index' => $this->convertToInteger(
'unique_column_index'
),
'unique_column_type' => $this->convertToString(
'unique_column_type'
),
// spectre values:
'connection' => $this->convertToString('connection'),
'identifier' => $this->convertToString('identifier'),
'ignore_spectre_categories' => $this->convertBoolean($this->get('ignore_spectre_categories')),
'connection' => $this->convertToString('connection'),
'identifier' => $this->convertToString('identifier'),
'ignore_spectre_categories' => $this->convertBoolean(
$this->get('ignore_spectre_categories')
),
// nordigen:
'nordigen_country' => $this->convertToString('nordigen_country'),
'nordigen_bank' => $this->convertToString('nordigen_bank'),
'nordigen_max_days' => $this->convertToString('nordigen_max_days'),
'nordigen_requisitions' => json_decode($this->convertToString('nordigen_requisitions'), true) ?? [],
// nordigen + spectre
'do_import' => $this->get('do_import') ?? [],
'accounts' => $this->get('accounts') ?? [],
'map_all_data' => $this->convertBoolean($this->get('map_all_data')),
'date_range' => $this->convertToString('date_range'),
'date_range_number' => $this->convertToInteger('date_range_number'),
'date_range_unit' => $this->convertToString('date_range_unit'),
'date_not_before' => $this->getCarbonDate('date_not_before'),
'date_not_after' => $this->getCarbonDate('date_not_after'),
'nordigen_country' => $this->convertToString('nordigen_country'),
'nordigen_bank' => $this->convertToString('nordigen_bank'),
'nordigen_max_days' => $this->convertToString('nordigen_max_days'),
'nordigen_requisitions' =>
json_decode(
$this->convertToString('nordigen_requisitions'),
true
) ?? [],
// nordigen + spectre - with decoded account IDs
'do_import' => $decodedDoImport,
'accounts' => $decodedAccounts,
'new_account' => $decodedNewAccount,
'map_all_data' => $this->convertBoolean($this->get('map_all_data')),
'date_range' => $this->convertToString('date_range'),
'date_range_number' => $this->convertToInteger('date_range_number'),
'date_range_unit' => $this->convertToString('date_range_unit'),
'date_not_before' => $this->getCarbonDate('date_not_before'),
'date_not_after' => $this->getCarbonDate('date_not_after'),
// utf8 conversion
'conversion' => $this->convertBoolean($this->get('conversion')),
'conversion' => $this->convertBoolean($this->get('conversion')),
// camt
'grouped_transaction_handling' => $this->convertToString('grouped_transaction_handling'),
'use_entire_opposing_address' => $this->convertBoolean($this->get('use_entire_opposing_address')),
'grouped_transaction_handling' => $this->convertToString(
'grouped_transaction_handling'
),
'use_entire_opposing_address' => $this->convertBoolean(
$this->get('use_entire_opposing_address')
),
];
}
public function rules(): array
{
$flow = request()->cookie(Constants::FLOW_COOKIE);
$columnOptions = implode(',', array_keys(config('csv.unique_column_options')));
$flow = request()->cookie(Constants::FLOW_COOKIE);
$columnOptions = implode(
',',
array_keys(config('csv.unique_column_options'))
);
if ('nordigen' === $flow) {
$columnOptions = implode(',', array_keys(config('nordigen.unique_column_options')));
$columnOptions = implode(
',',
array_keys(config('nordigen.unique_column_options'))
);
}
if ('simplefin' === $flow) {
$columnOptions = implode(
',',
array_keys(config('simplefin.unique_column_options'))
);
}
return [
'headers' => 'numeric|between:0,1',
'delimiter' => 'in:comma,semicolon,tab',
'date' => 'between:1,25',
'default_account' => 'required|numeric|min:1|max:100000',
'default_account' =>
'simplefin' === $flow
? 'nullable|numeric|min:1|max:100000'
: 'required|numeric|min:1|max:100000',
'rules' => 'numeric|between:0,1',
'ignore_duplicate_lines' => 'numeric|between:0,1',
'ignore_duplicate_transactions' => 'numeric|between:0,1',
@ -124,6 +211,15 @@ class ConfigurationPostRequest extends Request
// conversion
'conversion' => 'numeric|between:0,1',
// new account creation - updated to handle underscore-encoded field names
'new_account.*.name' => 'nullable|string|max:255',
'new_account.*.create' => 'nullable|string|in:0,1',
'new_account.*.type' =>
'nullable|string|in:asset,liability,expense,revenue',
'new_account.*.currency' =>
'nullable|string|size:3|regex:/^[A-Z]{3}$/',
'new_account.*.opening_balance' => 'nullable|numeric',
// camt
'grouped_transaction_handling' => 'in:single,group,split',
'use_entire_opposing_address' => 'numeric|between:0,1',
@ -135,16 +231,78 @@ class ConfigurationPostRequest extends Request
*/
public function withValidator(Validator $validator): void
{
$validator->after(
function (Validator $validator): void {
// validate all account info
$flow = request()->cookie(Constants::FLOW_COOKIE);
$data = $validator->getData();
$doImport = $data['do_import'] ?? [];
if (0 === count($doImport) && 'file' !== $flow) {
$validator->errors()->add('do_import', 'You must select at least one account to import from.');
$validator->after(function (Validator $validator): void {
// validate all account info
$flow = request()->cookie(Constants::FLOW_COOKIE);
$data = $validator->getData();
$doImport = $data['do_import'] ?? [];
if (0 === count($doImport) && 'file' !== $flow) {
$validator
->errors()
->add(
'do_import',
'You must select at least one account to import from.'
);
}
// validate new account creation data - both accounts and new_account now use encoded field names
$accounts = $data['accounts'] ?? [];
$newAccounts = $data['new_account'] ?? [];
app('log')->debug('DEBUG: withValidator account validation', [
'accounts' => $accounts,
'newAccounts' => array_keys($newAccounts),
'flow' => $flow,
]);
foreach ($accounts as $encodedAccountId => $selectedValue) {
if ($selectedValue === 'create_new') {
app('log')->debug(
'DEBUG: Validating new account creation',
[
'encodedAccountId' => $encodedAccountId,
'selectedValue' => $selectedValue,
'hasNameField' => isset(
$newAccounts[$encodedAccountId]['name']
),
'hasCreateField' => isset(
$newAccounts[$encodedAccountId]['create']
),
'nameValue' =>
$newAccounts[$encodedAccountId]['name'] ??
'NOT_SET',
'createValue' =>
$newAccounts[$encodedAccountId]['create'] ??
'NOT_SET',
]
);
// Validate that account name is provided and create flag is set
// Both arrays now use encoded keys, so they should match directly
if (
!isset($newAccounts[$encodedAccountId]['name']) ||
empty(trim($newAccounts[$encodedAccountId]['name']))
) {
$validator
->errors()
->add(
"new_account.{$encodedAccountId}.name",
'Account name is required when creating a new account.'
);
}
if (
!isset($newAccounts[$encodedAccountId]['create']) ||
$newAccounts[$encodedAccountId]['create'] !== '1'
) {
$validator
->errors()
->add(
"new_account.{$encodedAccountId}.create",
'Create flag must be set for new account creation.'
);
}
}
}
);
});
}
}

232
app/Jobs/ProcessImportSubmissionJob.php

@ -0,0 +1,232 @@
<?php
declare(strict_types=1);
namespace App\Jobs;
use App\Services\Shared\Configuration\Configuration;
use App\Services\Shared\Import\Routine\ApiSubmitter;
use App\Services\Shared\Import\Routine\InfoCollector;
use App\Services\Shared\Import\Routine\RoutineManager;
use App\Services\Shared\Import\Status\SubmissionStatus;
use App\Services\Shared\Import\Status\SubmissionStatusManager;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
class ProcessImportSubmissionJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* The number of times the job may be attempted.
*
* @var int
*/
public $tries = 1;
/**
* The maximum number of seconds the job can run for.
*
* @var int
*/
public $timeout = 1800; // 30 minutes
private string $identifier;
private Configuration $configuration;
private array $transactions;
private string $accessToken;
private string $baseUrl;
private ?string $vanityUrl;
/**
* Create a new job instance.
*/
public function __construct(
string $identifier,
Configuration $configuration,
array $transactions,
string $accessToken,
string $baseUrl,
?string $vanityUrl
) {
$this->identifier = $identifier;
$this->configuration = $configuration;
$this->transactions = $transactions;
$this->accessToken = $accessToken;
$this->baseUrl = $baseUrl;
$this->vanityUrl = $vanityUrl;
}
/**
* Execute the job.
*/
public function handle(): void
{
Log::info('ProcessImportSubmissionJob started', [
'identifier' => $this->identifier,
'transaction_count' => count($this->transactions),
]);
// Validate authentication credentials before proceeding
if (empty($this->accessToken)) {
throw new \Exception(
'Access token is empty - cannot authenticate with Firefly III'
);
}
if (empty($this->baseUrl)) {
throw new \Exception(
'Base URL is empty - cannot connect to Firefly III'
);
}
Log::info('Job authentication credentials validation', [
'identifier' => $this->identifier,
'access_token_length' => strlen($this->accessToken),
'access_token_preview' => substr($this->accessToken, 0, 20) . '...',
'base_url' => $this->baseUrl,
'vanity_url' => $this->vanityUrl ?? 'null',
]);
// Backup original configuration
$originalConfig = [
'importer.access_token' => config('importer.access_token'),
'importer.url' => config('importer.url'),
'importer.vanity_url' => config('importer.vanity_url'),
];
Log::debug('Original config backup', [
'identifier' => $this->identifier,
'original_token_length' => strlen(
$originalConfig['importer.access_token']
),
'original_url' => $originalConfig['importer.url'],
'original_vanity' => $originalConfig['importer.vanity_url'],
]);
try {
// Set authentication context for this job
config([
'importer.access_token' => $this->accessToken,
'importer.url' => $this->baseUrl,
'importer.vanity_url' => $this->vanityUrl ?? $this->baseUrl,
]);
Log::debug('Authentication context set for job', [
'identifier' => $this->identifier,
'base_url' => $this->baseUrl,
'vanity_url' => $this->vanityUrl ?? $this->baseUrl,
'access_token_length' => strlen($this->accessToken),
]);
// Verify config was actually set
$verifyToken = config('importer.access_token');
$verifyUrl = config('importer.url');
Log::debug('Config verification after setting', [
'identifier' => $this->identifier,
'config_token_matches' => $verifyToken === $this->accessToken,
'config_url_matches' => $verifyUrl === $this->baseUrl,
'config_token_length' => strlen($verifyToken),
'config_url' => $verifyUrl,
]);
if ($verifyToken !== $this->accessToken) {
throw new \Exception(
'Failed to set access token in config properly'
);
}
if ($verifyUrl !== $this->baseUrl) {
throw new \Exception(
'Failed to set base URL in config properly'
);
}
// Set initial running status
SubmissionStatusManager::setSubmissionStatus(
SubmissionStatus::SUBMISSION_RUNNING,
$this->identifier
);
// Initialize routine manager and execute import
$routine = new RoutineManager($this->identifier);
$routine->setConfiguration($this->configuration);
$routine->setTransactions($this->transactions);
Log::debug('Starting routine execution', [
'identifier' => $this->identifier,
]);
// Execute the import process
$routine->start();
// Set completion status
SubmissionStatusManager::setSubmissionStatus(
SubmissionStatus::SUBMISSION_DONE,
$this->identifier
);
Log::info('ProcessImportSubmissionJob completed successfully', [
'identifier' => $this->identifier,
'messages' => count($routine->getAllMessages()),
'warnings' => count($routine->getAllWarnings()),
'errors' => count($routine->getAllErrors()),
]);
} catch (Throwable $e) {
Log::error('ProcessImportSubmissionJob failed', [
'identifier' => $this->identifier,
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString(),
]);
// Set error status
SubmissionStatusManager::setSubmissionStatus(
SubmissionStatus::SUBMISSION_ERRORED,
$this->identifier
);
// Re-throw to mark job as failed
throw $e;
} finally {
// Always restore original configuration
config($originalConfig);
Log::debug('Authentication context restored', [
'identifier' => $this->identifier,
]);
}
}
/**
* Handle a job failure.
*/
public function failed(Throwable $exception): void
{
Log::error('ProcessImportSubmissionJob marked as failed', [
'identifier' => $this->identifier,
'exception' => $exception->getMessage(),
]);
// Ensure error status is set even if job fails catastrophically
SubmissionStatusManager::setSubmissionStatus(
SubmissionStatus::SUBMISSION_ERRORED,
$this->identifier
);
}
/**
* Get the tags that should be assigned to the job.
*
* @return array<int, string>
*/
public function tags(): array
{
return ['import-submission', $this->identifier];
}
}

143
app/Services/CSV/Mapper/ExpenseRevenueAccounts.php

@ -0,0 +1,143 @@
<?php
/*
* ExpenseRevenueAccounts.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\CSV\Mapper;
use App\Exceptions\ImporterErrorException;
use App\Services\Shared\Authentication\SecretManager;
use GrumpyDictator\FFIIIApiSupport\Exceptions\ApiHttpException;
use GrumpyDictator\FFIIIApiSupport\Model\Account;
use GrumpyDictator\FFIIIApiSupport\Request\GetAccountsRequest;
use GrumpyDictator\FFIIIApiSupport\Response\GetAccountsResponse;
/**
* Class ExpenseRevenueAccounts
*/
class ExpenseRevenueAccounts implements MapperInterface
{
/**
* Get map of expense and revenue accounts.
*
* @throws ImporterErrorException
*/
public function getMap(): array
{
$accounts = $this->getExpenseRevenueAccounts();
return $this->mergeExpenseRevenue($accounts);
}
/**
* Get expense and revenue accounts from Firefly III API.
*/
protected function getExpenseRevenueAccounts(): array
{
app('log')->debug('getExpenseRevenueAccounts: Fetching expense and revenue accounts.');
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
// Fetch all accounts since API doesn't have separate expense/revenue endpoints
$request = new GetAccountsRequest($url, $token);
$request->setVerify(config('importer.connection.verify'));
$request->setTimeOut(config('importer.connection.timeout'));
$request->setType(GetAccountsRequest::ALL);
try {
$response = $request->get();
} catch (ApiHttpException $e) {
app('log')->error($e->getMessage());
throw new ImporterErrorException(sprintf('Could not download accounts: %s', $e->getMessage()));
}
if (!$response instanceof GetAccountsResponse) {
throw new ImporterErrorException('Could not get list of accounts.');
}
$allAccounts = $this->toArray($response);
// Filter for expense and revenue accounts only
$expenseRevenueAccounts = array_filter($allAccounts, function (Account $account) {
return in_array($account->type, ['expense', 'revenue'], true);
});
app('log')->debug(sprintf('getExpenseRevenueAccounts: Found %d expense/revenue accounts', count($expenseRevenueAccounts)));
return $expenseRevenueAccounts;
}
/**
* Convert response to array of Account objects.
*/
protected function toArray(GetAccountsResponse $response): array
{
$accounts = [];
/** @var Account $account */
foreach ($response as $account) {
$accounts[] = $account;
}
return $accounts;
}
/**
* Merge expense and revenue accounts into select-ready list.
*/
protected function mergeExpenseRevenue(array $accounts): array
{
$result = [];
$invalidTypes = ['initial-balance', 'reconciliation'];
/** @var Account $account */
foreach ($accounts as $account) {
// Skip invalid types
if (in_array($account->type, $invalidTypes, true)) {
continue;
}
// Only include expense and revenue accounts
if (!in_array($account->type, ['expense', 'revenue'], true)) {
continue;
}
$name = $account->name;
// Add optgroup to result
$group = trans(sprintf('import.account_types_%s', $account->type));
$result[$group] ??= [];
$result[$group][$account->id] = $name;
}
// Sort each group
foreach ($result as $group => $accounts) {
asort($accounts, SORT_STRING);
$result[$group] = $accounts;
}
// Create new account functionality temporarily removed for stock compatibility
return $result;
}
}

6
app/Services/Session/Constants.php

@ -90,4 +90,10 @@ class Constants
public const string SUBMISSION_COMPLETE_INDICATOR = 'submission_complete';
public const string UPLOAD_CONFIG_FILE = 'config_file_path';
public const string UPLOAD_DATA_FILE = 'data_file_path';
// SimpleFIN specific constants
public const string SIMPLEFIN_TOKEN = 'simplefin_token';
public const string SIMPLEFIN_BRIDGE_URL = 'simplefin_bridge_url';
public const string SIMPLEFIN_ACCOUNTS_DATA = 'simplefin_accounts_data';
public const string SIMPLEFIN_IS_DEMO = 'simplefin_is_demo';
}

13
app/Services/Shared/Authentication/SecretManager.php

@ -67,7 +67,6 @@ class SecretManager
Log::debug('Access token is null, use config instead.');
$token = (string)config('importer.access_token');
}
return (string)$token;
}
@ -87,7 +86,8 @@ class SecretManager
*/
private static function hasBaseUrl(): bool
{
return session()->has(Constants::SESSION_BASE_URL) && '' !== session()->get(Constants::SESSION_BASE_URL);
return session()->has(Constants::SESSION_BASE_URL) &&
'' !== session()->get(Constants::SESSION_BASE_URL);
}
/**
@ -109,7 +109,8 @@ class SecretManager
*/
private static function hasClientId(): bool
{
return session()->has(Constants::SESSION_CLIENT_ID) && 0 !== session()->get(Constants::SESSION_CLIENT_ID);
return session()->has(Constants::SESSION_CLIENT_ID) &&
0 !== session()->get(Constants::SESSION_CLIENT_ID);
}
public static function getVanityUrl(): string
@ -131,7 +132,8 @@ class SecretManager
*/
private static function hasVanityUrl(): bool
{
return session()->has(Constants::SESSION_VANITY_URL) && '' !== session()->get(Constants::SESSION_VANITY_URL);
return session()->has(Constants::SESSION_VANITY_URL) &&
'' !== session()->get(Constants::SESSION_VANITY_URL);
}
/**
@ -155,7 +157,8 @@ class SecretManager
*/
private static function hasRefreshToken(): bool
{
return session()->has(Constants::SESSION_REFRESH_TOKEN) && '' !== session()->get(Constants::SESSION_REFRESH_TOKEN);
return session()->has(Constants::SESSION_REFRESH_TOKEN) &&
'' !== session()->get(Constants::SESSION_REFRESH_TOKEN);
}
/**

41
app/Services/Shared/Configuration/Configuration.php

@ -35,6 +35,7 @@ class Configuration
{
public const VERSION = 3;
private array $accounts;
private array $newAccounts;
private bool $addImportTag;
private string $connection;
private string $contentType;
@ -71,6 +72,9 @@ class Configuration
private bool $ignoreSpectreCategories;
private bool $mapAllData;
// simplefin configuration
private bool $pendingTransactions;
// date range settings
private array $mapping;
private string $nordigenBank;
@ -112,6 +116,8 @@ class Configuration
$this->roles = [];
$this->mapping = [];
$this->doMapping = [];
$this->accounts = [];
$this->newAccounts = [];
$this->flow = 'file';
$this->contentType = 'csv';
$this->customTag = '';
@ -144,6 +150,9 @@ class Configuration
// mapping for spectre + nordigen
$this->mapAllData = false;
// simplefin configuration
$this->pendingTransactions = true;
// double transaction detection:
$this->duplicateDetectionMethod = 'classic';
@ -342,6 +351,7 @@ class Configuration
// settings for spectre + nordigen
$object->mapAllData = $array['map_all_data'] ?? false;
$object->accounts = $array['accounts'] ?? [];
$object->newAccounts = $array['new_account'] ?? [];
// spectre
$object->identifier = $array['identifier'] ?? '0';
@ -393,6 +403,9 @@ class Configuration
// utf8
$object->conversion = $array['conversion'] ?? false;
// simplefin configuration
$object->pendingTransactions = $array['pending_transactions'] ?? true;
if ('csv' === $object->flow) {
$object->flow = 'file';
$object->contentType = 'csv';
@ -452,6 +465,7 @@ class Configuration
// spectre + nordigen
$object->accounts = $array['accounts'] ?? [];
$object->newAccounts = $array['new_account'] ?? [];
// date range settings
$object->dateRange = $array['date_range'] ?? 'all';
@ -477,6 +491,9 @@ class Configuration
// utf8 conversion
$object->conversion = $array['conversion'] ?? false;
// simplefin configuration
$object->pendingTransactions = $array['pending_transactions'] ?? true;
// flow
$object->flow = $array['flow'] ?? 'file';
@ -528,6 +545,16 @@ class Configuration
$this->accounts = $accounts;
}
public function getNewAccounts(): array
{
return $this->newAccounts;
}
public function setNewAccounts(array $newAccounts): void
{
$this->newAccounts = $newAccounts;
}
public function getConnection(): string
{
return $this->connection;
@ -698,6 +725,11 @@ class Configuration
$this->roles = $roles;
}
public function setPendingTransactions(bool $pendingTransactions): void
{
$this->pendingTransactions = $pendingTransactions;
}
public function getSpecifics(): array
{
return $this->specifics;
@ -713,6 +745,11 @@ class Configuration
return $this->uniqueColumnType;
}
public function getPendingTransactions(): bool
{
return $this->pendingTransactions;
}
public function hasSpecific(string $name): bool
{
return in_array($name, $this->specifics, true);
@ -817,8 +854,12 @@ class Configuration
// mapping for spectre + nordigen
'map_all_data' => $this->mapAllData,
// simplefin configuration
'pending_transactions' => $this->pendingTransactions,
// settings for spectre + nordigen
'accounts' => $this->accounts,
'new_account' => $this->newAccounts,
// date range settings:
'date_range' => $this->dateRange,

372
app/Services/Shared/Import/Routine/ApiSubmitter.php

@ -28,6 +28,7 @@ namespace App\Services\Shared\Import\Routine;
use App\Exceptions\ImporterErrorException;
use App\Services\Shared\Authentication\SecretManager;
use App\Services\Shared\Configuration\Configuration;
use App\Services\Shared\Import\Status\SubmissionStatusManager;
use App\Services\Shared\Submission\ProgressInformation;
use GrumpyDictator\FFIIIApiSupport\Exceptions\ApiHttpException;
use GrumpyDictator\FFIIIApiSupport\Model\Transaction;
@ -49,14 +50,14 @@ class ApiSubmitter
{
use ProgressInformation;
private array $accountInfo;
private bool $addTag;
private array $accountInfo;
private bool $addTag;
private Configuration $configuration;
private bool $createdTag;
private array $mapping;
private string $tag;
private string $tagDate;
private string $vanityURL;
private bool $createdTag;
private array $mapping;
private string $tag;
private string $tagDate;
private string $vanityURL;
/**
* @throws ImporterErrorException
@ -64,14 +65,18 @@ class ApiSubmitter
public function processTransactions(array $lines): void
{
$this->createdTag = false;
$this->tag = $this->parseTag();
$this->tagDate = date('Y-m-d');
$count = count($lines);
$uniqueCount = 0;
app('log')->info(sprintf('Going to submit %d transactions to your Firefly III instance.', $count));
$this->vanityURL = SecretManager::getVanityURL();
$this->tag = $this->parseTag();
$this->tagDate = date('Y-m-d');
$count = count($lines);
$uniqueCount = 0;
app('log')->info(
sprintf(
'Going to submit %d transactions to your Firefly III instance.',
$count
)
);
$this->vanityURL = SecretManager::getVanityURL();
app('log')->debug(sprintf('Vanity URL : "%s"', $this->vanityURL));
@ -80,19 +85,38 @@ class ApiSubmitter
* @var array $line
*/
foreach ($lines as $index => $line) {
app('log')->debug(sprintf('Now submitting transaction %d/%d', $index + 1, $count));
app('log')->debug(
sprintf('Now submitting transaction %d/%d', $index + 1, $count)
);
// Update progress tracking
SubmissionStatusManager::updateProgress(
$this->identifier,
$index + 1,
$count
);
// first do local duplicate transaction check (the "cell" method):
$unique = $this->uniqueTransaction($index, $line);
$unique = $this->uniqueTransaction($index, $line);
if (null === $unique) {
app('log')->debug(sprintf('Transaction #%d is not checked beforehand on uniqueness.', $index + 1));
app('log')->debug(
sprintf(
'Transaction #%d is not checked beforehand on uniqueness.',
$index + 1
)
);
++$uniqueCount;
}
if (true === $unique) {
app('log')->debug(sprintf('Transaction #%d is unique.', $index + 1));
app('log')->debug(
sprintf('Transaction #%d is unique.', $index + 1)
);
++$uniqueCount;
}
if (false === $unique) {
app('log')->debug(sprintf('Transaction #%d is NOT unique.', $index + 1));
app('log')->debug(
sprintf('Transaction #%d is NOT unique.', $index + 1)
);
continue;
}
@ -100,8 +124,15 @@ class ApiSubmitter
$this->addTagToGroups($groupInfo);
}
app('log')->info(sprintf('Done submitting %d transactions to your Firefly III instance.', $count));
app('log')->info(sprintf('Actually imported and not duplicate: %d.', $uniqueCount));
app('log')->info(
sprintf(
'Done submitting %d transactions to your Firefly III instance.',
$count
)
);
app('log')->info(
sprintf('Actually imported and not duplicate: %d.', $uniqueCount)
);
}
private function parseTag(): string
@ -112,7 +143,7 @@ class ApiSubmitter
// return default tag:
return sprintf('Data Import on %s', date('Y-m-d \@ H:i'));
}
$items = [
$items = [
'%year%' => date('Y'),
'%month%' => date('m'),
'%month_full%' => date('F'),
@ -126,8 +157,14 @@ class ApiSubmitter
'%datetime%' => date('Y-m-d \@ H:i'),
'%version%' => config('importer.version'),
];
$result = str_replace(array_keys($items), array_values($items), $customTag);
app('log')->debug(sprintf('Custom tag is "%s", parsed into "%s"', $customTag, $result));
$result = str_replace(
array_keys($items),
array_values($items),
$customTag
);
app('log')->debug(
sprintf('Custom tag is "%s", parsed into "%s"', $customTag, $result)
);
return $result;
}
@ -140,19 +177,22 @@ class ApiSubmitter
{
if ('cell' !== $this->configuration->getDuplicateDetectionMethod()) {
app('log')->debug(
sprintf('Duplicate detection method is "%s", so this method is skipped (return true).', $this->configuration->getDuplicateDetectionMethod())
sprintf(
'Duplicate detection method is "%s", so this method is skipped (return true).',
$this->configuration->getDuplicateDetectionMethod()
)
);
return null;
}
// do a search for the value and the field:
$transactions = $line['transactions'] ?? [];
$field = $this->configuration->getUniqueColumnType();
$field = 'external-id' === $field ? 'external_id' : $field;
$field = 'note' === $field ? 'notes' : $field;
$value = '';
$field = $this->configuration->getUniqueColumnType();
$field = 'external-id' === $field ? 'external_id' : $field;
$field = 'note' === $field ? 'notes' : $field;
$value = '';
foreach ($transactions as $transactionIndex => $transaction) {
$value = (string) ($transaction[$field] ?? '');
$value = (string) ($transaction[$field] ?? '');
if ('' === $value) {
app('log')->debug(
sprintf(
@ -168,7 +208,12 @@ class ApiSubmitter
$searchResult = $this->searchField($field, $value);
if (0 !== $searchResult) {
app('log')->debug(
sprintf('Looks like field "%s" with value "%s" is not unique, found in group #%d. Return false', $field, $value, $searchResult)
sprintf(
'Looks like field "%s" with value "%s" is not unique, found in group #%d. Return false',
$field,
$value,
$searchResult
)
);
$message = sprintf(
'[a115]: There is already a transaction with %s "%s" (<a href="%s/transactions/show/%d">link</a>).',
@ -184,7 +229,13 @@ class ApiSubmitter
return false;
}
}
app('log')->debug(sprintf('Looks like field "%s" with value "%s" is unique, return false.', $field, $value));
app('log')->debug(
sprintf(
'Looks like field "%s" with value "%s" is unique, return false.',
$field,
$value
)
);
return true;
}
@ -196,13 +247,20 @@ class ApiSubmitter
{
// search for the exact description and not just a part of it:
$searchModifier = config(sprintf('csv.search_modifier.%s', $field));
$query = sprintf('%s:"%s"', $searchModifier, $value);
app('log')->debug(sprintf('Going to search for %s:%s using query %s', $field, $value, $query));
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$request = new GetSearchTransactionsRequest($url, $token);
$query = sprintf('%s:"%s"', $searchModifier, $value);
app('log')->debug(
sprintf(
'Going to search for %s:%s using query %s',
$field,
$value,
$query
)
);
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$request = new GetSearchTransactionsRequest($url, $token);
$request->setTimeOut(config('importer.connection.timeout'));
$request->setVerify(config('importer.connection.verify'));
$request->setQuery($query);
@ -218,8 +276,14 @@ class ApiSubmitter
if (0 === $response->count()) {
return 0;
}
$first = $response->current();
app('log')->debug(sprintf('Found %d transaction(s). Return group ID #%d.', $response->count(), $first->id));
$first = $response->current();
app('log')->debug(
sprintf(
'Found %d transaction(s). Return group ID #%d.',
$response->count(),
$first->id
)
);
return $first->id;
}
@ -227,40 +291,56 @@ class ApiSubmitter
private function processTransaction(int $index, array $line): array
{
++$index;
$line = $this->cleanupLine($line);
$return = [];
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$line = $this->cleanupLine($line);
$return = [];
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$request = new PostTransactionRequest($url, $token);
$request->setVerify(config('importer.connection.verify'));
$request->setTimeOut(config('importer.connection.timeout'));
app('log')->debug(sprintf('Submitting to Firefly III: %s', json_encode($line)));
app('log')->debug(
sprintf('Submitting to Firefly III: %s', json_encode($line))
);
$request->setBody($line);
try {
$response = $request->post();
} catch (ApiHttpException $e) {
$isDeleted = false;
$body = $request->getResponseBody();
$json = json_decode($body, true);
$body = $request->getResponseBody();
$json = json_decode($body, true);
// before we complain, first check what the error is:
if (is_array($json) && array_key_exists('message', $json)) {
if (str_contains($json['message'], '200032')) {
$isDeleted = true;
}
}
if (true === $isDeleted && false === config('importer.ignore_not_found_transactions')) {
$this->addWarning($index, 'The transaction was created, but deleted by a rule.');
if (
true === $isDeleted &&
false === config('importer.ignore_not_found_transactions')
) {
$this->addWarning(
$index,
'The transaction was created, but deleted by a rule.'
);
app('log')->error($e->getMessage());
return $return;
}
if (true === $isDeleted && true === config('importer.ignore_not_found_transactions')) {
Log::info('The transaction was deleted by a rule, but this is ignored by the importer.');
if (
true === $isDeleted &&
true === config('importer.ignore_not_found_transactions')
) {
Log::info(
'The transaction was deleted by a rule, but this is ignored by the importer.'
);
return $return;
}
$message = sprintf('[a116]: Submission HTTP error: %s', e($e->getMessage()));
$message = sprintf(
'[a116]: Submission HTTP error: %s',
e($e->getMessage())
);
app('log')->error($e->getMessage());
$this->addError($index, $message);
@ -269,10 +349,21 @@ class ApiSubmitter
if ($response instanceof ValidationErrorResponse) {
foreach ($response->errors->messages() as $key => $errors) {
app('log')->error(sprintf('Submission error: %d', $key), $errors);
app('log')->error(
sprintf('Submission error: %d', $key),
$errors
);
foreach ($errors as $error) {
$msg = sprintf('[a117]: %s: %s (original value: "%s")', $key, $error, $this->getOriginalValue($key, $line));
if (false === $this->isDuplicationError($key, $error) || false === config('importer.ignore_duplicate_errors')) {
$msg = sprintf(
'[a117]: %s: %s (original value: "%s")',
$key,
$error,
$this->getOriginalValue($key, $line)
);
if (
false === $this->isDuplicationError($key, $error) ||
false === config('importer.ignore_duplicate_errors')
) {
$this->addError($index, $msg);
}
app('log')->error($msg);
@ -284,9 +375,10 @@ class ApiSubmitter
if ($response instanceof PostTransactionResponse) {
/** @var TransactionGroup $group */
$group = $response->getTransactionGroup();
$group = $response->getTransactionGroup();
if (null === $group) {
$message = '[a118]: Could not create transaction. Unexpected empty response from Firefly III. Check the logs.';
$message =
'[a118]: Could not create transaction. Unexpected empty response from Firefly III. Check the logs.';
app('log')->error($message, $response->getRawData());
$this->addError($index, $message);
@ -295,7 +387,8 @@ class ApiSubmitter
// perhaps zero transactions in the array.
if (0 === count($group->transactions)) {
$message = '[a119]: Could not create transaction. Transaction-count from Firefly III is zero. Check the logs.';
$message =
'[a119]: Could not create transaction. Transaction-count from Firefly III is zero. Check the logs.';
app('log')->error($message, $response->getRawData());
$this->addError($index, $message);
@ -307,14 +400,21 @@ class ApiSubmitter
'journals' => [],
];
foreach ($group->transactions as $transaction) {
$message = sprintf(
$message = sprintf(
'Created %s <a target="_blank" href="%s">#%d "%s"</a> (%s %s)',
$transaction->type,
sprintf('%s/transactions/show/%d', $this->vanityURL, $group->id),
sprintf(
'%s/transactions/show/%d',
$this->vanityURL,
$group->id
),
$group->id,
e($transaction->description),
$transaction->currencyCode,
round((float) $transaction->amount, (int) $transaction->currencyDecimalPlaces) // float but only for display purposes
round(
(float) $transaction->amount,
(int) $transaction->currencyDecimalPlaces
) // float but only for display purposes
);
// plus 1 to keep the count.
$this->addMessage($index, $message);
@ -331,7 +431,9 @@ class ApiSubmitter
{
app('log')->debug('Going to map data for this line.');
if (array_key_exists(0, $this->mapping)) {
app('log')->debug('Configuration has mapping for opposing account name!');
app('log')->debug(
'Configuration has mapping for opposing account name!'
);
/**
* @var int $index
@ -342,11 +444,19 @@ class ApiSubmitter
// replace destination_name with destination_id
$destination = $transaction['destination_name'] ?? '';
if (array_key_exists($destination, $this->mapping[0])) {
unset($transaction['destination_name'], $transaction['destination_iban']);
unset(
$transaction['destination_name'],
$transaction['destination_iban']
);
$transaction['destination_id'] = $this->mapping[0][$destination];
$transaction['destination_id'] =
$this->mapping[0][$destination];
app('log')->debug(
sprintf('Replaced destination name "%s" with a reference to account id #%d', $destination, $this->mapping[0][$destination])
sprintf(
'Replaced destination name "%s" with a reference to account id #%d',
$destination,
$this->mapping[0][$destination]
)
);
}
}
@ -354,16 +464,27 @@ class ApiSubmitter
// replace source_name with source_id
$source = $transaction['source_name'] ?? '';
if (array_key_exists($source, $this->mapping[0])) {
unset($transaction['source_name'], $transaction['source_iban']);
unset(
$transaction['source_name'],
$transaction['source_iban']
);
$transaction['source_id'] = $this->mapping[0][$source];
app('log')->debug(sprintf('Replaced source name "%s" with a reference to account id #%d', $source, $this->mapping[0][$source]));
app('log')->debug(
sprintf(
'Replaced source name "%s" with a reference to account id #%d',
$source,
$this->mapping[0][$source]
)
);
}
}
if ('' === trim((string) $transaction['description'] ?? '')) {
if ('' === trim((string)$transaction['description'] ?? '')) {
$transaction['description'] = '(no description)';
}
$line['transactions'][$index] = $this->updateTransactionType($transaction);
$line['transactions'][$index] = $this->updateTransactionType(
$transaction
);
}
}
@ -372,16 +493,23 @@ class ApiSubmitter
private function updateTransactionType(array $transaction): array
{
if (array_key_exists('source_id', $transaction) && array_key_exists('destination_id', $transaction)) {
if (
array_key_exists('source_id', $transaction) &&
array_key_exists('destination_id', $transaction)
) {
app('log')->debug('Transaction has source_id/destination_id');
$sourceId = (int) $transaction['source_id'];
$destinationId = (int) $transaction['destination_id'];
$sourceType = $this->accountInfo[$sourceId] ?? 'unknown';
$sourceId = (int)$transaction['source_id'];
$destinationId = (int)$transaction['destination_id'];
$sourceType = $this->accountInfo[$sourceId] ?? 'unknown';
$destinationType = $this->accountInfo[$destinationId] ?? 'unknown';
$combi = sprintf('%s-%s', $sourceType, $destinationType);
app('log')->debug(sprintf('Account type combination is "%s"', $combi));
$combi = sprintf('%s-%s', $sourceType, $destinationType);
app('log')->debug(
sprintf('Account type combination is "%s"', $combi)
);
if ('asset-asset' === $combi) {
app('log')->debug('Both accounts are assets, so this transaction is a transfer.');
app('log')->debug(
'Both accounts are assets, so this transaction is a transfer.'
);
$transaction['type'] = 'transfer';
}
}
@ -400,12 +528,16 @@ class ApiSubmitter
}
$index = (int) $parts[1];
return (string) ($transaction['transactions'][$index][$parts[2]] ?? '(not found)');
return (string) ($transaction['transactions'][$index][$parts[2]] ??
'(not found)');
}
private function isDuplicationError(string $key, string $error): bool
{
if ('transactions.0.description' === $key && str_contains($error, 'Duplicate of transaction #')) {
if (
'transactions.0.description' === $key &&
str_contains($error, 'Duplicate of transaction #')
) {
app('log')->debug('This is a duplicate transaction error');
return true;
@ -415,14 +547,23 @@ class ApiSubmitter
return false;
}
private function compareArrays(int $lineIndex, array $line, TransactionGroup $group): void
{
private function compareArrays(
int $lineIndex,
array $line,
TransactionGroup $group
): void {
// some fields may not have survived. Be sure to warn the user about this.
/** @var Transaction $transaction */
foreach ($group->transactions as $index => $transaction) {
// compare currency ID
if (array_key_exists('currency_id', $line['transactions'][$index]) && null !== $line['transactions'][$index]['currency_id']
&& (int) $line['transactions'][$index]['currency_id'] !== (int) $transaction->currencyId
if (
array_key_exists(
'currency_id',
$line['transactions'][$index]
) &&
null !== $line['transactions'][$index]['currency_id'] &&
(int)$line['transactions'][$index]['currency_id'] !==
(int) $transaction->currencyId
) {
$this->addWarning(
$lineIndex,
@ -435,8 +576,14 @@ class ApiSubmitter
);
}
// compare currency code:
if (array_key_exists('currency_code', $line['transactions'][$index]) && null !== $line['transactions'][$index]['currency_code']
&& $line['transactions'][$index]['currency_code'] !== $transaction->currencyCode
if (
array_key_exists(
'currency_code',
$line['transactions'][$index]
) &&
null !== $line['transactions'][$index]['currency_code'] &&
$line['transactions'][$index]['currency_code'] !==
$transaction->currencyCode
) {
$this->addWarning(
$lineIndex,
@ -454,7 +601,9 @@ class ApiSubmitter
private function addTagToGroups(array $groupInfo): void
{
if ([] === $groupInfo) {
app('log')->debug('Group is empty, may not have been stored correctly.');
app('log')->debug(
'Group is empty, may not have been stored correctly.'
);
return;
}
@ -468,9 +617,14 @@ class ApiSubmitter
$this->createdTag = true;
}
$groupId = (int) $groupInfo['group_id'];
app('log')->debug(sprintf('Going to add import tag to transaction group #%d', $groupId));
$body = [
$groupId = (int)$groupInfo['group_id'];
app('log')->debug(
sprintf(
'Going to add import tag to transaction group #%d',
$groupId
)
);
$body = [
'transactions' => [],
];
@ -479,14 +633,14 @@ class ApiSubmitter
* @var array $currentTags
*/
foreach ($groupInfo['journals'] as $journalId => $currentTags) {
$currentTags[] = $this->tag;
$currentTags[] = $this->tag;
$body['transactions'][] = [
'transaction_journal_id' => $journalId,
'tags' => $currentTags,
];
}
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$request = new PutTransactionRequest($url, $token, $groupId);
$request->setVerify(config('importer.connection.verify'));
$request->setTimeOut(config('importer.connection.timeout'));
@ -497,24 +651,31 @@ class ApiSubmitter
} catch (ApiHttpException $e) {
app('log')->error($e->getMessage());
// app('log')->error($e->getTraceAsString());
$this->addError(0, '[a120]: Could not store transaction: see the log files.');
$this->addError(
0,
'[a120]: Could not store transaction: see the log files.'
);
}
app('log')->debug(sprintf('Added import tag to transaction group #%d', $groupId));
app('log')->debug(
sprintf('Added import tag to transaction group #%d', $groupId)
);
}
private function createTag(): void
{
if (false === $this->addTag) {
app('log')->debug('Not instructed to add a tag, so will not create one.');
app('log')->debug(
'Not instructed to add a tag, so will not create one.'
);
return;
}
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$request = new PostTagRequest($url, $token);
$request->setVerify(config('importer.connection.verify'));
$request->setTimeOut(config('importer.connection.timeout'));
$body = [
$body = [
'tag' => $this->tag,
'date' => $this->tagDate,
];
@ -524,7 +685,10 @@ class ApiSubmitter
/** @var PostTagResponse $response */
$response = $request->post();
} catch (ApiHttpException $e) {
$message = sprintf('[a121]: Could not create tag. %s', $e->getMessage());
$message = sprintf(
'[a121]: Could not create tag. %s',
$e->getMessage()
);
app('log')->error($message);
// app('log')->error($e->getTraceAsString());
$this->addError(0, $message);
@ -537,7 +701,13 @@ class ApiSubmitter
return;
}
if (null !== $response->getTag()) {
app('log')->info(sprintf('Created tag #%d "%s"', $response->getTag()->id, $response->getTag()->tag));
app('log')->info(
sprintf(
'Created tag #%d "%s"',
$response->getTag()->id,
$response->getTag()->tag
)
);
}
}

38
app/Services/Shared/Import/Status/SubmissionStatus.php

@ -35,16 +35,22 @@ class SubmissionStatus
public array $messages;
public string $status;
public array $warnings;
public int $currentTransaction;
public int $totalTransactions;
public int $progressPercentage;
/**
* ImportJobStatus constructor.
*/
public function __construct()
{
$this->status = self::SUBMISSION_WAITING;
$this->errors = [];
$this->warnings = [];
$this->messages = [];
$this->status = self::SUBMISSION_WAITING;
$this->errors = [];
$this->warnings = [];
$this->messages = [];
$this->currentTransaction = 0;
$this->totalTransactions = 0;
$this->progressPercentage = 0;
}
/**
@ -52,11 +58,14 @@ class SubmissionStatus
*/
public static function fromArray(array $array): self
{
$config = new self();
$config->status = $array['status'];
$config->errors = $array['errors'] ?? [];
$config->warnings = $array['warnings'] ?? [];
$config->messages = $array['messages'] ?? [];
$config = new self();
$config->status = $array['status'];
$config->errors = $array['errors'] ?? [];
$config->warnings = $array['warnings'] ?? [];
$config->messages = $array['messages'] ?? [];
$config->currentTransaction = $array['currentTransaction'] ?? 0;
$config->totalTransactions = $array['totalTransactions'] ?? 0;
$config->progressPercentage = $array['progressPercentage'] ?? 0;
return $config;
}
@ -64,10 +73,13 @@ class SubmissionStatus
public function toArray(): array
{
return [
'status' => $this->status,
'errors' => $this->errors,
'warnings' => $this->warnings,
'messages' => $this->messages,
'status' => $this->status,
'errors' => $this->errors,
'warnings' => $this->warnings,
'messages' => $this->messages,
'currentTransaction' => $this->currentTransaction,
'totalTransactions' => $this->totalTransactions,
'progressPercentage' => $this->progressPercentage,
];
}
}

29
app/Services/Shared/Import/Status/SubmissionStatusManager.php

@ -109,6 +109,7 @@ class SubmissionStatusManager
{
$lineNo = $index + 1;
app('log')->debug(sprintf('Add warning on index #%d (line no. %d): %s', $index, $lineNo, $warning));
$disk = \Storage::disk(self::DISK_NAME);
try {
@ -130,6 +131,34 @@ class SubmissionStatusManager
}
}
public static function updateProgress(string $identifier, int $currentTransaction, int $totalTransactions): void
{
app('log')->debug(sprintf('Update progress for %s: %d/%d transactions', $identifier, $currentTransaction, $totalTransactions));
$disk = \Storage::disk(self::DISK_NAME);
try {
if ($disk->exists($identifier)) {
try {
$status = SubmissionStatus::fromArray(json_decode($disk->get($identifier), true, 512, JSON_THROW_ON_ERROR));
} catch (\JsonException $e) {
$status = new SubmissionStatus();
}
$status->currentTransaction = $currentTransaction;
$status->totalTransactions = $totalTransactions;
$status->progressPercentage = $totalTransactions > 0 ? (int) round(($currentTransaction / $totalTransactions) * 100) : 0;
self::storeSubmissionStatus($identifier, $status);
}
if (!$disk->exists($identifier)) {
app('log')->error(sprintf('Could not find file for job %s.', $identifier));
}
} catch (FileNotFoundException $e) {
app('log')->error($e->getMessage());
}
}
public static function setSubmissionStatus(string $status, ?string $identifier = null): SubmissionStatus
{
if (null === $identifier) {

42
app/Services/Shared/Response/ResponseInterface.php

@ -0,0 +1,42 @@
<?php
/*
* ResponseInterface.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\Shared\Response;
/**
* Interface ResponseInterface
*/
interface ResponseInterface
{
/**
* Check if the response has an error
*/
public function hasError(): bool;
/**
* Get the HTTP status code
*/
public function getStatusCode(): int;
}

2
app/Services/SimpleFIN/AuthenticationValidator.php

@ -41,7 +41,7 @@ class AuthenticationValidator implements AuthenticationValidatorInterface
app('log')->debug(sprintf('Now at %s', __METHOD__));
// needs an APP key which isn't blank or zero or whatever.
$key = (string) env('APP_KEY');
$key = (string) config('app.key');
if ('' === $key) {
return AuthenticationStatus::error();
}

485
app/Services/SimpleFIN/Conversion/AccountMapper.php

@ -0,0 +1,485 @@
<?php
/*
* AccountMapper.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Conversion;
use App\Exceptions\ImporterErrorException;
use App\Services\Shared\Authentication\SecretManager;
use App\Services\SimpleFIN\Model\Account as SimpleFINAccount;
use App\Services\SimpleFIN\Request\PostAccountRequest;
use App\Services\SimpleFIN\Response\PostAccountResponse;
use GrumpyDictator\FFIIIApiSupport\Exceptions\ApiHttpException;
use GrumpyDictator\FFIIIApiSupport\Model\Account;
use GrumpyDictator\FFIIIApiSupport\Model\AccountType;
use GrumpyDictator\FFIIIApiSupport\Request\GetAccountsRequest;
use GrumpyDictator\FFIIIApiSupport\Request\GetSearchAccountRequest;
use GrumpyDictator\FFIIIApiSupport\Response\GetAccountsResponse;
use GrumpyDictator\FFIIIApiSupport\Response\Response;
use GrumpyDictator\FFIIIApiSupport\Response\ValidationErrorResponse;
/**
* Class AccountMapper
*/
class AccountMapper
{
private array $fireflyAccounts = [];
private array $accountMapping = [];
private array $createdAccounts = [];
public function __construct()
{
// Defer account loading until actually needed to avoid authentication errors
// during constructor when authentication context may not be available
}
/**
* Map SimpleFIN accounts to Firefly III accounts
* @param array $simplefinAccounts
* @param array $configuration
* @return array
*/
public function mapAccounts(array $simplefinAccounts, array $configuration = []): array
{
$mapping = [];
foreach ($simplefinAccounts as $simplefinAccount) {
if (!$simplefinAccount instanceof SimpleFINAccount) {
continue;
}
$accountKey = $simplefinAccount->getId();
// Check if mapping is already configured
if (isset($configuration['account_mapping'][$accountKey])) {
$mappingConfig = $configuration['account_mapping'][$accountKey];
if ($mappingConfig['action'] === 'map' && isset($mappingConfig['firefly_account_id'])) {
// Map to existing account
$fireflyAccount = $this->getFireflyAccountById((int) $mappingConfig['firefly_account_id']);
if ($fireflyAccount) {
$mapping[$accountKey] = [
'simplefin_account' => $simplefinAccount,
'firefly_account_id' => $fireflyAccount->id,
'firefly_account_name' => $fireflyAccount->name,
'action' => 'map',
];
}
} elseif ($mappingConfig['action'] === 'create') {
// Create new account
$fireflyAccount = $this->createFireflyAccount($simplefinAccount, $mappingConfig);
if ($fireflyAccount) {
$mapping[$accountKey] = [
'simplefin_account' => $simplefinAccount,
'firefly_account_id' => $fireflyAccount->id,
'firefly_account_name' => $fireflyAccount->name,
'action' => 'create',
];
}
}
} else {
// Auto-map by searching for existing accounts
$fireflyAccount = $this->findMatchingFireflyAccount($simplefinAccount);
if ($fireflyAccount) {
$mapping[$accountKey] = [
'simplefin_account' => $simplefinAccount,
'firefly_account_id' => $fireflyAccount->id,
'firefly_account_name' => $fireflyAccount->name,
'action' => 'auto_map',
];
} else {
// No mapping found - will need user input
$mapping[$accountKey] = [
'simplefin_account' => $simplefinAccount,
'firefly_account_id' => null,
'firefly_account_name' => null,
'action' => 'unmapped',
];
}
}
}
return $mapping;
}
/**
* Get available Firefly III accounts for mapping
*/
public function getAvailableFireflyAccounts(): array
{
$this->loadFireflyAccounts();
return $this->fireflyAccounts;
}
/**
* Find a matching Firefly III account for a SimpleFIN account
*/
private function findMatchingFireflyAccount(SimpleFINAccount $simplefinAccount): ?Account
{
$this->loadFireflyAccounts();
// Try to find by name first
$matchingAccounts = array_filter($this->fireflyAccounts, function (Account $account) use ($simplefinAccount) {
return strtolower($account->name) === strtolower($simplefinAccount->getName());
});
if (!empty($matchingAccounts)) {
return reset($matchingAccounts);
}
// Try to search via API
try {
$request = new GetSearchAccountRequest(SecretManager::getBaseUrl(), SecretManager::getAccessToken());
$request->setQuery($simplefinAccount->getName());
$response = $request->get();
if ($response instanceof GetAccountsResponse && count($response) > 0) {
foreach ($response as $account) {
if (strtolower($account->name) === strtolower($simplefinAccount->getName())) {
return $account;
}
}
}
} catch (ApiHttpException $e) {
app('log')->warning(sprintf('Could not search for account "%s": %s', $simplefinAccount->getName(), $e->getMessage()));
}
return null;
}
/**
* Create account immediately via Firefly III API
* @param SimpleFINAccount $simplefinAccount
* @param array $config
* @return Account|null
*/
public function createFireflyAccount(SimpleFINAccount $simplefinAccount, array $config): ?Account
{
$accountName = $config['name'] ?? $simplefinAccount->getName();
$accountType = $this->determineAccountType($simplefinAccount, $config);
$currencyCode = $this->getCurrencyCode($simplefinAccount, $config);
$openingBalance = $config['opening_balance'] ?? '0.00';
app('log')->info(sprintf('Creating Firefly III account "%s" immediately via API', $accountName));
try {
$request = new PostAccountRequest(SecretManager::getBaseUrl(), SecretManager::getAccessToken());
// Build account creation payload
$payload = [
'name' => $accountName,
'type' => $accountType,
'currency_code' => $currencyCode,
'opening_balance' => $openingBalance,
'active' => true,
'include_net_worth' => true,
];
// Add opening balance date if opening balance is provided
if (!empty($config['opening_balance']) && is_numeric($config['opening_balance'])) {
$payload['opening_balance_date'] = $config['opening_balance_date'] ?? date('Y-m-d');
}
// Add account role for asset accounts
if ($accountType === AccountType::ASSET) {
$payload['account_role'] = $config['account_role'] ?? 'defaultAsset';
}
// Add liability-specific fields for liability accounts
if (in_array($accountType, [AccountType::DEBT, AccountType::LOAN, AccountType::MORTGAGE, AccountType::LIABILITIES, 'liability'], true)) {
// Map account type to liability type
$liabilityTypeMap = [
AccountType::DEBT => 'debt',
AccountType::LOAN => 'loan',
AccountType::MORTGAGE => 'mortgage',
AccountType::LIABILITIES => 'debt', // Default generic liabilities to debt
'liability' => 'debt', // Handle user-provided 'liability' type
];
$payload['liability_type'] = $config['liability_type'] ?? $liabilityTypeMap[$accountType] ?? 'debt';
$payload['liability_direction'] = $config['liability_direction'] ?? 'credit';
}
// Add IBAN if provided
if (!empty($config['iban'])) {
$payload['iban'] = $config['iban'];
}
// Add account number if provided
if (!empty($config['account_number'])) {
$payload['account_number'] = $config['account_number'];
}
$request->setBody($payload);
$response = $this->makeApiCallWithRetry($request, $accountName);
if ($response instanceof ValidationErrorResponse) {
app('log')->error(sprintf('Failed to create account "%s": %s', $accountName, json_encode($response->errors->toArray())));
return null;
}
if ($response instanceof PostAccountResponse) {
$account = $response->getAccount();
if ($account) {
app('log')->info(sprintf('Successfully created account "%s" with ID %d', $accountName, $account->id));
// Add to our local cache
$this->fireflyAccounts[] = $account;
$this->createdAccounts[] = $account;
return $account;
}
}
app('log')->error(sprintf('Unexpected response type when creating account "%s"', $accountName));
return null;
} catch (ApiHttpException $e) {
app('log')->error(sprintf('API error creating account "%s": %s', $accountName, $e->getMessage()));
return null;
} catch (\Exception $e) {
app('log')->error(sprintf('Unexpected error creating account "%s": %s', $accountName, $e->getMessage()));
return null;
}
}
/**
* Determine the appropriate Firefly III account type
* @param SimpleFINAccount $simplefinAccount
* @param array $config
* @return string
*/
private function determineAccountType(SimpleFINAccount $simplefinAccount, array $config): string
{
if (isset($config['type'])) {
return $config['type'];
}
// Default to asset account for most SimpleFIN accounts
return AccountType::ASSET;
}
/**
* Get currency code for account creation
*/
private function getCurrencyCode(SimpleFINAccount $simplefinAccount, array $config): string
{
// 1. Use user-configured currency first
if (!empty($config['currency'])) {
return $config['currency'];
}
// 2. Fall back to SimpleFIN account currency
$currency = $simplefinAccount->getCurrency();
if ($simplefinAccount->isCustomCurrency()) {
// For custom currencies, default to user's base currency or USD
return 'USD'; // Could be made configurable
}
// 3. Final fallback
return $currency ?: 'USD';
}
/**
* Get Firefly III account by ID
*/
private function getFireflyAccountById(int $id): ?Account
{
$this->loadFireflyAccounts();
foreach ($this->fireflyAccounts as $account) {
if ($account->id === $id) {
return $account;
}
}
return null;
}
/**
* Load all Firefly III accounts
*/
private function loadFireflyAccounts(): void
{
// Only load once
if (!empty($this->fireflyAccounts)) {
return;
}
try {
// Verify authentication context before making API calls
$baseUrl = SecretManager::getBaseUrl();
$accessToken = SecretManager::getAccessToken();
if (empty($baseUrl) || empty($accessToken)) {
app('log')->warning('Missing authentication context for Firefly III account loading');
throw new ImporterErrorException('Authentication context not available for account loading');
}
$request = new GetAccountsRequest($baseUrl, $accessToken);
$request->setType(AccountType::ASSET);
$response = $request->get();
if ($response instanceof GetAccountsResponse) {
$this->fireflyAccounts = iterator_to_array($response);
app('log')->debug(sprintf('Loaded %d Firefly III accounts', count($this->fireflyAccounts)));
}
} catch (ApiHttpException $e) {
app('log')->error(sprintf('Could not load Firefly III accounts: %s', $e->getMessage()));
throw new ImporterErrorException(sprintf('Could not load Firefly III accounts: %s', $e->getMessage()));
}
}
/**
* Make API call with DNS resilience retry pattern
* @param PostAccountRequest $request
* @param string $accountName
* @return Response
* @throws ApiHttpException
*/
private function makeApiCallWithRetry(PostAccountRequest $request, string $accountName): Response
{
$retryDelays = [0, 2, 5]; // immediate, 2s delay, 5s delay
$lastException = null;
foreach ($retryDelays as $attempt => $delay) {
try {
if ($delay > 0) {
app('log')->debug(sprintf('Retrying account creation for "%s" after %ds delay (attempt %d)', $accountName, $delay, $attempt + 1));
sleep($delay);
}
return $request->post();
} catch (ApiHttpException $e) {
$lastException = $e;
$errorMessage = $e->getMessage();
// Check if this is a DNS/connection timeout error that we should retry
$shouldRetry = $this->shouldRetryApiCall($errorMessage, $attempt, count($retryDelays));
if (!$shouldRetry) {
app('log')->error(sprintf('Non-retryable API error for account "%s": %s', $accountName, $errorMessage));
throw $e;
}
app('log')->warning(sprintf('DNS/connection error for account "%s" (attempt %d): %s', $accountName, $attempt + 1, $errorMessage));
// If this was the last attempt, we'll throw after the loop
if ($attempt === count($retryDelays) - 1) {
break;
}
}
}
// All retries exhausted
app('log')->error(sprintf('All retries exhausted for account "%s": %s', $accountName, $lastException->getMessage()));
throw $lastException;
}
/**
* Determine if an API call should be retried based on the error
* @param string $errorMessage
* @param int $attempt
* @param int $maxAttempts
* @return bool
*/
private function shouldRetryApiCall(string $errorMessage, int $attempt, int $maxAttempts): bool
{
// Don't retry if we've exhausted all attempts
if ($attempt >= $maxAttempts - 1) {
return false;
}
// Retry on DNS resolution timeouts, connection timeouts, and network errors
$retryableErrors = [
'Resolving timed out',
'cURL error 28',
'Connection timed out',
'cURL error 6', // Couldn't resolve host
'cURL error 7', // Couldn't connect to host
'Failed to connect',
'Name or service not known',
'Temporary failure in name resolution'
];
foreach ($retryableErrors as $retryableError) {
if (stripos($errorMessage, $retryableError) !== false) {
return true;
}
}
return false;
}
/**
* Get mapping options for UI
* @param SimpleFINAccount $simplefinAccount
* @return array<string,mixed>
*/
public function getMappingOptions(SimpleFINAccount $simplefinAccount): array
{
$this->loadFireflyAccounts();
$options = [
'account_name' => $simplefinAccount->getName(),
'account_id' => $simplefinAccount->getId(),
'currency' => $simplefinAccount->getCurrency(),
'balance' => $simplefinAccount->getBalance(),
'organization' => $simplefinAccount->getOrganizationName() ?? $simplefinAccount->getOrganizationDomain(),
'firefly_accounts' => [],
'suggested_account' => null,
];
// Add all available Firefly accounts as options
foreach ($this->fireflyAccounts as $account) {
$options['firefly_accounts'][] = [
'id' => $account->id,
'name' => $account->name,
'type' => $account->type,
'currency_code' => $account->currencyCode ?? 'USD',
];
}
// Try to suggest a matching account
$suggested = $this->findMatchingFireflyAccount($simplefinAccount);
if ($suggested) {
$options['suggested_account'] = [
'id' => $suggested->id,
'name' => $suggested->name,
'type' => $suggested->type,
];
}
return $options;
}
/**
* Get created accounts during this session
*/
public function getCreatedAccounts(): array
{
return $this->createdAccounts;
}
}

352
app/Services/SimpleFIN/Conversion/RoutineManager.php

@ -0,0 +1,352 @@
<?php
/*
* RoutineManager.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Conversion;
use App\Exceptions\ImporterErrorException;
use App\Services\Session\Constants;
use App\Services\Shared\Configuration\Configuration;
use App\Services\Shared\Conversion\RoutineManagerInterface;
use App\Services\SimpleFIN\SimpleFINService;
use App\Services\SimpleFIN\Conversion\AccountMapper;
use App\Services\SimpleFIN\Conversion\TransactionTransformer;
use App\Services\SimpleFIN\Model\Account;
use App\Services\Storage\StorageService;
use Illuminate\Support\Str;
/**
* Class RoutineManager
*/
class RoutineManager implements RoutineManagerInterface
{
private string $identifier;
private Configuration $configuration;
private SimpleFINService $simpleFINService;
private AccountMapper $accountMapper;
private TransactionTransformer $transformer;
/**
* RoutineManager constructor.
*/
public function __construct(?string $identifier = null)
{
app('log')->debug('Constructed SimpleFIN RoutineManager');
$this->identifier = $identifier ?? Str::random(16);
$this->simpleFINService = app(SimpleFINService::class);
$this->accountMapper = new AccountMapper();
$this->transformer = new TransactionTransformer($this->accountMapper);
}
/**
* @throws ImporterErrorException
*/
public function start(): array
{
app('log')->debug('Now in SimpleFIN RoutineManager::start()');
$token = session()->get(Constants::SIMPLEFIN_TOKEN); // Retained for general session validation
$bridgeUrl = session()->get(Constants::SIMPLEFIN_BRIDGE_URL); // Retained for general session validation
$allAccountsSimpleFINData = session()->get(
Constants::SIMPLEFIN_ACCOUNTS_DATA,
[]
);
if (
empty($token) ||
empty($bridgeUrl) ||
empty($allAccountsSimpleFINData)
) {
app('log')->error(
'SimpleFIN session data incomplete for conversion.',
[
'has_token' => !empty($token),
'has_bridge_url' => !empty($bridgeUrl),
'has_accounts_data' => !empty($allAccountsSimpleFINData),
]
);
throw new ImporterErrorException(
'SimpleFIN session data (token, URL, or accounts data) not found or incomplete'
);
}
$transactions = [];
$accounts = $this->configuration->getAccounts();
$dateRange = $this->getDateRange();
app('log')->info('Processing SimpleFIN accounts', [
'account_count' => count($accounts),
'date_range' => $dateRange,
]);
foreach ($accounts as $simplefinAccountId => $fireflyAccountId) {
// Handle account creation if requested (fireflyAccountId === 0 means "create_new")
if (0 === $fireflyAccountId) {
$newAccountData =
$this->configuration->getNewAccounts()[
$simplefinAccountId
] ?? null;
if (!$newAccountData) {
app('log')->error(
"No new account data found for SimpleFIN account: {$simplefinAccountId}"
);
continue;
}
// Validate required fields for account creation
if (empty($newAccountData['name'])) {
app('log')->error(
"Account name is required for creating SimpleFIN account: {$simplefinAccountId}"
);
continue;
}
// Find the SimpleFIN account data for account creation
$simplefinAccountData = null;
foreach ($allAccountsSimpleFINData as $accountData) {
if ($accountData['id'] === $simplefinAccountId) {
$simplefinAccountData = $accountData;
break;
}
}
if (!$simplefinAccountData) {
app('log')->error(
"SimpleFIN account data not found for ID: {$simplefinAccountId}"
);
continue;
}
// Prepare account creation configuration with defaults
$accountConfig = [
'name' => $newAccountData['name'],
'type' => $newAccountData['type'] ?? 'asset',
'currency' => $newAccountData['currency'] ?? 'USD',
];
// Add opening balance if provided
if (
!empty($newAccountData['opening_balance']) &&
is_numeric($newAccountData['opening_balance'])
) {
$accountConfig['opening_balance'] =
$newAccountData['opening_balance'];
$accountConfig['opening_balance_date'] = date('Y-m-d');
}
app('log')->info('Creating new Firefly III account', [
'simplefin_account_id' => $simplefinAccountId,
'account_config' => $accountConfig,
]);
// Create SimpleFIN Account object and create Firefly III account
$simplefinAccount = Account::fromArray($simplefinAccountData);
$accountMapper = new AccountMapper();
$createdAccount = $accountMapper->createFireflyAccount(
$simplefinAccount,
$accountConfig
);
if ($createdAccount) {
// Account was created immediately - update configuration
$fireflyAccountId = $createdAccount->id;
$updatedAccounts = $this->configuration->getAccounts();
$updatedAccounts[$simplefinAccountId] = $fireflyAccountId;
$this->configuration->setAccounts($updatedAccounts);
// CRITICAL: Update local accounts mapping to reflect the new account ID
// This ensures TransactionTransformer receives the correct account ID mapping
$accounts = $this->configuration->getAccounts();
app('log')->info(
'Successfully created new Firefly III account',
[
'simplefin_account_id' => $simplefinAccountId,
'firefly_account_id' => $fireflyAccountId,
'account_name' => $createdAccount->name,
'account_type' => $accountConfig['type'],
'currency' => $accountConfig['currency'],
]
);
} else {
// Account creation failed - this is a critical error that must be reported
$errorMessage = sprintf(
'CRITICAL: Failed to create Firefly III account "%s" (type: %s, currency: %s). Cannot proceed with transaction import for this account.',
$accountConfig['name'],
$accountConfig['type'],
$accountConfig['currency']
);
app('log')->error($errorMessage, [
'simplefin_account_id' => $simplefinAccountId,
'account_name' => $accountConfig['name'],
'account_type' => $accountConfig['type'],
'currency' => $accountConfig['currency'],
]);
// Throw exception to prevent silent failure - user must be notified
throw new ImporterErrorException($errorMessage);
}
}
// Find the specific SimpleFIN account data array for the current $simplefinAccountId.
// $allAccountsSimpleFINData is an indexed array of account data arrays.
$currentSimpleFINAccountData = null;
foreach ($allAccountsSimpleFINData as $accountDataFromArrayInLoop) {
if (
isset($accountDataFromArrayInLoop['id']) &&
$accountDataFromArrayInLoop['id'] === $simplefinAccountId
) {
$currentSimpleFINAccountData = $accountDataFromArrayInLoop;
break;
}
}
if (null === $currentSimpleFINAccountData) {
app('log')->error(
'Failed to find SimpleFIN account raw data in session for current account ID during transformation.',
['simplefin_account_id_sought' => $simplefinAccountId]
);
// If the account data for this ID isn't found, we can't process its transactions.
// This might indicate an inconsistency in session data or configuration.
continue; // Skip to the next account in $accounts.
}
try {
app('log')->debug(
"Extracting transactions for account {$simplefinAccountId} from stored data"
);
// Fetch transactions for the current account using the new method signature,
// passing the complete SimpleFIN accounts data retrieved from the session.
$accountTransactions = $this->simpleFINService->fetchTransactions(
$allAccountsSimpleFINData, // Pass the full dataset
$simplefinAccountId,
$dateRange
);
app('log')->debug(
"Extracted {count} transactions for account {$simplefinAccountId}",
['count' => count($accountTransactions)]
);
// $accountTransactions now contains raw transaction data arrays (from SimpleFIN JSON)
foreach ($accountTransactions as $transactionData) {
// Renamed $transactionObject to $transactionData for clarity
try {
// Use current account mapping (accounts are created immediately, no deferred creation)
$accountMappingForTransformer = $accounts;
// The transformer now expects:
// 1. Raw transaction data (array)
// 2. Parent SimpleFIN account data (array)
// 3. Full Firefly III account mapping configuration (array)
// 4. New account configuration data (array) - contains user-provided names
$transformedTransaction = $this->transformer->transform(
$transactionData,
$currentSimpleFINAccountData, // The specific SimpleFIN account data for this transaction's parent
$accountMappingForTransformer, // Current mapping with actual account IDs
$this->configuration->getNewAccounts() // User-provided account configuration data
);
// Skip zero-amount transactions that transformer filtered out
if (empty($transformedTransaction)) {
continue;
}
// Wrap transaction in group structure expected by Firefly III
$transactionGroup = [
'group_title' =>
$transformedTransaction['description'] ??
'SimpleFIN Transaction',
'transactions' => [$transformedTransaction],
];
$transactions[] = $transactionGroup;
} catch (ImporterErrorException $e) {
app('log')->warning(
'Transaction transformation failed for a specific transaction.',
[
'simplefin_account_id' => $simplefinAccountId,
'transaction_id' =>
isset($transactionData['id']) &&
is_scalar($transactionData['id'])
? (string)$transactionData['id']
: 'unknown',
'error' => $e->getMessage(),
// Avoid logging full $transactionData unless necessary for deep debug, could be large/sensitive.
]
);
}
}
} catch (ImporterErrorException $e) {
app('log')->error('Failed to fetch transactions for account', [
'account' => $simplefinAccountId,
'error' => $e->getMessage(),
]);
throw $e;
}
}
app('log')->info('SimpleFIN conversion completed', [
'total_transactions' => count($transactions),
]);
return $transactions;
}
public function setConfiguration(Configuration $configuration): void
{
$this->configuration = $configuration;
}
public function getIdentifier(): string
{
return $this->identifier;
}
#[\Override]
public function getServiceAccounts(): array
{
return session()->get(Constants::SIMPLEFIN_ACCOUNTS_DATA, []);
}
/**
* Get date range for transaction fetching
*/
private function getDateRange(): array
{
$dateAfter = $this->configuration->getDateNotBefore();
$dateBefore = $this->configuration->getDateNotAfter();
return [
'start' => !empty($dateAfter) ? $dateAfter : null,
'end' => !empty($dateBefore) ? $dateBefore : null,
];
}
}

700
app/Services/SimpleFIN/Conversion/TransactionTransformer.php

@ -0,0 +1,700 @@
<?php
/*
* TransactionTransformer.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Conversion;
use App\Support\Http\CollectsAccounts;
use Carbon\Carbon;
// Removed SimpleFINModel imports as we now use arrays
/**
* Class TransactionTransformer
*/
class TransactionTransformer
{
use CollectsAccounts;
private array $expenseAccounts = [];
private array $revenueAccounts = [];
private bool $accountsCollected = false;
private array $pendingTransactionClusters = []; // For clustering similar transactions in clean instances
/**
* Transform SimpleFIN transaction data (array) to Firefly III transaction format
* @param array $transactionData Raw transaction data from SimpleFIN JSON
* @param array $simpleFINAccountData Raw account data from SimpleFIN JSON for the account this transaction belongs to
* @param array $accountMapping Mapping configuration for Firefly III accounts
* @param array $newAccountConfig User-provided new account configuration data
* @return array
*/
public function transform(array $transactionData, array $simpleFINAccountData, array $accountMapping = [], array $newAccountConfig = []): array
{
// Ensure amount is a float. SimpleFIN provides it as a string.
$amount = isset($transactionData['amount']) ? (float) $transactionData['amount'] : 0.0;
// Skip zero-amount transactions as they're invalid for Firefly III
if (abs($amount) === 0.0) {
app('log')->warning('Skipping zero-amount transaction', [
'transaction_id' => $transactionData['id'] ?? 'unknown',
'description' => $transactionData['description'] ?? 'unknown'
]);
return [];
}
$isDeposit = $amount >= 0;
$absoluteAmount = abs($amount);
// Determine transaction type and accounts
if ($isDeposit) {
$type = 'deposit';
$sourceAccount = $this->getCounterAccount($transactionData, $isDeposit);
$destinationAccount = $this->getFireflyAccount($simpleFINAccountData, $accountMapping, $newAccountConfig);
} else {
$type = 'withdrawal';
$sourceAccount = $this->getFireflyAccount($simpleFINAccountData, $accountMapping, $newAccountConfig);
$destinationAccount = $this->getCounterAccount($transactionData, $isDeposit);
}
// Use 'posted' date as the primary transaction date.
// SimpleFIN 'posted' is a UNIX timestamp.
$transactionTimestamp = isset($transactionData['posted']) ? (int)$transactionData['posted'] : Carbon::now()->timestamp;
$transactionDateCarbon = Carbon::createFromTimestamp($transactionTimestamp);
return [
'type' => $type,
'date' => $transactionDateCarbon->format('Y-m-d'),
'amount' => number_format($absoluteAmount, 2, '.', ''),
'description' => $this->sanitizeDescription($transactionData['description'] ?? 'N/A'),
'source_id' => $sourceAccount['id'] ?? null,
'source_name' => $sourceAccount['name'] ?? null,
'source_iban' => $sourceAccount['iban'] ?? null,
'source_number' => $sourceAccount['number'] ?? null,
'source_bic' => $sourceAccount['bic'] ?? null,
'destination_id' => $destinationAccount['id'] ?? null,
'destination_name' => $destinationAccount['name'] ?? null,
'destination_iban' => $destinationAccount['iban'] ?? null,
'destination_number' => $destinationAccount['number'] ?? null,
'destination_bic' => $destinationAccount['bic'] ?? null,
'currency_id' => null,
'currency_code' => $this->getCurrencyCode($simpleFINAccountData),
'foreign_currency_id' => null,
'foreign_currency_code' => null,
'foreign_amount' => null,
'budget_id' => null,
'budget_name' => null,
'category_id' => null,
'category_name' => $this->extractCategory($transactionData),
'bill_id' => null,
'bill_name' => null,
'reconciled' => false,
'notes' => $this->buildNotes($transactionData),
'tags' => $this->extractTags($transactionData),
'internal_reference' => $transactionData['id'] ?? null,
'external_id' => $this->buildExternalId($transactionData, $simpleFINAccountData),
'external_url' => null,
'original_source' => 'simplefin-v1',
'recurrence_id' => null,
'bunq_payment_id' => null,
'import_hash_v2' => $this->generateImportHash($transactionData, $simpleFINAccountData),
'sepa_cc' => null,
'sepa_ct_op' => null,
'sepa_ct_id' => null,
'sepa_db' => null,
'sepa_country' => null,
'sepa_ep' => null,
'sepa_ci' => null,
'sepa_batch_id' => null,
'interest_date' => null,
'book_date' => $this->getBookDate($transactionData),
'process_date' => $this->getProcessDate($transactionData),
'due_date' => null,
'payment_date' => null,
'invoice_date' => null,
];
}
/**
* Get the Firefly III account information from mapping or account data
*/
private function getFireflyAccount(array $simpleFINAccountData, array $accountMapping, array $newAccountConfig = []): array
{
$accountKey = $simpleFINAccountData['id'] ?? null;
// Check for user-provided account name first, then fall back to SimpleFIN account name
$userProvidedName = null;
if ($accountKey && isset($newAccountConfig[$accountKey]['name'])) {
$userProvidedName = $newAccountConfig[$accountKey]['name'];
}
$accountName = $userProvidedName ?? $simpleFINAccountData['name'] ?? 'Unknown SimpleFIN Account';
// Check if account is mapped and has a valid (non-zero) Firefly III account ID
if ($accountKey && isset($accountMapping[$accountKey]) && $accountMapping[$accountKey] > 0) {
return [
'id' => $accountMapping[$accountKey], // Configuration maps SimpleFIN account ID directly to Firefly account ID
'name' => $accountName,
'iban' => null,
'number' => $accountKey,
'bic' => null,
];
}
// No mapping or mapped to 0 (deferred creation) - return null ID to trigger name-based account creation
return [
'id' => null,
'name' => $accountName,
'iban' => null,
'number' => $accountKey,
'bic' => null,
];
}
/**
* Get counter account (revenue/expense account) based on transaction data
*/
private function getCounterAccount(array $transactionData, bool $isDeposit): array
{
$description = $transactionData['description'] ?? 'N/A';
// Ensure accounts are collected
$this->ensureAccountsCollected();
// Try to find existing expense or revenue account first
$existingAccount = $this->findExistingAccount($description, $isDeposit);
if ($existingAccount) {
return [
'id' => $existingAccount['id'],
'name' => $existingAccount['name'],
'iban' => null,
'number' => null,
'bic' => null,
];
}
// For clean instances: try clustering when no existing accounts found
// This includes both clean instances (successful collection of zero accounts)
// and failed collection scenarios
if (config('simplefin.enable_transaction_clustering', true)) {
$accountsToCheck = $isDeposit ? $this->revenueAccounts : $this->expenseAccounts;
if (empty($accountsToCheck)) {
$clusteredAccountName = $this->findClusteredAccountName($description, $isDeposit);
if ($clusteredAccountName) {
return [
'id' => null,
'name' => $clusteredAccountName,
'iban' => null,
'number' => null,
'bic' => null,
];
}
}
}
// Check if automatic account creation is enabled
if (!config('simplefin.auto_create_expense_accounts', true)) {
app('log')->warning(sprintf('Auto-creation disabled. No %s account will be created for "%s"',
$isDeposit ? 'revenue' : 'expense',
$description
));
// Return a generic account name instead of creating new ones
$genericAccountName = $isDeposit ? 'Unmatched Revenue' : 'Unmatched Expenses';
return [
'id' => null,
'name' => $genericAccountName,
'iban' => null,
'number' => null,
'bic' => null,
];
}
// Fallback: extract meaningful counter account name from description
$counterAccountName = $this->extractCounterAccountName($description);
app('log')->info(sprintf('Creating new %s account "%s" for transaction "%s"',
$isDeposit ? 'revenue' : 'expense',
$counterAccountName,
$description
));
return [
'id' => null,
'name' => $counterAccountName,
'iban' => null,
'number' => null,
'bic' => null,
];
}
/**
* Extract a meaningful counter account name from transaction description
*/
private function extractCounterAccountName(string $description): string
{
// Clean up and format the description for use as account name
$cleaned = trim($description);
// Remove common prefixes/suffixes that don't help identify the account
$patterns = [
'/^(PAYMENT|DEPOSIT|TRANSFER|DEBIT|CREDIT)\s+/i',
'/\s+(PAYMENT|DEPOSIT|TRANSFER|DEBIT|CREDIT)$/i',
'/^(FROM|TO)\s+/i',
'/\s+\d{4}[-\/]\d{2}[-\/]\d{2}.*$/', // Remove trailing dates
];
foreach ($patterns as $pattern) {
$cleaned = preg_replace($pattern, '', $cleaned);
}
$cleaned = trim($cleaned);
// If we end up with an empty string, use a generic name
if (empty($cleaned)) {
$cleaned = 'Unknown';
}
// Limit length to reasonable size
if (strlen($cleaned) > 100) {
$cleaned = substr($cleaned, 0, 97) . '...';
}
return $cleaned;
}
/**
* Get currency code, handling custom currencies
*/
private function getCurrencyCode(array $simpleFINAccountData): string
{
$currency = $simpleFINAccountData['currency'] ?? 'USD'; // Default to USD if not present
// Replicate basic logic from SimpleFINAccount::isCustomCurrency() if it checked for 'XXX' or non-standard codes.
// For now, pass through, or use a simple check. Let Firefly III handle currency validation.
// If currency code is not 3 uppercase letters, SimpleFIN spec might imply it's "custom".
// The previous code returned 'XXX' for custom.
if (strlen($currency) === 3 && ctype_upper($currency)) {
return $currency;
}
return 'XXX'; // Default for non-standard or missing currency codes, matching previous behavior.
}
/**
* Extract category from transaction extra data
*/
private function extractCategory(array $transactionData): ?string
{
$extra = $transactionData['extra'] ?? null;
if (!is_array($extra)) {
return null;
}
// Check common category field names
$categoryFields = ['category', 'Category', 'CATEGORY', 'merchant_category', 'transaction_category'];
foreach ($categoryFields as $field) {
if (isset($extra[$field]) && !empty($extra[$field])) {
return (string) $extra[$field];
}
}
return null;
}
/**
* Extract tags from transaction extra data
*/
private function extractTags(array $transactionData): array
{
$tags = [];
if (isset($transactionData['pending']) && $transactionData['pending'] === true) {
$tags[] = 'pending';
}
$extra = $transactionData['extra'] ?? null;
if (!is_array($extra)) {
// If no extra data, or not an array, return current tags (e.g. only 'pending' if applicable)
return array_unique($tags);
}
// Look for tags in extra data
if (isset($extra['tags']) && is_array($extra['tags'])) {
$tags = array_merge($tags, $extra['tags']);
}
// Add organization domain as tag if available
// Note: We don't have account info here, so this would need to be passed in
return array_unique($tags);
}
/**
* Build notes from transaction extra data
*/
private function buildNotes(array $transactionData): ?string
{
$notes = [];
$extra = $transactionData['extra'] ?? null;
if (isset($transactionData['pending']) && $transactionData['pending'] === true) {
$notes[] = 'Transaction is pending';
}
if (is_array($extra)) {
// Add any extra fields that might be useful as notes
$noteFields = ['memo', 'notes', 'reference', 'check_number'];
foreach ($noteFields as $field) {
if (isset($extra[$field]) && !empty($extra[$field])) {
$notes[] = sprintf('%s: %s', ucfirst($field), $extra[$field]);
}
}
}
return empty($notes) ? null : implode("\n", $notes);
}
/**
* Build external ID for transaction
*/
private function buildExternalId(array $transactionData, array $simpleFINAccountData): string
{
return sprintf('simplefin-%s-%s', $simpleFINAccountData['id'] ?? 'unknown_account', $transactionData['id'] ?? 'unknown_transaction');
}
/**
* Generate import hash for duplicate detection
*/
private function generateImportHash(array $transactionData, array $simpleFINAccountData): string
{
$postedTimestamp = isset($transactionData['posted']) ? (int)$transactionData['posted'] : time();
$date = Carbon::createFromTimestamp($postedTimestamp)->format('Y-m-d');
$data = [
'account_id' => $simpleFINAccountData['id'] ?? 'unknown_account',
'transaction_id' => $transactionData['id'] ?? 'unknown_transaction',
'amount' => $transactionData['amount'] ?? '0.00',
'description' => $transactionData['description'] ?? 'N/A',
'date' => $date,
];
return hash('sha256', json_encode($data));
}
/**
* Get book date from transaction data (using 'posted' timestamp)
*/
private function getBookDate(array $transactionData): ?string
{
if (isset($transactionData['posted']) && (int)$transactionData['posted'] > 0) {
return Carbon::createFromTimestamp((int)$transactionData['posted'])->format('Y-m-d');
}
return null;
}
/**
* Get process date from transaction data
* SimpleFIN JSON does not typically include a separate 'transacted_at'.
* This method will return null unless 'transacted_at' is explicitly in $transactionData.
*/
private function getProcessDate(array $transactionData): ?string
{
if (isset($transactionData['transacted_at']) && (int)$transactionData['transacted_at'] > 0) {
return Carbon::createFromTimestamp((int)$transactionData['transacted_at'])->format('Y-m-d');
}
return null;
}
/**
* Ensure expense and revenue accounts are collected from Firefly III
*/
private function ensureAccountsCollected(): void
{
if ($this->accountsCollected) {
return;
}
// Check if smart matching is enabled before attempting collection
if (!config('simplefin.smart_expense_matching', true)) {
app('log')->debug('Smart expense matching is disabled, skipping account collection');
$this->expenseAccounts = [];
$this->revenueAccounts = [];
$this->accountsCollected = true;
return;
}
try {
// Verify authentication context exists before making API calls
$baseUrl = \App\Services\Shared\Authentication\SecretManager::getBaseUrl();
$accessToken = \App\Services\Shared\Authentication\SecretManager::getAccessToken();
if (empty($baseUrl) || empty($accessToken)) {
app('log')->warning('Missing authentication context for account collection, skipping smart matching');
$this->expenseAccounts = [];
$this->revenueAccounts = [];
$this->accountsCollected = true;
return;
}
app('log')->debug('Collecting expense accounts from Firefly III');
$this->expenseAccounts = $this->collectExpenseAccounts();
app('log')->debug('Collecting revenue accounts from Firefly III');
$this->revenueAccounts = $this->collectRevenueAccounts();
app('log')->debug(sprintf('Collected %d expense accounts and %d revenue accounts',
count($this->expenseAccounts),
count($this->revenueAccounts)
));
$this->accountsCollected = true;
} catch (\Exception $e) {
app('log')->error(sprintf('Failed to collect accounts: %s', $e->getMessage()));
app('log')->debug('Continuing without smart expense matching due to collection failure');
$this->expenseAccounts = [];
$this->revenueAccounts = [];
$this->accountsCollected = true; // Mark as collected to avoid repeated failures
}
}
/**
* Find existing expense or revenue account that matches the transaction description
*/
private function findExistingAccount(string $description, bool $isDeposit): ?array
{
$accountsToSearch = $isDeposit ? $this->revenueAccounts : $this->expenseAccounts;
$accountType = $isDeposit ? 'revenue' : 'expense';
if (empty($accountsToSearch)) {
app('log')->debug(sprintf('No %s accounts to search', $accountType));
return null;
}
// Normalize description for matching
$normalizedDescription = $this->normalizeForMatching($description);
// Try exact matches first
foreach ($accountsToSearch as $key => $account) {
$normalizedAccountName = $this->normalizeForMatching($account['name']);
// Check for exact match
if ($normalizedAccountName === $normalizedDescription) {
app('log')->debug(sprintf('Exact match found: "%s" -> "%s"', $description, $account['name']));
return $account;
}
}
// Try fuzzy matching if no exact match found
$bestMatch = $this->findBestFuzzyMatch($normalizedDescription, $accountsToSearch);
if ($bestMatch) {
app('log')->debug(sprintf('Fuzzy match found: "%s" -> "%s" (similarity: %.2f)',
$description,
$bestMatch['account']['name'],
$bestMatch['similarity']
));
return $bestMatch['account'];
}
return null;
}
/**
* Normalize string for matching (lowercase, remove special chars, etc.)
*/
private function normalizeForMatching(string $text): string
{
// Convert to lowercase
$normalized = strtolower($text);
// Remove common transaction prefixes/suffixes
$patterns = [
'/^(payment|deposit|transfer|debit|credit)\s+/i',
'/\s+(payment|deposit|transfer|debit|credit)$/i',
'/^(from|to)\s+/i',
'/\s+\d{4}[-\/]\d{2}[-\/]\d{2}.*$/', // Remove trailing dates
'/\s+#\w+.*$/', // Remove trailing reference numbers
];
foreach ($patterns as $pattern) {
$normalized = preg_replace($pattern, '', $normalized);
}
// Remove special characters and extra spaces
$normalized = preg_replace('/[^a-z0-9\s]/', '', $normalized);
$normalized = preg_replace('/\s+/', ' ', $normalized);
return trim($normalized);
}
/**
* Find best fuzzy match using similarity algorithms
*/
private function findBestFuzzyMatch(string $normalizedDescription, array $accounts): ?array
{
// Check if smart matching is enabled
if (!config('simplefin.smart_expense_matching', true)) {
return null;
}
$bestMatch = null;
$bestSimilarity = 0;
$threshold = config('simplefin.expense_matching_threshold', 0.7);
foreach ($accounts as $key => $account) {
$normalizedAccountName = $this->normalizeForMatching($account['name']);
// Calculate similarity using multiple algorithms
$similarity = $this->calculateSimilarity($normalizedDescription, $normalizedAccountName);
if ($similarity > $bestSimilarity && $similarity >= $threshold) {
$bestSimilarity = $similarity;
$bestMatch = [
'account' => $account,
'similarity' => $similarity
];
}
}
return $bestMatch;
}
/**
* Calculate similarity between two strings using multiple algorithms
*/
private function calculateSimilarity(string $str1, string $str2): float
{
// Use Levenshtein distance for similarity
$maxLen = max(strlen($str1), strlen($str2));
if ($maxLen === 0) {
return 1.0;
}
$levenshtein = levenshtein($str1, $str2);
$levenshteinSimilarity = 1 - ($levenshtein / $maxLen);
// Use similar_text for additional comparison
similar_text($str1, $str2, $percent);
$similarTextSimilarity = $percent / 100;
// Check for substring matches (give bonus for contains)
$substringBonus = 0;
if (strpos($str1, $str2) !== false || strpos($str2, $str1) !== false) {
$substringBonus = 0.2;
}
// Weighted average of different similarity measures
$finalSimilarity = ($levenshteinSimilarity * 0.5) + ($similarTextSimilarity * 0.4) + $substringBonus;
return min(1.0, $finalSimilarity);
}
/**
* Sanitize description for safe display
*/
private function sanitizeDescription(string $description): string
{
// Remove any potentially harmful characters
$sanitized = strip_tags($description);
$sanitized = trim($sanitized);
// Ensure we have a non-empty description
if (empty($sanitized)) {
$sanitized = 'SimpleFIN Transaction';
}
return $sanitized;
}
/**
* Find clustered account name for clean instances without existing accounts
*/
private function findClusteredAccountName(string $description, bool $isDeposit): ?string
{
$accountType = $isDeposit ? 'revenue' : 'expense';
$normalizedDescription = $this->normalizeForMatching($description);
$threshold = config('simplefin.clustering_similarity_threshold', 0.7);
// Check existing clusters for similar descriptions
foreach ($this->pendingTransactionClusters as $clusterName => $cluster) {
if ($cluster['type'] !== $accountType) {
continue;
}
// Check similarity against cluster representative
$similarity = $this->calculateSimilarity($normalizedDescription, $cluster['normalized_name']);
if ($similarity >= $threshold) {
app('log')->debug(sprintf('Clustering "%s" with existing cluster "%s" (similarity: %.2f)',
$description, $clusterName, $similarity));
// Add to existing cluster
$this->pendingTransactionClusters[$clusterName]['descriptions'][] = $description;
$this->pendingTransactionClusters[$clusterName]['count']++;
return $clusterName;
}
}
// No matching cluster found, create new cluster
$clusterName = $this->generateClusterName($description);
$this->pendingTransactionClusters[$clusterName] = [
'type' => $accountType,
'normalized_name' => $normalizedDescription,
'descriptions' => [$description],
'count' => 1,
'created_at' => time()
];
app('log')->debug(sprintf('Created new %s cluster "%s" for "%s"', $accountType, $clusterName, $description));
return $clusterName;
}
/**
* Generate meaningful cluster name from transaction description
*/
private function generateClusterName(string $description): string
{
// Extract core business/merchant name for clustering
$cleaned = $this->extractCounterAccountName($description);
// Further normalize for cluster naming
$clusterName = preg_replace('/\b(payment|deposit|transfer|debit|credit|from|to)\b/i', '', $cleaned);
$clusterName = preg_replace('/\s+/', ' ', trim($clusterName));
// Remove trailing numbers/references that could vary
$clusterName = preg_replace('/\s+\d+\s*$/', '', $clusterName);
$clusterName = preg_replace('/\s+#\w+.*$/', '', $clusterName);
// Ensure minimum meaningful length
if (strlen($clusterName) < 3) {
$clusterName = $cleaned; // Fall back to basic cleaning
}
return trim($clusterName);
}
}

208
app/Services/SimpleFIN/Model/Account.php

@ -0,0 +1,208 @@
<?php
/*
* Account.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Model;
use Carbon\Carbon;
/**
* Class Account
*/
class Account
{
private array $org;
private string $id;
private string $name;
private string $currency;
private string $balance;
private ?string $availableBalance;
private int $balanceDate;
private array $transactions;
private array $extra;
public function __construct(array $data)
{
$this->validateRequiredFields($data);
$this->org = $data['org'];
$this->id = $data['id'];
$this->name = $data['name'];
$this->currency = $data['currency'];
$this->balance = $data['balance'];
$this->availableBalance = $data['available-balance'] ?? null;
$this->balanceDate = $data['balance-date'];
$this->transactions = $data['transactions'] ?? [];
$this->extra = $data['extra'] ?? [];
}
public static function fromArray(array $data): self
{
return new self($data);
}
public function getOrganization(): array
{
return $this->org;
}
public function getOrganizationDomain(): ?string
{
return $this->org['domain'] ?? null;
}
public function getOrganizationName(): ?string
{
return $this->org['name'] ?? null;
}
public function getOrganizationSfinUrl(): string
{
return $this->org['sfin-url'];
}
public function getId(): string
{
return $this->id;
}
public function getName(): string
{
return $this->name;
}
public function getCurrency(): string
{
return $this->currency;
}
public function isCustomCurrency(): bool
{
return str_starts_with($this->currency, 'http://') || str_starts_with($this->currency, 'https://');
}
public function getBalance(): string
{
return $this->balance;
}
public function getBalanceAsFloat(): float
{
return (float) $this->balance;
}
public function getAvailableBalance(): ?string
{
return $this->availableBalance;
}
public function getAvailableBalanceAsFloat(): ?float
{
return $this->availableBalance ? (float) $this->availableBalance : null;
}
public function getBalanceDate(): int
{
return $this->balanceDate;
}
public function getBalanceDateAsCarbon(): Carbon
{
return Carbon::createFromTimestamp($this->balanceDate);
}
public function getTransactions(): array
{
return $this->transactions;
}
public function getTransactionCount(): int
{
return count($this->transactions);
}
public function hasTransactions(): bool
{
return !empty($this->transactions);
}
public function getExtra(): array
{
return $this->extra;
}
public function getExtraValue(string $key): mixed
{
return $this->extra[$key] ?? null;
}
public function hasExtra(string $key): bool
{
return array_key_exists($key, $this->extra);
}
public function toArray(): array
{
return [
'org' => $this->org,
'id' => $this->id,
'name' => $this->name,
'currency' => $this->currency,
'balance' => $this->balance,
'available-balance' => $this->availableBalance,
'balance-date' => $this->balanceDate,
'transactions' => $this->transactions,
'extra' => $this->extra,
];
}
private function validateRequiredFields(array $data): void
{
$requiredFields = ['org', 'id', 'name', 'currency', 'balance', 'balance-date'];
foreach ($requiredFields as $field) {
if (!array_key_exists($field, $data)) {
throw new \InvalidArgumentException(sprintf('Missing required field: %s', $field));
}
}
// Validate organization structure
if (!is_array($data['org'])) {
throw new \InvalidArgumentException('Organization must be an array');
}
if (!isset($data['org']['sfin-url'])) {
throw new \InvalidArgumentException('Organization must have sfin-url');
}
if (!isset($data['org']['domain']) && !isset($data['org']['name'])) {
throw new \InvalidArgumentException('Organization must have either domain or name');
}
// Validate balance-date is numeric
if (!is_numeric($data['balance-date'])) {
throw new \InvalidArgumentException('Balance date must be a numeric timestamp');
}
}
}

199
app/Services/SimpleFIN/Model/Transaction.php

@ -0,0 +1,199 @@
<?php
/*
* Transaction.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Model;
use Carbon\Carbon;
/**
* Class Transaction
*/
class Transaction
{
private string $id;
private int $posted;
private string $amount;
private string $description;
private ?int $transactedAt;
private bool $pending;
private array $extra;
public function __construct(array $data)
{
$this->validateRequiredFields($data);
$this->id = $data['id'];
$this->posted = $data['posted'];
$this->amount = $data['amount'];
$this->description = $data['description'];
$this->transactedAt = $data['transacted_at'] ?? null;
$this->pending = $data['pending'] ?? false;
$this->extra = $data['extra'] ?? [];
}
public static function fromArray(array $data): self
{
return new self($data);
}
public function getId(): string
{
return $this->id;
}
public function getPosted(): int
{
return $this->posted;
}
public function getPostedAsCarbon(): ?Carbon
{
return $this->posted === 0 ? null : Carbon::createFromTimestamp($this->posted);
}
public function getAmount(): string
{
return $this->amount;
}
public function getAmountAsFloat(): float
{
return (float) $this->amount;
}
public function isDeposit(): bool
{
return $this->getAmountAsFloat() >= 0;
}
public function isWithdrawal(): bool
{
return $this->getAmountAsFloat() < 0;
}
public function getAbsoluteAmount(): float
{
return abs($this->getAmountAsFloat());
}
public function getDescription(): string
{
return $this->description;
}
public function getTransactedAt(): ?int
{
return $this->transactedAt;
}
public function getTransactedAtAsCarbon(): ?Carbon
{
return $this->transactedAt ? Carbon::createFromTimestamp($this->transactedAt) : null;
}
public function isPending(): bool
{
return $this->pending;
}
public function isPosted(): bool
{
return !$this->pending && $this->posted > 0;
}
public function getExtra(): array
{
return $this->extra;
}
public function getExtraValue(string $key): mixed
{
return $this->extra[$key] ?? null;
}
public function hasExtra(string $key): bool
{
return array_key_exists($key, $this->extra);
}
public function getEffectiveDate(): Carbon
{
// Use transacted_at if available, otherwise fall back to posted date
if ($this->transactedAt && $this->transactedAt > 0) {
return Carbon::createFromTimestamp($this->transactedAt);
}
if ($this->posted > 0) {
return Carbon::createFromTimestamp($this->posted);
}
// If both are 0 or invalid, return current time
return Carbon::now();
}
public function toArray(): array
{
return [
'id' => $this->id,
'posted' => $this->posted,
'amount' => $this->amount,
'description' => $this->description,
'transacted_at' => $this->transactedAt,
'pending' => $this->pending,
'extra' => $this->extra,
];
}
private function validateRequiredFields(array $data): void
{
$requiredFields = ['id', 'posted', 'amount', 'description'];
foreach ($requiredFields as $field) {
if (!array_key_exists($field, $data)) {
throw new \InvalidArgumentException(sprintf('Missing required field: %s', $field));
}
}
// Validate posted is numeric
if (!is_numeric($data['posted'])) {
throw new \InvalidArgumentException('Posted date must be a numeric timestamp');
}
// Validate amount is numeric string
if (!is_numeric($data['amount'])) {
throw new \InvalidArgumentException('Amount must be a numeric string');
}
// Validate transacted_at if present
if (isset($data['transacted_at']) && !is_numeric($data['transacted_at'])) {
throw new \InvalidArgumentException('Transacted at must be a numeric timestamp');
}
// Validate pending if present
if (isset($data['pending']) && !is_bool($data['pending'])) {
throw new \InvalidArgumentException('Pending must be a boolean');
}
}
}

48
app/Services/SimpleFIN/Request/AccountsRequest.php

@ -0,0 +1,48 @@
<?php
/*
* AccountsRequest.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Request;
use App\Exceptions\ImporterHttpException;
use App\Services\SimpleFIN\Response\AccountsResponse;
use App\Services\Shared\Response\ResponseInterface as SharedResponseInterface;
/**
* Class AccountsRequest
*/
class AccountsRequest extends SimpleFINRequest
{
/**
* @throws ImporterHttpException
*/
public function get(): AccountsResponse
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
$response = $this->authenticatedGet('/accounts');
return new AccountsResponse($response);
}
}

72
app/Services/SimpleFIN/Request/PostAccountRequest.php

@ -0,0 +1,72 @@
<?php
declare(strict_types=1);
namespace App\Services\SimpleFIN\Request;
use GrumpyDictator\FFIIIApiSupport\Request\Request;
use GrumpyDictator\FFIIIApiSupport\Response\Response;
use GrumpyDictator\FFIIIApiSupport\Response\ValidationErrorResponse;
use App\Services\SimpleFIN\Response\PostAccountResponse;
/**
* Class PostAccountRequest
* POST an account to Firefly III.
*/
class PostAccountRequest extends Request
{
/**
* PostAccountRequest constructor.
*
* @param string $url
* @param string $token
*/
public function __construct(string $url, string $token)
{
$this->setBase($url);
$this->setToken($token);
$this->setUri('accounts');
}
/**
* @return Response
*/
public function get(): Response
{
// TODO: Implement get() method.
}
/**
* @return Response
*/
public function post(): Response
{
$data = $this->authenticatedPost();
// found error in response:
if (array_key_exists('errors', $data) && is_array($data['errors'])) {
return new ValidationErrorResponse($data['errors']);
}
// should be impossible to get here (see previous code) but still check.
if (!array_key_exists('data', $data)) {
// return with error array:
if (array_key_exists('errors', $data) && is_array($data['errors'])) {
return new ValidationErrorResponse($data['errors']);
}
// no data array and no error info, that's weird!
$info = ['unknown_field' => [sprintf('Unknown error: %s', json_encode($data, 0, 16))]];
return new ValidationErrorResponse($info);
}
return new PostAccountResponse($data['data'] ?? []);
}
/**
* {@inheritdoc}
*/
public function put(): Response
{
// TODO: Implement put() method.
}
}

159
app/Services/SimpleFIN/Request/SimpleFINRequest.php

@ -0,0 +1,159 @@
<?php
/*
* SimpleFINRequest.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Request;
use App\Exceptions\ImporterErrorException;
use App\Exceptions\ImporterHttpException;
use App\Services\Session\Constants;
use App\Services\Shared\Response\ResponseInterface as SharedResponseInterface;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ClientException;
use GuzzleHttp\Exception\GuzzleException;
use GuzzleHttp\Exception\ServerException;
use Psr\Http\Message\ResponseInterface;
/**
* Class SimpleFINRequest
*/
abstract class SimpleFINRequest
{
private string $apiUrl;
private string $token;
private array $parameters = [];
private float $timeOut;
/**
* @throws ImporterHttpException
*/
abstract public function get(): SharedResponseInterface;
public function setApiUrl(string $apiUrl): void
{
$this->apiUrl = rtrim($apiUrl, '/');
}
public function setToken(string $token): void
{
$this->token = $token;
}
public function setParameters(array $parameters): void
{
app('log')->debug('SimpleFIN request parameters set to: ', $parameters);
$this->parameters = $parameters;
}
public function setTimeOut(float $timeOut): void
{
$this->timeOut = $timeOut;
}
protected function authenticatedGet(string $endpoint): ResponseInterface
{
app('log')->debug(sprintf('SimpleFIN authenticated GET to %s%s', $this->apiUrl, $endpoint));
$client = new Client();
$fullUrl = sprintf('%s%s', $this->apiUrl, $endpoint);
$origin = session()->get(Constants::SIMPLEFIN_BRIDGE_URL);
$options = [
'timeout' => $this->timeOut,
'headers' => [
'Accept' => 'application/json',
'Content-Type' => 'application/json',
'Origin' => $origin,
],
];
// Only add basic auth if userinfo is not already in the apiUrl
// and a token is provided. SimpleFIN typically uses userinfo in the Access URL.
if (strpos($this->apiUrl, '@') === false && !empty($this->token)) {
$options['auth'] = [$this->token, ''];
}
if (!empty($this->parameters)) {
$options['query'] = $this->parameters;
}
try {
$response = $client->get($fullUrl, $options);
} catch (ClientException $e) {
app('log')->error(sprintf('SimpleFIN ClientException: %s', $e->getMessage()));
$this->handleClientException($e);
throw new ImporterHttpException($e->getMessage(), $e->getCode(), $e);
} catch (ServerException $e) {
app('log')->error(sprintf('SimpleFIN ServerException: %s', $e->getMessage()));
throw new ImporterHttpException($e->getMessage(), $e->getCode(), $e);
} catch (GuzzleException $e) {
app('log')->error(sprintf('SimpleFIN GuzzleException: %s', $e->getMessage()));
throw new ImporterHttpException($e->getMessage(), $e->getCode(), $e);
}
return $response;
}
private function handleClientException(ClientException $e): void
{
$statusCode = $e->getResponse()->getStatusCode();
$body = (string) $e->getResponse()->getBody();
app('log')->error(sprintf('SimpleFIN HTTP %d error: %s', $statusCode, $body));
switch ($statusCode) {
case 401:
throw new ImporterErrorException('Invalid SimpleFIN token or authentication failed');
case 403:
throw new ImporterErrorException('Access denied to SimpleFIN resource');
case 404:
throw new ImporterErrorException('SimpleFIN resource not found');
case 429:
throw new ImporterErrorException('SimpleFIN rate limit exceeded');
default:
throw new ImporterErrorException(sprintf('SimpleFIN API error (HTTP %d): %s', $statusCode, $body));
}
}
protected function getApiUrl(): string
{
return $this->apiUrl;
}
protected function getToken(): string
{
return $this->token;
}
protected function getParameters(): array
{
return $this->parameters;
}
protected function getTimeOut(): float
{
return $this->timeOut;
}
}

48
app/Services/SimpleFIN/Request/TransactionsRequest.php

@ -0,0 +1,48 @@
<?php
/*
* TransactionsRequest.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Request;
use App\Exceptions\ImporterHttpException;
use App\Services\SimpleFIN\Response\TransactionsResponse;
use App\Services\Shared\Response\ResponseInterface as SharedResponseInterface;
/**
* Class TransactionsRequest
*/
class TransactionsRequest extends SimpleFINRequest
{
/**
* @throws ImporterHttpException
*/
public function get(): SharedResponseInterface
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
$response = $this->authenticatedGet('');
return new TransactionsResponse($response);
}
}

76
app/Services/SimpleFIN/Response/AccountsResponse.php

@ -0,0 +1,76 @@
<?php
/*
* AccountsResponse.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Response;
use Psr\Http\Message\ResponseInterface;
/**
* Class AccountsResponse
*/
class AccountsResponse extends SimpleFINResponse
{
private array $accounts = [];
public function __construct(ResponseInterface $response)
{
parent::__construct($response);
$this->parseAccounts();
}
public function getAccounts(): array
{
return $this->accounts;
}
public function getAccountCount(): int
{
return count($this->accounts);
}
public function hasAccounts(): bool
{
return !empty($this->accounts);
}
private function parseAccounts(): void
{
$data = $this->getData();
if (empty($data)) {
app('log')->warning('SimpleFIN AccountsResponse: No data to parse');
return;
}
// SimpleFIN API returns accounts in the 'accounts' array
if (isset($data['accounts']) && is_array($data['accounts'])) {
$this->accounts = $data['accounts'];
app('log')->debug(sprintf('SimpleFIN AccountsResponse: Parsed %d accounts', count($this->accounts)));
} else {
app('log')->warning('SimpleFIN AccountsResponse: No accounts array found in response');
$this->accounts = [];
}
}
}

47
app/Services/SimpleFIN/Response/PostAccountResponse.php

@ -0,0 +1,47 @@
<?php
declare(strict_types=1);
namespace App\Services\SimpleFIN\Response;
use GrumpyDictator\FFIIIApiSupport\Response\Response;
use GrumpyDictator\FFIIIApiSupport\Model\Account;
/**
* Class PostAccountResponse.
*/
class PostAccountResponse extends Response
{
private ?Account $account;
private array $rawData;
/**
* Response constructor.
*
* @param array $data
*/
public function __construct(array $data)
{
$this->account = null;
if (isset($data['id'])) {
$this->account = Account::fromArray($data);
}
$this->rawData = $data;
}
/**
* @return array
*/
public function getRawData(): array
{
return $this->rawData;
}
/**
* @return Account|null
*/
public function getAccount(): ?Account
{
return $this->account;
}
}

107
app/Services/SimpleFIN/Response/SimpleFINResponse.php

@ -0,0 +1,107 @@
<?php
/*
* SimpleFINResponse.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Response;
use App\Services\Shared\Response\ResponseInterface as SharedResponseInterface;
use Psr\Http\Message\ResponseInterface;
/**
* Class SimpleFINResponse
*/
abstract class SimpleFINResponse implements SharedResponseInterface
{
private array $data = [];
private int $statusCode;
private string $rawBody;
public function __construct(ResponseInterface $response)
{
$this->statusCode = $response->getStatusCode();
$this->rawBody = (string) $response->getBody();
app('log')->debug(sprintf('SimpleFIN Response: HTTP %d', $this->statusCode));
app('log')->debug(sprintf('SimpleFIN Response body: %s', $this->rawBody));
$this->parseResponse();
}
/**
* Check if the response has an error
*/
public function hasError(): bool
{
return $this->statusCode >= 400;
}
/**
* Get the HTTP status code
*/
public function getStatusCode(): int
{
return $this->statusCode;
}
public function getRawBody(): string
{
return $this->rawBody;
}
public function getData(): array
{
return $this->data;
}
protected function setData(array $data): void
{
$this->data = $data;
}
private function parseResponse(): void
{
if (empty($this->rawBody)) {
app('log')->warning('SimpleFIN Response body is empty');
$this->data = [];
return;
}
$decoded = json_decode($this->rawBody, true);
if (json_last_error() !== JSON_ERROR_NONE) {
app('log')->error(sprintf('SimpleFIN JSON decode error: %s', json_last_error_msg()));
$this->data = [];
return;
}
if (!is_array($decoded)) {
app('log')->error('SimpleFIN Response is not a valid JSON array');
$this->data = [];
return;
}
$this->data = $decoded;
app('log')->debug('SimpleFIN Response parsed successfully');
}
}

82
app/Services/SimpleFIN/Response/TransactionsResponse.php

@ -0,0 +1,82 @@
<?php
/*
* TransactionsResponse.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Response;
use Psr\Http\Message\ResponseInterface;
/**
* Class TransactionsResponse
*/
class TransactionsResponse extends SimpleFINResponse
{
private array $transactions = [];
public function __construct(ResponseInterface $response)
{
parent::__construct($response);
$this->parseTransactions();
}
public function getTransactions(): array
{
return $this->transactions;
}
public function getTransactionCount(): int
{
return count($this->transactions);
}
public function hasTransactions(): bool
{
return !empty($this->transactions);
}
private function parseTransactions(): void
{
$data = $this->getData();
if (empty($data)) {
app('log')->warning('SimpleFIN TransactionsResponse: No data to parse');
return;
}
// SimpleFIN API returns transactions in the 'transactions' array within accounts
if (isset($data['accounts']) && is_array($data['accounts'])) {
$transactions = [];
foreach ($data['accounts'] as $account) {
if (isset($account['transactions']) && is_array($account['transactions'])) {
$transactions = array_merge($transactions, $account['transactions']);
}
}
$this->transactions = $transactions;
app('log')->debug(sprintf('SimpleFIN TransactionsResponse: Parsed %d transactions', count($this->transactions)));
} else {
app('log')->warning('SimpleFIN TransactionsResponse: No accounts array found in response');
$this->transactions = [];
}
}
}

341
app/Services/SimpleFIN/SimpleFINService.php

@ -0,0 +1,341 @@
<?php
/*
* SimpleFINService.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN;
use App\Exceptions\ImporterErrorException;
use App\Services\Session\Constants;
use App\Services\Shared\Configuration\Configuration;
use App\Services\SimpleFIN\Request\AccountsRequest;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ClientException;
use GuzzleHttp\Exception\GuzzleException;
use App\Services\SimpleFIN\Request\TransactionsRequest;
/**
* Class SimpleFINService
*/
class SimpleFINService
{
/**
* @throws ImporterHttpException
* @throws ImporterErrorException
*/
public function fetchAccountsAndInitialData(string $token, string $apiUrl, ?Configuration $configuration = null): array
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
// Check if token is a base64-encoded claim URL
$actualApiUrl = $apiUrl;
$actualToken = $token;
if ($this->isBase64ClaimUrl($token)) {
app('log')->debug('Token appears to be a base64-encoded claim URL, processing exchange');
$actualApiUrl = $this->exchangeClaimUrlForAccessUrl($token);
$actualToken = ''; // Access URL contains auth info
app('log')->debug(sprintf('Successfully exchanged claim URL for access URL: %s', $actualApiUrl));
} else {
// Token is not a base64 claim URL, we need an API URL
if (empty($apiUrl)) {
throw new ImporterErrorException('SimpleFIN API URL is required when token is not a base64-encoded claim URL');
}
}
app('log')->debug(sprintf('SimpleFIN fetching accounts from: %s', $actualApiUrl));
$request = new AccountsRequest();
$request->setToken($actualToken);
$request->setApiUrl($actualApiUrl);
$request->setTimeOut($this->getTimeout());
// Set parameters to retrieve all transactions
// Use a very old start-date (Jan 1, 2000) to ensure we get all historical transactions
$parameters = [
'start-date' => 946684800, // January 1, 2000 00:00:00 UTC
'pending' => ($configuration && $configuration->getPendingTransactions()) ? 1 : 0,
];
$request->setParameters($parameters);
app('log')->debug('SimpleFIN requesting all transactions with parameters', $parameters);
$response = $request->get();
if ($response->hasError()) {
throw new ImporterErrorException(sprintf('SimpleFIN API error: HTTP %d', $response->getStatusCode()));
}
$accounts = $response->getAccounts();
if (empty($accounts)) {
app('log')->warning('SimpleFIN API returned no accounts');
return [];
}
app('log')->debug(sprintf('SimpleFIN fetched %d accounts successfully', count($accounts)));
return $accounts;
}
/**
* Extracts transactions for a specific account from the pre-fetched SimpleFIN accounts data.
* Applies date filtering if specified.
*
* @param array $allAccountsData Array of account data (associative arrays from SimpleFIN JSON).
* @param string $accountId The ID of the account for which to extract transactions.
* @param array|null $dateRange Optional date range for filtering transactions. Expects ['start' => 'Y-m-d', 'end' => 'Y-m-d'].
* @return array List of transaction data (associative arrays from SimpleFIN JSON).
*/
public function fetchTransactions(array $allAccountsData, string $accountId, ?array $dateRange = null): array
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
app('log')->debug(sprintf('SimpleFIN extracting transactions for account ID: "%s" from provided data structure.', $accountId));
$accountTransactions = [];
$accountFound = false;
foreach ($allAccountsData as $accountData) {
// $accountData is now an associative array from the SimpleFIN JSON response.
// Ensure $accountData is an array and has an 'id' key before accessing.
if (is_array($accountData) && isset($accountData['id']) && is_string($accountData['id']) && $accountData['id'] === $accountId) {
$accountFound = true;
// Transactions are expected to be in $accountData['transactions'] as an array
if (isset($accountData['transactions']) && is_array($accountData['transactions'])) {
$accountTransactions = $accountData['transactions'];
} else {
// If 'transactions' key is missing or not an array, treat as no transactions.
$accountTransactions = [];
}
break;
}
}
if (!$accountFound) {
app('log')->warning(sprintf('Account with ID "%s" not found in provided SimpleFIN accounts data.', $accountId));
return [];
}
if (empty($accountTransactions)) {
app('log')->debug(sprintf('No transactions found internally for account ID "%s".', $accountId));
return [];
}
// Apply date range filtering
$filteredTransactions = [];
if (!empty($dateRange) && (isset($dateRange['start']) || isset($dateRange['end']))) {
$startDateTimestamp = null;
$endDateTimestamp = null;
if (isset($dateRange['start']) && !empty($dateRange['start'])) {
try {
$startDateTimestamp = (new \DateTime($dateRange['start'], new \DateTimeZone('UTC')))->setTime(0,0,0)->getTimestamp();
} catch (\Exception $e) {
app('log')->warning('Invalid start date format for SimpleFIN transaction filtering.', ['date' => $dateRange['start'], 'error' => $e->getMessage()]);
}
}
if (isset($dateRange['end']) && !empty($dateRange['end'])) {
try {
$endDateTimestamp = (new \DateTime($dateRange['end'], new \DateTimeZone('UTC')))->setTime(23, 59, 59)->getTimestamp();
} catch (\Exception $e) {
app('log')->warning('Invalid end date format for SimpleFIN transaction filtering.', ['date' => $dateRange['end'], 'error' => $e->getMessage()]);
}
}
foreach ($accountTransactions as $transaction) {
// $transaction is now an associative array from the SimpleFIN JSON response.
// Ensure $transaction is an array and has a 'posted' key before accessing.
if (!is_array($transaction) || !isset($transaction['posted']) || !is_numeric($transaction['posted'])) {
$transactionIdForLog = (is_array($transaction) && isset($transaction['id']) && is_string($transaction['id'])) ? $transaction['id'] : 'unknown';
app('log')->warning('Transaction array missing, not an array, or has invalid "posted" field.', ['transaction_id' => $transactionIdForLog, 'transaction_data' => $transaction]);
continue;
}
$postedTimestamp = (int)$transaction['posted']; // Ensure it's an integer for comparison
$passesFilter = true;
if ($startDateTimestamp !== null && $postedTimestamp < $startDateTimestamp) {
$passesFilter = false;
}
if ($endDateTimestamp !== null && $postedTimestamp > $endDateTimestamp) {
$passesFilter = false;
}
if ($passesFilter) {
$filteredTransactions[] = $transaction;
}
}
app('log')->debug(sprintf('Applied date filtering. Start: %s, End: %s. Original count: %d, Filtered count: %d',
$dateRange['start'] ?? 'N/A', $dateRange['end'] ?? 'N/A', count($accountTransactions), count($filteredTransactions)
));
} else {
$filteredTransactions = $accountTransactions;
}
app('log')->debug(sprintf('SimpleFIN extracted %d transactions for account ID "%s" (after potential filtering).', count($filteredTransactions), $accountId));
return $filteredTransactions;
}
/**
* Test connectivity to SimpleFIN API with given credentials
*/
public function testConnection(string $token, string $apiUrl): bool
{
app('log')->debug(sprintf('Now at %s', __METHOD__));
try {
$accounts = $this->fetchAccountsAndInitialData($token, $apiUrl);
app('log')->debug('SimpleFIN connection test successful');
return true;
} catch (ImporterHttpException|ImporterErrorException $e) {
app('log')->error(sprintf('SimpleFIN connection test failed: %s', $e->getMessage()));
return false;
}
}
/**
* Get demo credentials for testing
*/
public function getDemoCredentials(): array
{
return [
'token' => config('importer.simplefin.demo_token'),
'url' => config('importer.simplefin.demo_url'),
];
}
/**
* Check if a token is a base64-encoded claim URL
*/
private function isBase64ClaimUrl(string $token): bool
{
// Try to decode as base64
$decoded = base64_decode($token, true);
// Check if decode was successful and result looks like a URL
if ($decoded === false) {
return false;
}
// Check if decoded string looks like a SimpleFIN claim URL
return (bool) preg_match('/^https?:\/\/.+\/simplefin\/claim\/.+$/', $decoded);
}
/**
* Exchange a base64-encoded claim URL for an access URL
*
* @throws ImporterErrorException
*/
private function exchangeClaimUrlForAccessUrl(string $base64ClaimUrl): string
{
app('log')->debug('Exchanging SimpleFIN claim URL for access URL');
// Decode the base64 claim URL
$claimUrl = base64_decode($base64ClaimUrl, true);
if ($claimUrl === false) {
throw new ImporterErrorException('Invalid base64 encoding in SimpleFIN token');
}
app('log')->debug(sprintf('Decoded claim URL: %s', $claimUrl));
try {
$client = new Client([
'timeout' => $this->getTimeout(),
'verify' => config('importer.connection.verify'),
]);
// Make POST request to claim URL with empty body
// Use user-provided bridge URL as Origin header for CORS
$origin = session()->get(Constants::SIMPLEFIN_BRIDGE_URL);
if (empty($origin)) {
throw new ImporterErrorException('SimpleFIN bridge URL not found in session. Please provide a valid bridge URL.');
}
app('log')->debug(sprintf('SimpleFIN using user-provided Origin: %s', $origin));
$response = $client->post($claimUrl, [
'headers' => [
'Content-Length' => '0',
'Origin' => $origin,
],
]);
$accessUrl = (string) $response->getBody();
if (empty($accessUrl)) {
throw new ImporterErrorException('Empty access URL returned from SimpleFIN claim exchange');
}
// Validate access URL format
if (!filter_var($accessUrl, FILTER_VALIDATE_URL)) {
throw new ImporterErrorException('Invalid access URL format returned from SimpleFIN claim exchange');
}
app('log')->debug('Successfully exchanged claim URL for access URL');
return $accessUrl;
} catch (ClientException $e) {
$statusCode = $e->getResponse()->getStatusCode();
$responseBody = (string) $e->getResponse()->getBody();
app('log')->error(sprintf('SimpleFIN claim URL exchange failed with HTTP %d: %s', $statusCode, $e->getMessage()));
app('log')->error(sprintf('SimpleFIN 403 response body: %s', $responseBody));
if ($statusCode === 403) {
// Log the actual response for debugging
app('log')->error(sprintf('DETAILED 403 ERROR - URL: %s, Response: %s', $claimUrl, $responseBody));
throw new ImporterErrorException(sprintf('SimpleFIN claim URL exchange failed (403 Forbidden): %s', $responseBody ?: 'No response body available'));
}
throw new ImporterErrorException(sprintf('Failed to exchange SimpleFIN claim URL: HTTP %d error - %s', $statusCode, $responseBody ?: $e->getMessage()));
} catch (GuzzleException $e) {
app('log')->error(sprintf('Failed to exchange SimpleFIN claim URL: %s', $e->getMessage()));
throw new ImporterErrorException(sprintf('Failed to exchange SimpleFIN claim URL: %s', $e->getMessage()));
}
}
/**
* Validate SimpleFIN credentials format
*/
public function validateCredentials(string $token, string $apiUrl): array
{
$errors = [];
if (empty($token)) {
$errors[] = 'SimpleFIN token is required';
}
if (empty($apiUrl)) {
$errors[] = 'SimpleFIN bridge URL is required';
} elseif (!filter_var($apiUrl, FILTER_VALIDATE_URL)) {
$errors[] = 'SimpleFIN bridge URL must be a valid URL';
} elseif (!str_starts_with($apiUrl, 'https://')) {
$errors[] = 'SimpleFIN bridge URL must use HTTPS';
}
return $errors;
}
private function getTimeout(): float
{
return (float) config('importer.simplefin.timeout', 30.0);
}
}

511
app/Services/SimpleFIN/Validation/ConfigurationContractValidator.php

@ -0,0 +1,511 @@
<?php
/*
* ConfigurationContractValidator.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\SimpleFIN\Validation;
use App\Exceptions\ImporterErrorException;
use App\Services\Shared\Authentication\SecretManager;
use App\Services\Shared\Configuration\Configuration;
use GrumpyDictator\FFIIIApiSupport\Exceptions\ApiHttpException;
use GrumpyDictator\FFIIIApiSupport\Model\Account;
use GrumpyDictator\FFIIIApiSupport\Request\GetAccountsRequest;
use GrumpyDictator\FFIIIApiSupport\Response\GetAccountsResponse;
use Illuminate\Support\Facades\Log;
/**
* Class ConfigurationContractValidator
*
* Validates the data contract between SimpleFIN configuration and conversion steps
*/
class ConfigurationContractValidator
{
private array $errors = [];
private array $warnings = [];
private array $existingAccounts = [];
private const REQUIRED_ACCOUNT_TYPES = ['asset', 'liability', 'expense', 'revenue'];
private const VALID_ACCOUNT_ROLES = ['defaultAsset', 'sharedAsset', 'savingAsset', 'ccAsset', 'cashWalletAsset'];
private const VALID_LIABILITY_TYPES = ['debt', 'loan', 'mortgage'];
private const VALID_LIABILITY_DIRECTIONS = ['credit', 'debit'];
public function validateConfigurationContract(Configuration $configuration): ValidationResult
{
$this->errors = [];
$this->warnings = [];
Log::debug('Starting SimpleFIN configuration contract validation');
// Load existing accounts first for duplicate validation
$this->loadExistingAccounts();
// Core validation
$this->validateSimpleFINFlow($configuration);
$this->validateSessionData($configuration);
$this->validateAccountMappings($configuration);
$this->validateNewAccountConfigurations($configuration);
$this->validateImportSelections($configuration);
return new ValidationResult(
count($this->errors) === 0,
$this->errors,
$this->warnings
);
}
private function validateSimpleFINFlow(Configuration $configuration): void
{
if ('simplefin' !== $configuration->getFlow()) {
$this->addError('configuration.flow', 'Configuration must be SimpleFIN flow', $configuration->getFlow());
}
}
private function validateSessionData(Configuration $configuration): void
{
// Check for SimpleFIN accounts data in session
$sessionData = session()->get('simplefin_accounts_data');
if (empty($sessionData) || !is_array($sessionData)) {
$this->addError('session.simplefin_accounts_data', 'SimpleFIN accounts data missing from session');
return;
}
// Validate SimpleFIN account structure
foreach ($sessionData as $index => $account) {
$this->validateSimpleFINAccount($account, $index);
}
}
private function validateSimpleFINAccount(array $account, int $index): void
{
$requiredFields = ['id', 'name', 'currency', 'balance', 'balance-date', 'org'];
foreach ($requiredFields as $field) {
if (!array_key_exists($field, $account)) {
$this->addError("session.simplefin_accounts_data.{$index}.{$field}", "Required field '{$field}' missing from SimpleFIN account");
}
}
// Validate currency format (should be 3-letter ISO code)
if (isset($account['currency']) && !preg_match('/^[A-Z]{3}$/', $account['currency'])) {
$this->addWarning("session.simplefin_accounts_data.{$index}.currency", 'Currency should be 3-letter ISO code', $account['currency']);
}
// Validate balance is numeric
if (isset($account['balance']) && !is_numeric($account['balance'])) {
$this->addError("session.simplefin_accounts_data.{$index}.balance", 'Balance must be numeric', $account['balance']);
}
// Validate balance-date is unix timestamp
if (isset($account['balance-date']) && (!is_numeric($account['balance-date']) || $account['balance-date'] < 0)) {
$this->addError("session.simplefin_accounts_data.{$index}.balance-date", 'Balance date must be valid unix timestamp', $account['balance-date']);
}
}
private function validateAccountMappings(Configuration $configuration): void
{
$accounts = $configuration->getAccounts();
if (empty($accounts)) {
$this->addError('configuration.accounts', 'Account mappings cannot be empty');
return;
}
foreach ($accounts as $simplefinId => $fireflyId) {
// Validate SimpleFIN ID format
if (!is_string($simplefinId) || empty($simplefinId)) {
$this->addError('configuration.accounts.key', 'SimpleFIN account ID must be non-empty string', $simplefinId);
}
// Validate Firefly III ID (0 means create new, positive integer means existing account)
if (!is_int($fireflyId) || $fireflyId < 0) {
$this->addError("configuration.accounts.{$simplefinId}", 'Firefly III account ID must be non-negative integer', $fireflyId);
}
}
}
private function validateNewAccountConfigurations(Configuration $configuration): void
{
$newAccounts = $configuration->getNewAccounts();
$accounts = $configuration->getAccounts();
foreach ($accounts as $simplefinId => $fireflyId) {
if (0 === $fireflyId) {
// This account should be created, validate its configuration
if (!isset($newAccounts[$simplefinId])) {
$this->addError("configuration.new_account.{$simplefinId}", 'New account configuration missing for account marked for creation');
continue;
}
$this->validateNewAccountConfig($newAccounts[$simplefinId], $simplefinId);
}
}
// Check for orphaned new account configurations
foreach ($newAccounts as $simplefinId => $config) {
if (!isset($accounts[$simplefinId]) || 0 !== $accounts[$simplefinId]) {
$this->addWarning("configuration.new_account.{$simplefinId}", 'New account configuration exists but account not marked for creation');
}
}
}
private function validateNewAccountConfig(array $config, string $simplefinId): void
{
$requiredFields = ['name', 'type', 'currency', 'opening_balance'];
foreach ($requiredFields as $field) {
if (!isset($config[$field]) || (is_string($config[$field]) && trim($config[$field]) === '')) {
$this->addError("configuration.new_account.{$simplefinId}.{$field}", "Required field '{$field}' missing or empty");
}
}
// Validate account type
if (isset($config['type']) && !in_array($config['type'], self::REQUIRED_ACCOUNT_TYPES, true)) {
$this->addError("configuration.new_account.{$simplefinId}.type", 'Invalid account type', $config['type']);
}
// Validate account role for asset accounts
if (isset($config['type']) && 'asset' === $config['type'] && isset($config['account_role'])) {
if (!in_array($config['account_role'], self::VALID_ACCOUNT_ROLES, true)) {
$this->addError("configuration.new_account.{$simplefinId}.account_role", 'Invalid account role for asset account', $config['account_role']);
}
}
// Validate liability-specific fields
if (isset($config['type']) && 'liability' === $config['type']) {
if (!isset($config['liability_type']) || !in_array($config['liability_type'], self::VALID_LIABILITY_TYPES, true)) {
$this->addError("configuration.new_account.{$simplefinId}.liability_type", 'Liability type required and must be valid', $config['liability_type'] ?? null);
}
if (!isset($config['liability_direction']) || !in_array($config['liability_direction'], self::VALID_LIABILITY_DIRECTIONS, true)) {
$this->addError("configuration.new_account.{$simplefinId}.liability_direction", 'Liability direction required and must be valid', $config['liability_direction'] ?? null);
}
}
// Validate currency format
if (isset($config['currency']) && !preg_match('/^[A-Z]{3}$/', $config['currency'])) {
$this->addError("configuration.new_account.{$simplefinId}.currency", 'Currency must be 3-letter ISO code', $config['currency']);
}
// Validate opening balance is numeric
if (isset($config['opening_balance']) && !is_numeric($config['opening_balance'])) {
$this->addError("configuration.new_account.{$simplefinId}.opening_balance", 'Opening balance must be numeric', $config['opening_balance']);
}
// Validate opening balance date format if provided
if (isset($config['opening_balance_date']) && !preg_match('/^\d{4}-\d{2}-\d{2}$/', $config['opening_balance_date'])) {
$this->addError("configuration.new_account.{$simplefinId}.opening_balance_date", 'Opening balance date must be YYYY-MM-DD format', $config['opening_balance_date']);
}
// Validate account name length and content
if (isset($config['name'])) {
if (strlen($config['name']) > 255) {
$this->addError("configuration.new_account.{$simplefinId}.name", 'Account name too long (max 255 characters)');
}
if (strlen(trim($config['name'])) === 0) {
$this->addError("configuration.new_account.{$simplefinId}.name", 'Account name cannot be empty');
}
}
// Validate no duplicate account name/type combinations
$this->validateNoDuplicateAccount($config, $simplefinId);
}
private function validateImportSelections(Configuration $configuration): void
{
$doImport = session()->get('do_import', []);
if (empty($doImport)) {
$this->addError('session.do_import', 'No accounts selected for import');
return;
}
$accounts = $configuration->getAccounts();
foreach ($doImport as $accountId => $selected) {
if ('1' !== $selected) {
continue; // Not selected for import
}
if (!isset($accounts[$accountId])) {
$this->addError('session.do_import', 'Account selected for import but not in account mappings', $accountId);
}
}
// Validate that all mapped accounts are either selected for import or explicitly excluded
foreach ($accounts as $simplefinId => $fireflyId) {
if (!isset($doImport[$simplefinId])) {
$this->addWarning('session.do_import', 'Account mapped but no import selection specified', $simplefinId);
}
}
}
private function addError(string $field, string $message, $value = null): void
{
$this->errors[] = [
'field' => $field,
'message' => $message,
'value' => $value,
];
Log::error("Configuration contract validation error: {$field} - {$message}", [
'value' => $value,
]);
}
private function addWarning(string $field, string $message, $value = null): void
{
$this->warnings[] = [
'field' => $field,
'message' => $message,
'value' => $value,
];
Log::warning("Configuration contract validation warning: {$field} - {$message}", [
'value' => $value,
]);
}
private function loadExistingAccounts(): void
{
try {
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$request = new GetAccountsRequest($url, $token);
$request->setType(GetAccountsRequest::ALL);
$request->setVerify(config('importer.connection.verify'));
$request->setTimeOut(config('importer.connection.timeout'));
$response = $request->get();
if ($response instanceof GetAccountsResponse) {
$this->existingAccounts = iterator_to_array($response);
Log::debug(sprintf('Loaded %d existing Firefly III accounts for duplicate validation', count($this->existingAccounts)));
}
} catch (ApiHttpException $e) {
Log::warning(sprintf('Could not load existing accounts for duplicate validation: %s', $e->getMessage()));
// Don't fail validation entirely, just log the warning
$this->existingAccounts = [];
}
}
private function validateNoDuplicateAccount(array $config, string $simplefinId): void
{
if (!isset($config['name']) || !isset($config['type'])) {
return; // Cannot validate without name and type
}
$accountName = trim($config['name']);
$accountType = $config['type'];
// Check against existing accounts
foreach ($this->existingAccounts as $existingAccount) {
if (!$existingAccount instanceof Account) {
continue;
}
// Check for exact name match (case-insensitive) and type match
if (strtolower($existingAccount->name) === strtolower($accountName) &&
$existingAccount->type === $accountType) {
$this->addError(
"configuration.new_account.{$simplefinId}.name",
sprintf('Account "%s" of type "%s" already exists. Cannot create duplicate account.', $accountName, $accountType),
$accountName
);
return;
}
}
Log::debug(sprintf('No duplicate found for account "%s" of type "%s"', $accountName, $accountType));
}
/**
* Check if a single account name and type combination already exists
* Used for AJAX duplicate checking during account creation
*
* @param string $accountName
* @param string $accountType
* @return bool
*/
public function checkSingleAccountDuplicate(string $accountName, string $accountType): bool
{
$accountName = trim($accountName);
$accountType = trim($accountType);
// Empty name or type cannot be duplicate
if (empty($accountName) || empty($accountType)) {
Log::debug('DUPLICATE_CHECK: Empty name or type provided');
return false;
}
// Load existing accounts if not already loaded
if (empty($this->existingAccounts)) {
Log::debug('DUPLICATE_CHECK: Loading existing accounts for validation');
$this->loadExistingAccounts();
}
// If loading failed, return false to avoid blocking user (graceful degradation)
if (empty($this->existingAccounts)) {
Log::warning('DUPLICATE_CHECK: No existing accounts loaded, cannot validate duplicates');
return false;
}
// Check against existing accounts
foreach ($this->existingAccounts as $existingAccount) {
if (!$existingAccount instanceof Account) {
continue;
}
// Check for exact name match (case-insensitive) and type match
if (strtolower($existingAccount->name) === strtolower($accountName) &&
$existingAccount->type === $accountType) {
Log::debug('DUPLICATE_CHECK: Found duplicate account', [
'requested_name' => $accountName,
'requested_type' => $accountType,
'existing_name' => $existingAccount->name,
'existing_type' => $existingAccount->type
]);
return true;
}
}
Log::debug('DUPLICATE_CHECK: No duplicate found', [
'requested_name' => $accountName,
'requested_type' => $accountType,
'checked_accounts' => count($this->existingAccounts)
]);
return false;
}
public function validateFormFieldStructure(array $formData): ValidationResult
{
$this->errors = [];
$this->warnings = [];
Log::debug('Validating SimpleFIN form field structure');
// Validate expected form structure
$expectedStructure = [
'do_import' => 'array',
'accounts' => 'array',
'new_account' => 'array'
];
foreach ($expectedStructure as $field => $expectedType) {
if (!isset($formData[$field])) {
$this->addError("form.{$field}", "Required form field '{$field}' missing");
continue;
}
if ('array' === $expectedType && !is_array($formData[$field])) {
$this->addError("form.{$field}", "Form field '{$field}' must be array");
}
}
// Validate new_account structure follows expected pattern
if (isset($formData['new_account']) && is_array($formData['new_account'])) {
foreach ($formData['new_account'] as $accountId => $accountData) {
if (!is_array($accountData)) {
$this->addError("form.new_account.{$accountId}", 'Account data must be array');
continue;
}
$this->validateFormAccountData($accountData, $accountId);
}
}
return new ValidationResult(
count($this->errors) === 0,
$this->errors,
$this->warnings
);
}
private function validateFormAccountData(array $accountData, string $accountId): void
{
$expectedFields = ['name', 'type', 'currency', 'opening_balance'];
foreach ($expectedFields as $field) {
if (!isset($accountData[$field])) {
$this->addError("form.new_account.{$accountId}.{$field}", "Required form field '{$field}' missing");
}
}
// Check for properly structured field names
if (isset($accountData['account_role']) && 'asset' !== ($accountData['type'] ?? '')) {
$this->addWarning("form.new_account.{$accountId}.account_role", 'Account role specified for non-asset account');
}
}
}
/**
* Validation result container
*/
class ValidationResult
{
public function __construct(
private bool $isValid,
private array $errors = [],
private array $warnings = []
) {}
public function isValid(): bool
{
return $this->isValid;
}
public function getErrors(): array
{
return $this->errors;
}
public function getWarnings(): array
{
return $this->warnings;
}
public function hasErrors(): bool
{
return count($this->errors) > 0;
}
public function hasWarnings(): bool
{
return count($this->warnings) > 0;
}
public function getErrorMessages(): array
{
return array_map(fn($error) => $error['message'], $this->errors);
}
public function getWarningMessages(): array
{
return array_map(fn($warning) => $warning['message'], $this->warnings);
}
public function toArray(): array
{
return [
'valid' => $this->isValid,
'errors' => $this->errors,
'warnings' => $this->warnings,
];
}
}

16
app/Services/Spectre/Request/Request.php.rej

@ -0,0 +1,16 @@
diff a/app/Services/Spectre/Request/Request.php b/app/Services/Spectre/Request/Request.php (rejected hunks)
@@ -101,10 +101,10 @@ abstract class Request
$fullUrl,
[
'headers' => [
- 'Accept' => 'application/json',
- 'Content-Type' => 'application/json',
- 'App-id' => $this->getAppId(),
- 'Secret' => $this->getSecret(),
+ 'Accept' => 'application/json',
+ 'Content-Type' => 'application/json',
+ 'App-id' => $this->getAppId(),
+ 'Secret' => $this->getSecret(),
'User-Agent' => sprintf('Firefly III Spectre importer / %s / %s', config('importer.version'), config('auth.line_c')),
],
]

10
app/Support/Http/CollectsAccounts.php

@ -132,6 +132,16 @@ trait CollectsAccounts
Log::debug(sprintf('Processing account #%d ("%s") with type "%s"', $entry->id, $entry->name, $entry->type));
$type = $entry->type;
$iban = (string) $entry->iban;
// For expense and revenue accounts, use account ID as key since they don't have IBANs
if (in_array($type, ['expense', 'revenue'], true)) {
$key = sprintf('id_%d', $entry->id);
Log::debug(sprintf('Collected %s account "%s" under key "%s"', $type, $entry->name, $key));
$return[$key] = ['id' => $entry->id, 'type' => $entry->type, 'name' => $entry->name, 'number' => $entry->number];
continue;
}
// For asset/liability accounts, continue with IBAN-based logic
if ('' === $iban) {
continue;
}

12
app/Support/Http/CollectsAccounts.php.rej

@ -0,0 +1,12 @@
diff a/app/Support/Http/CollectsAccounts.php b/app/Support/Http/CollectsAccounts.php (rejected hunks)
@@ -115,8 +115,8 @@ trait CollectsAccounts
Log::debug(sprintf('Now in collectAccounts("%s")', $type));
// send account list request to Firefly III.
- $token = SecretManager::getAccessToken();
- $url = SecretManager::getBaseUrl();
+ $token = SecretManager::getAccessToken();
+ $url = SecretManager::getBaseUrl();
$request = new GetAccountsRequest($url, $token);
$request->setType($type);
$request->setVerify(config('importer.connection.verify'));

86
app/Support/Internal/CollectsAccounts.php

@ -43,6 +43,7 @@ use GrumpyDictator\FFIIIApiSupport\Model\Account;
use GrumpyDictator\FFIIIApiSupport\Request\GetAccountsRequest;
use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log;
trait CollectsAccounts
{
@ -51,38 +52,77 @@ trait CollectsAccounts
*/
protected function getFireflyIIIAccounts(): array
{
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$accounts = [
Constants::ASSET_ACCOUNTS => [],
Constants::LIABILITIES => [],
];
$url = null;
$token = null;
$request = new GetAccountsRequest($url, $token);
$request->setType(GetAccountsRequest::ASSET);
$request->setVerify(config('importer.connection.verify'));
$request->setTimeOut(config('importer.connection.timeout'));
$response = $request->get();
try {
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
/** @var Account $account */
foreach ($response as $account) {
$accounts[Constants::ASSET_ACCOUNTS][$account->id] = $account;
}
if (empty($url) || empty($token)) {
Log::error('CollectsAccounts::getFireflyIIIAccounts - Base URL or Access Token is empty. Cannot fetch accounts.', ['url_empty' => empty($url), 'token_empty' => empty($token)]);
return $accounts; // Return empty accounts if auth details are missing
}
// also get liabilities
$url = SecretManager::getBaseUrl();
$token = SecretManager::getAccessToken();
$request = new GetAccountsRequest($url, $token);
$request->setVerify(config('importer.connection.verify'));
$request->setTimeOut(config('importer.connection.timeout'));
$request->setType(GetAccountsRequest::LIABILITIES);
$response = $request->get();
// Fetch ASSET accounts
Log::debug('CollectsAccounts::getFireflyIIIAccounts - Fetching ASSET accounts from Firefly III.', ['url' => $url]);
$requestAsset = new GetAccountsRequest($url, $token);
$requestAsset->setType(GetAccountsRequest::ASSET);
$requestAsset->setVerify(config('importer.connection.verify'));
$requestAsset->setTimeOut(config('importer.connection.timeout'));
$responseAsset = $requestAsset->get();
/** @var Account $account */
foreach ($responseAsset as $account) {
$accounts[Constants::ASSET_ACCOUNTS][$account->id] = $account;
}
Log::debug('CollectsAccounts::getFireflyIIIAccounts - Fetched ' . count($accounts[Constants::ASSET_ACCOUNTS]) . ' ASSET accounts.');
/** @var Account $account */
foreach ($response as $account) {
$accounts[Constants::LIABILITIES][$account->id] = $account;
}
// Fetch LIABILITY accounts
// URL and token are likely the same, but re-fetching defensively or if SecretManager has internal state
$url = SecretManager::getBaseUrl(); // Re-fetch in case of any state change, though unlikely
$token = SecretManager::getAccessToken();
if (empty($url) || empty($token)) { // Check again, though highly unlikely to change if first call succeeded.
Log::error('CollectsAccounts::getFireflyIIIAccounts - Base URL or Access Token became empty before fetching LIABILITY accounts.');
return $accounts; // Return partially filled or empty accounts
}
Log::debug('CollectsAccounts::getFireflyIIIAccounts - Fetching LIABILITY accounts from Firefly III.', ['url' => $url]);
$requestLiability = new GetAccountsRequest($url, $token);
$requestLiability->setVerify(config('importer.connection.verify'));
$requestLiability->setTimeOut(config('importer.connection.timeout'));
$requestLiability->setType(GetAccountsRequest::LIABILITIES);
$responseLiability = $requestLiability->get();
/** @var Account $account */
foreach ($responseLiability as $account) {
$accounts[Constants::LIABILITIES][$account->id] = $account;
}
Log::debug('CollectsAccounts::getFireflyIIIAccounts - Fetched ' . count($accounts[Constants::LIABILITIES]) . ' LIABILITY accounts.');
} catch (ApiHttpException $e) {
Log::error('CollectsAccounts::getFireflyIIIAccounts - ApiHttpException while fetching Firefly III accounts.', [
'message' => $e->getMessage(),
'code' => $e->getCode(),
'url' => $url, // Log URL that might have caused issue
'trace' => $e->getTraceAsString(),
]);
// Return the (potentially partially filled) $accounts array so the app doesn't hard crash.
// The view should handle cases where account lists are empty.
} catch (\Exception $e) {
Log::error('CollectsAccounts::getFireflyIIIAccounts - Generic Exception while fetching Firefly III accounts.', [
'message' => $e->getMessage(),
'code' => $e->getCode(),
'url' => $url,
'trace' => $e->getTraceAsString(),
]);
}
Log::debug('CollectsAccounts::getFireflyIIIAccounts - Returning accounts structure.', $accounts);
return $accounts;
}

8
config/importer.php

@ -30,7 +30,7 @@ return [
'nordigen' => true,
'spectre' => true,
'file' => true,
'simplefin' => false,
'simplefin' => true,
],
'flow_titles' => [
'file' => 'File',
@ -38,6 +38,12 @@ return [
'spectre' => 'Spectre',
'simplefin' => 'SimpleFIN',
],
'simplefin' => [
'demo_url' => env('SIMPLEFIN_DEMO_URL', 'https://demo:demo@beta-bridge.simplefin.org/simplefin'),
'demo_token' => env('SIMPLEFIN_DEMO_TOKEN', 'demo'), // This token is used as the password in the demo_url
'bridge_url' => env('SIMPLEFIN_BRIDGE_URL'),
'timeout' => (int) env('SIMPLEFIN_TIMEOUT', 30),
],
'fallback_in_dir' => env('FALLBACK_IN_DIR', false),
'fallback_configuration' => '_fallback.json',
'import_dir_allowlist' => explode(',', env('IMPORT_DIR_ALLOWLIST', '')),

113
config/simplefin.php

@ -0,0 +1,113 @@
<?php
/*
* simplefin.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
return [
/*
|--------------------------------------------------------------------------
| SimpleFIN Configuration
|--------------------------------------------------------------------------
|
| Configuration for SimpleFIN integration
|
*/
/*
|--------------------------------------------------------------------------
| Demo Configuration
|--------------------------------------------------------------------------
*/
'demo_url' => env('SIMPLEFIN_DEMO_URL', ''),
'demo_token' => env('SIMPLEFIN_DEMO_TOKEN', ''),
/*
|--------------------------------------------------------------------------
| Connection Settings
|--------------------------------------------------------------------------
*/
'connection_timeout' => env('SIMPLEFIN_CONNECTION_TIMEOUT', 30),
'request_timeout' => env('SIMPLEFIN_REQUEST_TIMEOUT', 60),
/*
|--------------------------------------------------------------------------
| Transaction Processing
|--------------------------------------------------------------------------
*/
'unique_column_options' => [
'id' => 'Transaction ID',
'account_id' => 'Account ID',
'posted' => 'Posted Date',
'amount' => 'Amount',
'description' => 'Description',
],
/*
|--------------------------------------------------------------------------
| Account Mapping
|--------------------------------------------------------------------------
*/
'account_types' => [
'checking' => 'asset',
'savings' => 'asset',
'credit' => 'debt', // Credit cards are debt accounts
'loan' => 'loan', // Loans use specific loan account type
'mortgage' => 'mortgage', // Mortgages use specific mortgage account type
'investment' => 'asset',
],
/*
|--------------------------------------------------------------------------
| Import Settings
|--------------------------------------------------------------------------
*/
'max_transactions' => env('SIMPLEFIN_MAX_TRANSACTIONS', 10000),
'default_date_range' => env('SIMPLEFIN_DEFAULT_DATE_RANGE', 90), // days
'enable_caching' => env('SIMPLEFIN_ENABLE_CACHING', true),
'cache_duration' => env('SIMPLEFIN_CACHE_DURATION', 3600), // seconds
/*
|--------------------------------------------------------------------------
| Expense Account Assignment
|--------------------------------------------------------------------------
*/
'smart_expense_matching' => env('SIMPLEFIN_SMART_EXPENSE_MATCHING', true),
'expense_matching_threshold' => env('SIMPLEFIN_EXPENSE_MATCHING_THRESHOLD', 0.7), // Restored default for better clustering
'auto_create_expense_accounts' => env('SIMPLEFIN_AUTO_CREATE_EXPENSE_ACCOUNTS', true),
/*
|--------------------------------------------------------------------------
| Transaction Clustering (Clean Instances)
|--------------------------------------------------------------------------
*/
'enable_transaction_clustering' => env('SIMPLEFIN_ENABLE_TRANSACTION_CLUSTERING', true),
'clustering_similarity_threshold' => env('SIMPLEFIN_CLUSTERING_SIMILARITY_THRESHOLD', 0.7),
/*
|--------------------------------------------------------------------------
| Error Handling
|--------------------------------------------------------------------------
*/
'retry_attempts' => env('SIMPLEFIN_RETRY_ATTEMPTS', 3),
'retry_delay' => env('SIMPLEFIN_RETRY_DELAY', 1), // seconds
];

28
resources/js/v2/src/pages/conversion/index.js

@ -84,11 +84,37 @@ let index = function () {
this.pageStatus.status = 'waiting_to_start';
this.postJobStart();
},
collectNewAccountData() {
const newAccountData = {};
const forms = document.querySelectorAll('.new-account-form');
forms.forEach(form => {
const accountId = form.dataset.accountId;
const formData = new FormData(form);
newAccountData[accountId] = {
name: formData.get('account_name'),
type: formData.get('account_type'),
currency: formData.get('account_currency'),
opening_balance: formData.get('opening_balance') || null
};
});
return newAccountData;
},
postJobStart() {
this.triedToStart = true;
this.post.running = true;
const jobStartUrl = './import/convert/start';
window.axios.post(jobStartUrl, null,{params: {identifier: this.identifier}}).then((response) => {
// Collect new account data for SimpleFIN
const newAccountData = this.collectNewAccountData();
const postData = {
identifier: this.identifier,
new_account_data: newAccountData
};
window.axios.post(jobStartUrl, postData).then((response) => {
console.log('POST was OK');
this.getJobStatus();
this.post.running = false;

7
resources/js/v2/src/pages/index/index.js

@ -63,6 +63,13 @@ let index = function () {
this.importFunctions.file = true;
return;
}
if ('NEEDS_OAUTH' === message) {
console.log('OAuth authentication required, redirecting to token page');
window.location.href = tokenPageUrl;
return;
}
// disable all
this.loadingFunctions.file = false;
this.loadingFunctions.gocardless = false;

36
resources/js/v2/src/pages/submit/index.js

@ -40,10 +40,31 @@ let index = function () {
warnings: [],
errors: [],
},
progress: {
currentTransaction: 0,
totalTransactions: 0,
progressPercentage: 0,
},
checkCount: 0,
maxCheckCount: 600,
maxCheckCount: 3600,
manualRefreshAvailable: false,
functionName() {
},
getProgressPercentage() {
return this.progress.progressPercentage;
},
getProgressWidth() {
return this.progress.progressPercentage + '%';
},
getProgressDisplay() {
if (this.progress.totalTransactions === 0) {
return '';
}
return this.progress.currentTransaction + ' / ' + this.progress.totalTransactions + ' transactions';
},
hasProgressData() {
return this.progress.totalTransactions > 0;
},
showJobMessages() {
console.log(this.messages);
@ -58,6 +79,13 @@ let index = function () {
showTooManyChecks() {
return 'too_long_checks' === this.pageStatus.status;
},
refreshStatus() {
console.log('Manual refresh triggered');
this.checkCount = 0;
this.manualRefreshAvailable = false;
this.pageStatus.status = 'submission_running';
this.getJobStatus();
},
showPostError() {
return 'submission_errored' === this.pageStatus.status || this.post.errored
},
@ -105,6 +133,7 @@ let index = function () {
if (this.checkCount >= this.maxCheckCount) {
console.log('Block getJobStatus (' + this.checkCount + ')');
this.pageStatus.status = 'too_long_checks';
this.manualRefreshAvailable = true;
return;
}
const submitUrl = './import/submit/status';
@ -123,6 +152,11 @@ let index = function () {
this.messages.warnings = response.data.warnings;
this.messages.messages = response.data.messages;
// process progress data:
this.progress.currentTransaction = response.data.currentTransaction || 0;
this.progress.totalTransactions = response.data.totalTransactions || 0;
this.progress.progressPercentage = response.data.progressPercentage || 0;
// job has not started yet. Let's wait.
if (false === this.pageStatus.triedToStart && 'waiting_to_start' === this.pageStatus.status) {
this.pageStatus.status = response.data.status;

612
resources/views/v2/components/create-account-widget.blade.php

@ -0,0 +1,612 @@
<!-- Create Account Widget - Hidden by default, shown when "Create New Account" is selected -->
<div class="collapse mt-1" id="create-account-widget-{{ $account['import_account']->id }}">
<div class="card">
<div class="card-body">
<!-- Account Name -->
<div class="row mb-3">
<div class="col-12">
<div class="d-flex justify-content-between align-items-center">
<div class="d-flex align-items-center flex-grow-1">
<label class="form-label mb-0 me-3" style="min-width: 60px;">Name:</label>
<span id="account-name-display-{{ $account['import_account']->id }}" class="fw-bold">{{ $account['import_account']->name ?? 'New Account' }}</span>
<input type="text"
class="form-control form-control-sm d-none"
id="account-name-edit-{{ $account['import_account']->id }}"
name="new_account[{{ str_replace(' ', '_', $account['import_account']->id) }}][name]"
value="{{ $account['import_account']->name ?? 'New Account' }}"
style="display: inline-block; width: auto; min-width: 200px;">
</div>
<div class="btn-group btn-group-sm" role="group">
<button type="button"
class="btn btn-outline-secondary btn-sm"
id="edit-name-btn-{{ $account['import_account']->id }}"
onclick="toggleAccountNameEdit('{{ $account['import_account']->id }}', true)"
title="Edit account name">
<i class="fas fa-pencil-alt"></i>
</button>
<button type="button"
class="btn btn-success btn-sm d-none"
id="commit-name-btn-{{ $account['import_account']->id }}"
onclick="commitAccountNameEdit('{{ $account['import_account']->id }}')"
title="Save account name">
<i class="fas fa-check"></i>
</button>
<button type="button"
class="btn btn-outline-secondary btn-sm d-none"
id="cancel-name-btn-{{ $account['import_account']->id }}"
onclick="toggleAccountNameEdit('{{ $account['import_account']->id }}', false)"
title="Cancel edit">
<i class="fas fa-times"></i>
</button>
</div>
</div>
</div>
</div>
<!-- Account Type -->
<div class="row mb-3">
<div class="col-12">
<div class="d-flex align-items-center">
<label class="form-label mb-0 me-3" style="min-width: 60px;">Type:</label>
<select class="form-control"
id="new-account-type-{{ $account['import_account']->id }}"
name="new_account[{{ str_replace(' ', '_', $account['import_account']->id) }}][type]"
onchange="updateAccountTypeVisibility('{{ $account['import_account']->id }}')"
required>
@php
// Simplified account type inference - default all to Asset Account
$inferredType = 'asset'; // All accounts default to Asset Account
$accountName = strtolower($account['import_account']->name ?? '');
@endphp
<option value="asset" @if($inferredType === 'asset') selected @endif>Asset Account</option>
<option value="liability" @if($inferredType === 'liability') selected @endif>Liability Account</option>
<option value="expense" @if($inferredType === 'expense') selected @endif>Expense Account</option>
<option value="revenue" @if($inferredType === 'revenue') selected @endif>Revenue Account</option>
</select>
<div class="invalid-feedback">
Please select an account type.
</div>
</div>
</div>
</div>
<!-- Account Role Section (Asset accounts only) -->
<div class="row mb-3" id="account-role-section-{{ $account['import_account']->id }}" style="@if($inferredType === 'asset') display: block; @else display: none; @endif">
<div class="col-12">
<div class="d-flex align-items-center">
<label class="form-label mb-0 me-3" style="min-width: 60px;">Role:</label>
<select class="form-control"
id="new-account-role-{{ $account['import_account']->id }}"
name="new_account[{{ str_replace(' ', '_', $account['import_account']->id) }}][account_role]">
@php
// Intelligent role detection for asset accounts
$inferredRole = 'defaultAsset'; // Default fallback
if ($inferredType === 'asset') {
// Negative balance detection (priority for credit cards)
if (isset($account['import_account']->balance) &&
floatval($account['import_account']->balance) < 0) {
$inferredRole = 'ccAsset';
}
// Credit card name pattern detection
elseif (preg_match('/credit\s*card|visa|mastercard|amex|american\s*express|discover/', $accountName)) {
$inferredRole = 'ccAsset';
}
// Savings account detection
elseif (preg_match('/savings|save|high\s*yield|money\s*market|cd|certificate/', $accountName)) {
$inferredRole = 'savingAsset';
}
// Cash wallet detection
elseif (preg_match('/cash|wallet|petty\s*cash/', $accountName) ||
(isset($account['import_account']->balance) &&
floatval($account['import_account']->balance) < 1000 &&
floatval($account['import_account']->balance) > 0)) {
$inferredRole = 'cashWalletAsset';
}
// Shared asset detection (joint accounts)
elseif (preg_match('/joint|shared|family|couple/', $accountName)) {
$inferredRole = 'sharedAsset';
}
}
@endphp
<option value="defaultAsset" @if($inferredRole === 'defaultAsset') selected @endif>Default Asset</option>
<option value="sharedAsset" @if($inferredRole === 'sharedAsset') selected @endif>Shared Asset</option>
<option value="savingAsset" @if($inferredRole === 'savingAsset') selected @endif>Savings Account</option>
<option value="ccAsset" @if($inferredRole === 'ccAsset') selected @endif>Credit Card</option>
<option value="cashWalletAsset" @if($inferredRole === 'cashWalletAsset') selected @endif>Cash Wallet</option>
</select>
<div class="invalid-feedback">
Please select an account role.
</div>
</div>
</div>
</div>
<!-- Liability Role and Direction Section (Liability accounts only) -->
<div id="liability-fields-section-{{ $account['import_account']->id }}" style="@if($inferredType === 'liability') display: block; @else display: none; @endif">
<!-- Role Selection -->
<div class="row mb-3">
<div class="col-12">
<div class="d-flex align-items-center">
<label class="form-label mb-0 me-3" style="min-width: 60px;">Role:</label>
<select class="form-control"
id="liability-type-{{ $account['import_account']->id }}"
name="new_account[{{ str_replace(' ', '_', $account['import_account']->id) }}][liability_type]">
@php
// Intelligent liability role detection
$inferredLiabilityRole = 'debt'; // Default fallback
if (preg_match('/mortgage|home\s*loan/', $accountName)) {
$inferredLiabilityRole = 'mortgage';
}
elseif (preg_match('/loan|auto\s*loan|student\s*loan|personal\s*loan/', $accountName)) {
$inferredLiabilityRole = 'loan';
}
@endphp
<option value="debt" @if($inferredLiabilityRole === 'debt') selected @endif>Debt</option>
<option value="loan" @if($inferredLiabilityRole === 'loan') selected @endif>Loan</option>
<option value="mortgage" @if($inferredLiabilityRole === 'mortgage') selected @endif>Mortgage</option>
</select>
<div class="invalid-feedback">
Please select a liability role.
</div>
</div>
</div>
</div>
<!-- Direction Selection -->
<div class="row mb-3">
<div class="col-12">
<div class="d-flex align-items-center">
<label class="form-label mb-0 me-3" style="min-width: 60px;">Direction:</label>
@php
// Balance-based direction logic
$inferredDirection = 'credit'; // Default fallback
if (isset($account['import_account']->balance)) {
$balance = floatval($account['import_account']->balance);
// Negative balance = we owe them (credit)
// Positive balance = they owe us (debit)
$inferredDirection = ($balance < 0) ? 'credit' : 'debit';
}
@endphp
<div class="btn-group" role="group" data-bs-toggle="buttons">
<input type="radio" class="btn-check"
name="new_account[{{ str_replace(' ', '_', $account['import_account']->id) }}][liability_direction]"
id="direction-credit-{{ $account['import_account']->id }}"
value="credit"
@if($inferredDirection === 'credit') checked @endif>
<label class="btn btn-outline-primary btn-sm" for="direction-credit-{{ $account['import_account']->id }}">
We owe them
</label>
<input type="radio" class="btn-check"
name="new_account[{{ str_replace(' ', '_', $account['import_account']->id) }}][liability_direction]"
id="direction-debit-{{ $account['import_account']->id }}"
value="debit"
@if($inferredDirection === 'debit') checked @endif>
<label class="btn btn-outline-primary btn-sm" for="direction-debit-{{ $account['import_account']->id }}">
They owe us
</label>
</div>
<div class="invalid-feedback">
Please select a liability direction.
</div>
</div>
</div>
</div>
</div>
<!-- Balance and Currency Section -->
<div class="row mb-3">
<div class="col-12">
<!-- Balance Label Row with Edit Button -->
<div class="d-flex justify-content-between align-items-center mb-2">
<label class="form-label mb-0"><i class="fas fa-coins me-2"></i>Balance:</label>
<div class="btn-group btn-group-sm" role="group">
<button type="button"
class="btn btn-outline-secondary btn-sm"
id="edit-balance-btn-{{ $account['import_account']->id }}"
onclick="toggleBalanceCurrencyEdit('{{ $account['import_account']->id }}', true)"
title="Edit balance and currency">
<i class="fas fa-pencil-alt"></i>
</button>
<button type="button"
class="btn btn-success btn-sm d-none"
id="commit-balance-btn-{{ $account['import_account']->id }}"
onclick="commitBalanceCurrencyEdit('{{ $account['import_account']->id }}')"
title="Save changes">
<i class="fas fa-check"></i>
</button>
<button type="button"
class="btn btn-outline-secondary btn-sm d-none"
id="cancel-balance-btn-{{ $account['import_account']->id }}"
onclick="toggleBalanceCurrencyEdit('{{ $account['import_account']->id }}', false)"
title="Cancel edit">
<i class="fas fa-times"></i>
</button>
</div>
</div>
<!-- Balance and Currency Values Row -->
<div class="d-flex align-items-center">
@php
$rawBalance = $account['import_account']->balance ?? null;
$convertedFloat = (float)($rawBalance ?? '0.00');
$displayFormat = number_format($convertedFloat, 2);
@endphp
<!-- Balance Display/Edit -->
<div class="me-3 flex-grow-1">
<span id="balance-display-{{ $account['import_account']->id }}" class="fw-bold">
{{ $displayFormat }}
</span>
<input type="number"
step="0.01"
class="form-control form-control-sm d-none"
id="balance-edit-{{ $account['import_account']->id }}"
name="new_account[{{ str_replace(' ', '_', $account['import_account']->id) }}][opening_balance]"
value="{{ $convertedFloat }}"
placeholder="0.00">
</div>
<!-- Currency Display/Edit -->
<div>
@php
// Check for previously committed currency selection, fall back to SimpleFIN currency, then USD
$accountId = $account['import_account']->id;
$newAccounts = $configuration->getNewAccounts();
$committedCurrency = $newAccounts[$accountId]['currency'] ?? null;
$displayCurrency = $committedCurrency ?? $account['import_account']->currency ?? 'USD';
@endphp
<span id="currency-display-{{ $account['import_account']->id }}" class="fw-bold">
{{ $displayCurrency }}
</span>
<select class="form-control form-control-sm d-none"
id="currency-edit-{{ $account['import_account']->id }}"
name="new_account[{{ str_replace(' ', '_', $account['import_account']->id) }}][currency]">
@php
// Use the same committed currency logic for option selection
$defaultCurrency = $displayCurrency;
@endphp
@foreach($currencies ?? [] as $currencyId => $currencyDisplay)
@php
// Extract ISO code from currency display (e.g., "US Dollar (USD)" -> "USD")
preg_match('/\(([A-Z]{3})\)/', $currencyDisplay, $matches);
$isoCode = $matches[1] ?? 'USD';
@endphp
<option value="{{ $isoCode }}" @if($isoCode === $defaultCurrency) selected @endif>
{{ $currencyDisplay }}
</option>
@endforeach
@if(empty($currencies))
<option value="USD">USD (US Dollar)</option>
@endif
</select>
</div>
</div>
</div>
</div>
<!-- Metadata section removed - debug information not valuable in production widget -->
<!-- Dynamic status footer area -->
<div class="row">
<div class="col-12">
<div class="text-end">
<small id="widget-status-{{ $account['import_account']->id }}" class="text-muted">
<i class="fas fa-check-circle me-1"></i>
Ready for import
</small>
</div>
</div>
</div>
</div>
</div>
</div>
<script>
// Track edit states to prevent form submission during edits
window.accountEditStates = window.accountEditStates || {};
// Initialize visibility on page load
document.addEventListener('DOMContentLoaded', function() {
@foreach($accounts ?? [] as $account)
updateAccountTypeVisibility('{{ $account['import_account']->id }}');
@endforeach
});
// Updated function to handle both account role and liability field visibility
function updateAccountTypeVisibility(accountId) {
updateAccountRoleVisibility(accountId);
updateLiabilityFieldsVisibility(accountId);
}
function updateAccountRoleVisibility(accountId) {
const typeSelect = document.getElementById('new-account-type-' + accountId);
const roleSection = document.getElementById('account-role-section-' + accountId);
if (!typeSelect || !roleSection) {
console.error('Required elements not found for role visibility update:', accountId);
return;
}
if (typeSelect.value === 'asset') {
roleSection.style.display = 'block';
console.log('Showed role section for asset account:', accountId);
} else {
roleSection.style.display = 'none';
console.log('Hidden role section for non-asset account:', accountId);
}
}
function updateLiabilityFieldsVisibility(accountId) {
const typeSelect = document.getElementById('new-account-type-' + accountId);
const liabilitySection = document.getElementById('liability-fields-section-' + accountId);
if (!typeSelect || !liabilitySection) {
console.error('Required elements not found for liability visibility update:', accountId);
return;
}
if (typeSelect.value === 'liability') {
liabilitySection.style.display = 'block';
console.log('Showed liability fields for account:', accountId);
} else {
liabilitySection.style.display = 'none';
console.log('Hidden liability fields for account:', accountId);
}
}
// Enhanced widget visibility control functions with error handling
function showAccountWidget(accountId) {
try {
const widget = document.getElementById('create-account-widget-' + accountId);
if (!widget) {
console.error('Widget element not found for account:', accountId);
return false;
}
if (typeof bootstrap !== 'undefined' && bootstrap.Collapse) {
const collapse = new bootstrap.Collapse(widget, { show: true });
console.log('Showed account widget using Bootstrap for:', accountId);
} else {
widget.classList.add('show');
console.log('Showed account widget using fallback for:', accountId);
}
// Initialize edit states
window.accountEditStates[accountId] = {
nameEditing: false,
balanceEditing: false
};
return true;
} catch (error) {
console.error('Error showing widget for account:', accountId, error);
return false;
}
}
// Account name inline editing functions moved to global layout
// Form submission state management and status updates
function updateFormSubmitState() {
const form = document.querySelector('form');
if (!form) return;
const submitButtons = form.querySelectorAll('[type="submit"]');
let hasActiveEdits = false;
// Check if any account is in edit state and update status text
for (const accountId in window.accountEditStates) {
const state = window.accountEditStates[accountId];
const statusElement = document.getElementById('widget-status-' + accountId);
if (state.nameEditing || state.balanceEditing) {
hasActiveEdits = true;
if (statusElement) {
statusElement.innerHTML = '<i class="fas fa-exclamation-triangle me-1 text-warning"></i>Finish editing before import';
statusElement.className = 'text-warning';
}
} else {
if (statusElement) {
statusElement.innerHTML = '<i class="fas fa-check-circle me-1"></i>Ready for import';
statusElement.className = 'text-muted';
}
}
}
// Disable/enable submit buttons based on edit state
submitButtons.forEach(button => {
if (hasActiveEdits) {
button.disabled = true;
button.title = 'Complete all edits before submitting';
} else {
button.disabled = false;
button.title = '';
}
});
}
function hideAccountWidget(accountId) {
try {
const widget = document.getElementById('create-account-widget-' + accountId);
const select = document.getElementById('account-select-' + accountId);
if (widget) {
if (typeof bootstrap !== 'undefined' && bootstrap.Collapse) {
const collapse = new bootstrap.Collapse(widget, { hide: true });
console.log('Hid account widget using Bootstrap for:', accountId);
} else {
widget.classList.remove('show');
console.log('Hid account widget using fallback for:', accountId);
}
// Clear any validation states
const inputs = widget.querySelectorAll('.is-invalid');
inputs.forEach(input => input.classList.remove('is-invalid'));
}
// Reset dropdown to first non-create option
if (select && select.options.length > 1) {
for (let i = 1; i < select.options.length; i++) {
if (select.options[i].value !== 'create_new') {
select.selectedIndex = i;
break;
}
}
console.log('Reset dropdown selection for account:', accountId);
}
return true;
} catch (error) {
console.error('Error hiding widget for account:', accountId, error);
return false;
}
}
// Real-time validation feedback
function setupWidgetValidation(accountId) {
const widget = document.getElementById('create-account-widget-' + accountId);
if (!widget) return;
const nameInput = widget.querySelector('input[name*="[name]"]');
const typeSelect = widget.querySelector('select[name*="[type]"]');
if (nameInput) {
nameInput.addEventListener('blur', function() {
if (!this.value.trim()) {
this.classList.add('is-invalid');
console.warn('Account name validation failed for:', accountId);
} else {
this.classList.remove('is-invalid');
}
});
nameInput.addEventListener('input', function() {
if (this.value.trim()) {
this.classList.remove('is-invalid');
}
});
}
if (typeSelect) {
typeSelect.addEventListener('change', function() {
if (!this.value) {
this.classList.add('is-invalid');
} else {
this.classList.remove('is-invalid');
}
});
}
}
// Initialize widget on page load
document.addEventListener('DOMContentLoaded', function() {
const accountId = '{{ $account['import_account']->id }}';
console.log('Initializing create account widget for:', accountId);
// Initialize edit states
window.accountEditStates = window.accountEditStates || {};
window.accountEditStates[accountId] = {
nameEditing: false,
balanceEditing: false
};
// Set up real-time validation
setupWidgetValidation(accountId);
// Initialize duplicate checking
if (window.updateDuplicateStatus) {
window.updateDuplicateStatus(accountId);
}
// Set up custom event listeners
document.addEventListener('accountWidgetToggled', function(event) {
const detail = event.detail;
console.log('Account widget toggle event:', detail);
if (detail.accountId === accountId && detail.success && detail.isCreateNew) {
// Widget was successfully shown, set up any additional handlers
setupWidgetValidation(accountId);
// Initialize edit states
window.accountEditStates[accountId] = {
nameEditing: false,
balanceEditing: false
};
}
});
// Set up keyboard shortcuts for inline editing
document.addEventListener('keydown', function(event) {
if (event.key === 'Escape') {
// Cancel any active edits on Escape
const state = window.accountEditStates[accountId];
if (state && state.nameEditing) {
toggleAccountNameEdit(accountId, false);
}
if (state && state.balanceEditing) {
toggleBalanceCurrencyEdit(accountId, false);
}
}
if (event.key === 'Enter' && (event.target.id.includes('name-edit') || event.target.id.includes('balance-edit'))) {
// Commit edits on Enter
if (event.target.id.includes('name-edit')) {
commitAccountNameEdit(accountId);
} else if (event.target.id.includes('balance-edit')) {
commitBalanceCurrencyEdit(accountId);
}
event.preventDefault();
}
});
});
// Duplicate function definitions removed - using the globally accessible versions above
// Form validation enhancement
document.addEventListener('DOMContentLoaded', function() {
const form = document.querySelector('form');
if (form) {
form.addEventListener('submit', function(event) {
// Validate create account widgets that are visible
let isValid = true;
const visibleWidgets = document.querySelectorAll('.collapse.show[id^="create-account-widget-"]');
console.log('Form submission: Validating', visibleWidgets.length, 'visible widgets');
visibleWidgets.forEach(widget => {
const nameInput = widget.querySelector('input[name*="[name]"]');
const typeSelect = widget.querySelector('select[name*="[type]"]');
if (nameInput && !nameInput.value.trim()) {
nameInput.classList.add('is-invalid');
isValid = false;
console.warn('Form validation failed: Account name empty');
} else if (nameInput) {
nameInput.classList.remove('is-invalid');
}
if (typeSelect && !typeSelect.value) {
typeSelect.classList.add('is-invalid');
isValid = false;
console.warn('Form validation failed: Account type not selected');
} else if (typeSelect) {
typeSelect.classList.remove('is-invalid');
}
});
if (!isValid) {
console.error('Form validation failed, preventing submission');
event.preventDefault();
event.stopPropagation();
} else {
console.log('Form validation passed');
}
});
}
});
</script>

246
resources/views/v2/components/firefly-iii-account-generic.blade.php

@ -1,22 +1,238 @@
@if('disabled' !== $account['import_account']->status)
<select style="width:100%;"
class="custom-select custom-select-sm form-control"
name="accounts[{{ $account['import_account']->id }}]">
<!-- loop all Firefly III accounts -->
@foreach($account['firefly_iii_accounts'] as $ff3Account)
<option value="{{ $ff3Account->id }}"
{{-- loop configuration --}}
@foreach($configuration->getAccounts() as $key => $preConfig)
{{-- if this account matches, pre-select dropdown. --}}
@if((string) $key === (string) $account['import_account']->id && $preConfig === $ff3Account->id) selected="selected"
@endif
name="accounts[{{ str_replace(' ', '_', $account['import_account']->id) }}]"
onchange="handleAccountSelection('{{ $account['import_account']->id }}', this.value)"
id="account-select-{{ $account['import_account']->id }}">
<!-- Create New Account option -->
<option value="create_new"
@php
$configuredAccount = $configuration->getAccounts()[$account['import_account']->id] ?? null;
$mappedTo = $account['mapped_to'] ?? null;
$isCreateNewSelected = (!$configuredAccount || $configuredAccount === 'create_new') && !$mappedTo;
@endphp
@if($isCreateNewSelected) selected @endif> Create New Account</option>
<!-- loop all Firefly III account groups (assets, liabilities) -->
@foreach($account['firefly_iii_accounts'] as $accountGroupKey => $accountGroup)
{{-- $accountGroupKey is 'assets' or 'liabilities' --}}
{{-- $accountGroup is the array of account objects --}}
@if(is_array($accountGroup) && count($accountGroup) > 0)
<optgroup label="{{ ucfirst($accountGroupKey) }}">
@foreach($accountGroup as $ff3Account) {{-- $ff3Account is now a single Firefly III Account object/array --}}
<option value="{{ $ff3Account->id ?? '' }}"
@php
$isSelected = false;
// First check if mapped_to matches this account
if (isset($account['mapped_to']) && (string) $account['mapped_to'] === (string) ($ff3Account->id ?? '')) {
$isSelected = true;
}
// Otherwise check configuration for pre-selection
else {
foreach($configuration->getAccounts() as $key => $preConfig) {
if((string) $key === (string) $account['import_account']->id && (int) $preConfig === (int) ($ff3Account->id ?? null)) {
$isSelected = true;
break;
}
}
}
@endphp
@if($isSelected) selected="selected" @endif
label="{{ $ff3Account->name ?? 'Unknown Account' }} @if($ff3Account->iban ?? null) ({{ $ff3Account->iban ?? '' }}) @endif">
{{ $ff3Account->name ?? 'Unknown Account' }} @if($ff3Account->iban ?? null) ({{ $ff3Account->iban ?? '' }}) @endif
</option>
@endforeach
label="{{ $ff3Account->name }} @if($ff3Account->iban) ({{ $ff3Account->iban }}) @endif">
{{ $ff3Account->id }}:
{{ $ff3Account->name }} @if($ff3Account->iban)
({{ $ff3Account->iban }})
@endif
</option>
</optgroup>
@endif
@endforeach
</select>
<!-- Status text for existing account selection -->
<div id="existing-account-status-container-{{ $account['import_account']->id }}" class="p-3">
<div class="row">
<div class="col-12">
<div class="text-end">
<small id="existing-account-status-{{ $account['import_account']->id }}" class="text-success">
<i class="fas fa-check-circle me-1"></i>Ready for import
</small>
</div>
</div>
</div>
</div>
<!-- Hidden field to indicate account creation is requested when create_new is selected -->
<input type="hidden"
id="create-new-indicator-{{ $account['import_account']->id }}"
name="new_account[{{ str_replace(' ', '_', $account['import_account']->id) }}][create]"
value="0">
<input type="hidden"
name="do_import[{{ str_replace(' ', '_', $account['import_account']->id) }}]"
value="1">
<script>
function handleAccountSelection(accountId, selectedValue) {
const isCreateNew = selectedValue === 'create_new';
console.log('Account selection changed:', {
accountId: accountId,
selectedValue: selectedValue,
isCreateNew: isCreateNew
});
// Update hidden field to indicate create new status
const createIndicator = document.getElementById('create-new-indicator-' + accountId);
if (createIndicator) {
createIndicator.value = isCreateNew ? '1' : '0';
console.log('Updated create indicator for account:', accountId, 'value:', createIndicator.value);
} else {
console.warn('Create new indicator not found for account:', accountId);
}
// Toggle widget visibility with enhanced error handling
try {
if (typeof window.toggleAccountNameEditing === 'function') {
window.toggleAccountNameEditing(accountId, isCreateNew);
} else {
// Fallback: directly control widget if toggle function not available
console.warn('toggleAccountNameEditing function not found, using fallback');
const widget = document.getElementById('create-account-widget-' + accountId);
if (widget) {
if (isCreateNew) {
// Show widget
if (typeof bootstrap !== 'undefined' && bootstrap.Collapse) {
const collapse = new bootstrap.Collapse(widget, { show: true });
} else {
widget.classList.add('show');
}
console.log('Showed widget for account:', accountId);
} else {
// Hide widget
if (typeof bootstrap !== 'undefined' && bootstrap.Collapse) {
const collapse = new bootstrap.Collapse(widget, { hide: true });
} else {
widget.classList.remove('show');
}
console.log('Hid widget for account:', accountId);
}
} else {
console.error('Widget not found for account:', accountId);
}
}
} catch (error) {
console.error('Error in widget coordination for account:', accountId, error);
}
// Toggle existing account status container visibility
const existingAccountStatusContainer = document.getElementById('existing-account-status-container-' + accountId);
if (existingAccountStatusContainer) {
if (isCreateNew) {
// Hide entire status container when creating new account
existingAccountStatusContainer.style.display = 'none';
} else {
// Show status container when existing account selected
existingAccountStatusContainer.style.display = 'block';
}
console.log('Updated existing account status container visibility for account:', accountId, 'visible:', !isCreateNew);
} else {
console.warn('Existing account status container not found for account:', accountId);
}
// Validate form fields if widget is being shown
if (isCreateNew) {
setTimeout(() => validateAccountWidget(accountId), 100);
}
}
// Validate widget form fields
function validateAccountWidget(accountId) {
const widget = document.getElementById('create-account-widget-' + accountId);
if (!widget || !widget.classList.contains('show')) {
return;
}
const nameInput = widget.querySelector('input[name*="[name]"]');
const typeSelect = widget.querySelector('select[name*="[type]"]');
if (nameInput && !nameInput.value.trim()) {
console.warn('Account name is empty for:', accountId);
}
if (typeSelect && !typeSelect.value) {
console.warn('Account type not selected for:', accountId);
}
}
// Handle import checkbox state changes - Visibility-based UX
function handleImportToggle(accountId, isImportEnabled) {
const ff3AccountContent = document.getElementById('firefly-account-content-' + accountId);
const notImportedText = document.getElementById('not-imported-text-' + accountId);
if (ff3AccountContent && notImportedText) {
if (isImportEnabled) {
// Show Firefly III account content
ff3AccountContent.style.display = 'block';
notImportedText.style.display = 'none';
} else {
// Hide Firefly III account content and show "Not Imported" text
ff3AccountContent.style.display = 'none';
notImportedText.style.display = 'block';
}
console.log('Import toggle for account:', accountId, 'enabled:', isImportEnabled);
} else {
console.error('Could not find firefly account content or not imported text elements for account:', accountId);
}
}
// Enhanced initialization with error handling
document.addEventListener('DOMContentLoaded', function() {
const select = document.getElementById('account-select-{{ $account['import_account']->id }}');
const importCheckbox = document.getElementById('do_import_{{ $account['import_account']->id }}');
const accountId = '{{ $account['import_account']->id }}';
if (select) {
console.log('Initializing account selection for:', accountId);
// Set up change event listener with enhanced handling
select.addEventListener('change', function(event) {
try {
handleAccountSelection(accountId, event.target.value);
} catch (error) {
console.error('Error handling account selection change:', error);
}
});
// Initialize current state
try {
if (importCheckbox && !importCheckbox.checked) {
handleImportToggle(accountId, false);
} else {
handleAccountSelection(accountId, select.value);
}
} catch (error) {
console.error('Error during initialization for account:', accountId, error);
}
} else {
console.error('Account select element not found for:', accountId);
}
// Set up import checkbox listener
if (importCheckbox) {
importCheckbox.addEventListener('change', function(event) {
try {
handleImportToggle(accountId, event.target.checked);
} catch (error) {
console.error('Error handling import checkbox change:', error);
}
});
console.log('Import checkbox listener attached for:', accountId);
} else {
console.error('Import checkbox not found for:', accountId);
}
});
</script>
@endif

134
resources/views/v2/components/importer-account-title.blade.php

@ -1,31 +1,105 @@
<input
id="do_import_{{ $account['import_account']->id }}"
type="checkbox"
name="do_import[{{ $account['import_account']->id }}]"
value="1"
aria-describedby="accountsHelp"
@if('disabled' === $account['import_account']->status) disabled="disabled" @endif
@if(0 !== ($configuration->getAccounts()[$account['import_account']->id] ?? '')) checked="checked" @endif
/>
<label
class="form-check-label"
for="do_import_{{ $account['import_account']->id }}"
@if('' !== $account['import_account']->iban) title="IBAN: {{ $account['import_account']->iban }}" @endif
>
@if('' !== $account['import_account']->name)
Account "{{ $account['import_account']->name }}"
@else
Account with no name
@endif
</label>
<br>
<small>
@foreach($account['import_account']->extra as $key => $item)
@if('' !== $item)
{{ $key }}: {{ $item }}<br>
<div class="d-flex align-items-start p-3 bg-dark text-light rounded mb-2 border border-secondary">
<div class="form-check me-3">
<input
id="do_import_{{ $account['import_account']->id }}"
type="checkbox"
name="do_import[{{ $account['import_account']->id }}]"
value="1"
class="form-check-input"
aria-describedby="accountsHelp"
@if('disabled' === $account['import_account']->status) disabled="disabled" @endif
@php
$accountId = $account['import_account']->id;
$configuredValue = $configuration->getAccounts()[$accountId] ?? null;
$allAccounts = $configuration->getAccounts();
$mappedTo = $account['mapped_to'] ?? null;
// Check if account should be checked:
// 1. If configured explicitly (any non-null value including 0 for "create new")
// 2. If no configuration exists yet - use sensible defaults
$shouldCheck = false;
if ($configuredValue !== null && $configuredValue !== '') {
// Explicitly configured (including 0 for "create new")
$shouldCheck = true;
} elseif (empty($allAccounts)) {
// No configuration yet - use sensible defaults
// Check if there's an automatic mapping
if ($mappedTo !== null) {
$shouldCheck = true; // Auto-mapped accounts should be checked
} else {
$shouldCheck = true; // Default to checked for user convenience
}
}
@endphp
@if($shouldCheck) checked="checked" @endif
/>
</div>
<div class="flex-grow-1">
<label
class="form-check-label d-block mb-2"
for="do_import_{{ $account['import_account']->id }}"
@if('' !== $account['import_account']->iban) title="IBAN: {{ $account['import_account']->iban }}" @endif
>
<div class="d-flex align-items-center mb-1">
<span class="fw-bold fs-6">{{ $account['import_account']->name ?? 'Unnamed SimpleFIN Account' }}</span>
</div>
@if(isset($account['import_account']->org) && is_array($account['import_account']->org) && !empty($account['import_account']->org['name']))
<div class="text-muted small">
<i class="fas fa-building me-1"></i>
{{ $account['import_account']->org['name'] }}
</div>
@endif
</label>
@if(isset($account['import_account']->balance))
<div class="mb-2">
<i class="fas fa-coins me-1"></i>
<span class="badge bg-secondary text-light px-3 py-1 fw-bold">
{{ number_format((float)$account['import_account']->balance, 2) }} {{ $account['import_account']->currency ?? '' }}
</span>
@if(isset($account['import_account']->balance_date) && $account['import_account']->balance_date)
<small class="text-muted ms-2">({{ date('M j, Y', (int)$account['import_account']->balance_date) }})</small>
@endif
</div>
@endif
@endforeach
</small>
@if('disabled' === $account['import_account']->status)
<small class="text-danger">(this account is disabled)</small>
@endif
@if(isset($account['import_account']->available_balance) && $account['import_account']->available_balance !== ($account['import_account']->balance ?? null))
<div class="mb-2">
<i class="fas fa-wallet me-1"></i>
<span class="badge bg-secondary text-light px-3 py-1 fw-bold">
{{ number_format((float)$account['import_account']->available_balance, 2) }} {{ $account['import_account']->currency ?? '' }}
</span>
</div>
@endif
<div class="d-flex align-items-center justify-content-between">
<small class="text-muted">
<i class="fas fa-id-card me-1"></i>
<code class="text-muted">{{ $account['import_account']->id ?? 'N/A' }}</code>
</small>
@if('disabled' === $account['import_account']->status)
<small class="text-warning">
<i class="fas fa-exclamation-triangle me-1"></i>
Disabled
</small>
@endif
</div>
{{-- Display 'extra' fields if any --}}
@php $extraData = (array)($account['import_account']->extra ?? []); @endphp
@if(count($extraData) > 0)
<div class="mt-2 pt-2 border-top border-secondary">
@foreach($extraData as $key => $item)
@if(!empty($item) && is_scalar($item))
<div class="d-flex justify-content-between align-items-center small text-muted mb-1">
<span>{{ ucfirst(str_replace(['_', '-'], ' ', $key)) }}:</span>
<span>{{ $item }}</span>
</div>
@endif
@endforeach
</div>
@endif
</div>
</div>

25
resources/views/v2/components/importer-account.blade.php

@ -4,18 +4,27 @@
</td>
<td style="width:10%">
@if('disabled' !== $account['import_account']->status)
@if(count($account['firefly_iii_accounts']) > 0)
@if( (isset($account['firefly_iii_accounts']['assets']) && count($account['firefly_iii_accounts']['assets']) > 0) || (isset($account['firefly_iii_accounts']['liabilities']) && count($account['firefly_iii_accounts']['liabilities']) > 0) )
&rarr;
@endif
@endif
</td>
<td style="width:45%">
<!-- TODO this is one of those things to merge into one generic type -->
@if(0 === count($account['firefly_iii_accounts']))
<span class="text-danger">There are no Firefly III accounts to import into</span>
@endif
@if(0 !== count($account['firefly_iii_accounts']))
<x-firefly-iii-account-generic :account="$account" :configuration="$configuration"/>
@endif
<!-- Firefly III Account Content - Visibility Controlled -->
<div id="firefly-account-content-{{ $account['import_account']->id }}">
<!-- TODO this is one of those things to merge into one generic type -->
@if( $flow !== 'simplefin' && (!isset($account['firefly_iii_accounts']['assets']) || count($account['firefly_iii_accounts']['assets']) === 0) && (!isset($account['firefly_iii_accounts']['liabilities']) || count($account['firefly_iii_accounts']['liabilities']) === 0) )
<span class="text-danger">There are no Firefly III accounts to import into</span>
@endif
@if( $flow === 'simplefin' || (isset($account['firefly_iii_accounts']['assets']) && count($account['firefly_iii_accounts']['assets']) > 0) || (isset($account['firefly_iii_accounts']['liabilities']) && count($account['firefly_iii_accounts']['liabilities']) > 0) )
<x-firefly-iii-account-generic :account="$account" :configuration="$configuration"/>
<x-create-account-widget :account="$account" :configuration="$configuration" :currencies="$currencies ?? []"/>
@endif
</div>
<!-- Not Imported Text - Hidden by Default -->
<div id="not-imported-text-{{ $account['import_account']->id }}" style="display: none;" class="text-muted py-2">
<small><i class="fas fa-info-circle fa-sm me-1"></i>Not Imported</small>
</div>
</td>
</tr>

103
resources/views/v2/import/003-upload/index.blade.php

@ -29,6 +29,10 @@
If your configuration already contains an encrypted SimpleFIN access URL, you do not need to fill in the "SimpleFIN token" field. If you are unsure,
using the SimpleFIN token field will overrule whatever (if any) access URL is in your configuration file.
</p>
<p>
<strong>Demo Mode:</strong> You can use demo mode to test the import process with sample data before connecting your real financial accounts.
Simply check the "Use demo mode" option below.
</p>
@endif
</div>
</div>
@ -45,9 +49,33 @@
enctype="multipart/form-data">
<input type="hidden" name="_token" value="{{ csrf_token() }}"/>
<!-- SimpleFIN token -->
<!-- SimpleFIN fields -->
@if('simplefin' === $flow)
<!-- Demo Mode Toggle -->
<div class="form-group row mb-3">
<label for="use_demo" class="col-sm-4 col-form-label">Demo Mode</label>
<div class="col-sm-8">
<div class="form-check">
<input class="form-check-input" type="checkbox" id="use_demo" name="use_demo" value="1">
<label class="form-check-label" for="use_demo">
Use demo mode (test with sample data)
</label>
</div>
<small class="form-text text-muted">
Enable this to test the import process with SimpleFIN demo data.
</small>
</div>
</div>
<!-- Connection Error Display -->
@if($errors->has('connection'))
<div class="alert alert-danger" role="alert">
<strong>Connection Error:</strong> {{ $errors->first('connection') }}
</div>
@endif
<!-- SimpleFIN token -->
<div class="form-group row mb-3" id="token-group">
<label for="simplefin_token" class="col-sm-4 col-form-label">SimpleFIN token</label>
<div class="col-sm-8">
<input type="text"
@ -62,6 +90,54 @@
@endif
</div>
</div>
<!-- SimpleFIN CORS Origin URL (Additional Options) -->
<div class="form-group row mb-3" id="bridge-url-group">
<label for="simplefin_bridge_url" class="col-sm-4 col-form-label">
CORS Origin URL
<small class="text-muted">(optional)</small>
</label>
<div class="col-sm-8">
<input type="url"
class="form-control
@if($errors->has('simplefin_bridge_url')) is-invalid @endif"
id="simplefin_bridge_url" name="simplefin_bridge_url"
placeholder="https://your-app.example.com"/>
<small class="form-text text-muted">
Enter the URL where you access this Firefly III Data Importer (e.g., https://your-domain.com). Leave blank if unsure.
</small>
@if($errors->has('simplefin_bridge_url'))
<div class="invalid-feedback">
{{ $errors->first('simplefin_bridge_url') }}
</div>
@endif
</div>
</div>
<script>
document.addEventListener('DOMContentLoaded', function() {
const demoCheckbox = document.getElementById('use_demo');
const bridgeUrlGroup = document.getElementById('bridge-url-group');
const tokenGroup = document.getElementById('token-group');
function toggleSimpleFINFields() {
if (demoCheckbox.checked) {
bridgeUrlGroup.style.display = 'none';
tokenGroup.style.display = 'none';
} else {
bridgeUrlGroup.style.display = 'flex';
tokenGroup.style.display = 'flex';
}
}
// Initial state
toggleSimpleFINFields();
// Listen for changes
demoCheckbox.addEventListener('change', toggleSimpleFINFields);
});
</script>
@endif
<!-- importable FILE -->
@ -134,4 +210,29 @@
</div>
</div>
</div>
@if('simplefin' === $flow)
<script>
document.addEventListener('DOMContentLoaded', function() {
const demoCheckbox = document.getElementById('use_demo');
const tokenGroup = document.getElementById('token-group');
const tokenInput = document.getElementById('simplefin_token');
function toggleDemoMode() {
if (demoCheckbox.checked) {
tokenGroup.style.display = 'none';
tokenInput.required = false;
} else {
tokenGroup.style.display = 'flex';
tokenInput.required = true;
}
}
demoCheckbox.addEventListener('change', toggleDemoMode);
// Initial state
toggleDemoMode();
});
</script>
@endif
@endsection

353
resources/views/v2/import/004-configure/index.blade.php

@ -35,7 +35,7 @@
<!-- user has no accounts -->
@if(0 === count($fireflyIIIaccounts['assets']) && 0 === count($fireflyIIIaccounts['liabilities']) )
@if(0 === count($fireflyIIIaccounts['assets']) && 0 === count($fireflyIIIaccounts['liabilities']) && $flow !== 'simplefin')
<div class="row mt-3">
<div class="col-lg-10 offset-lg-1">
<div class="card">
@ -76,7 +76,7 @@
</div>
@endif
<!-- user has accounts! -->
@if(count($fireflyIIIaccounts['assets']) > 0 || count($fireflyIIIaccounts['liabilities']) > 0)
@if(count($fireflyIIIaccounts['assets']) > 0 || count($fireflyIIIaccounts['liabilities']) > 0 || $flow === 'simplefin')
<div class="row mt-3">
<div class="col-lg-10 offset-lg-1">
<div class="card">
@ -120,6 +120,13 @@
out the documentation for this page.</a>
</p>
@endif
@if('simplefin' === $flow)
<p>
Configure how your SimpleFIN accounts will be mapped to Firefly III accounts.
You can map existing accounts or create new ones during import.
Accounts marked for import will have their transactions synchronized based on your date range settings.
</p>
@endif
</div>
</div>
</div>
@ -143,6 +150,226 @@
<input type="hidden" name="ignore_duplicate_transactions" value="1"/>
@endif
<!-- SimpleFIN account configuration -->
@if('simplefin' === $flow)
<!-- Hidden fields for SimpleFIN validation -->
<input type="hidden" name="unique_column_type" value="id">
<input type="hidden" name="duplicate_detection_method" value="none">
<div class="row mt-3">
<div class="col-lg-10 offset-lg-1">
<div class="card">
<div class="card-header">
SimpleFIN Account Configuration
</div>
<div class="card-body">
<p>Map your SimpleFIN accounts to Firefly III accounts. You can link to existing accounts or create new ones during import.</p>
@if(count($importerAccounts) > 0)
<div class="table-responsive">
<table class="table table-sm table-borderless">
<thead>
<tr>
<th style="width:45%">SimpleFIN Account</th>
<th style="width:10%"></th>
<th style="width:45%">Firefly III Account</th>
</tr>
</thead>
<tbody>
@foreach($importerAccounts as $information)
<x-importer-account :account="$information" :configuration="$configuration" :currencies="$currencies" :flow="$flow"/>
@endforeach
</tbody>
</table>
</div>
@else
<div class="alert alert-warning">
<strong>No SimpleFIN accounts found.</strong> Please ensure your SimpleFIN token is valid and try again.
</div>
@endif
</div>
</div>
</div>
</div>
<!-- SimpleFIN Import Options - Consolidated -->
<div class="row mt-3">
<div class="col-lg-10 offset-lg-1">
<div class="card">
<div class="card-header">
SimpleFIN Import Options
</div>
<div class="card-body">
<!-- Date Range Configuration -->
<div class="form-group row mb-3">
<label for="date_range" class="col-sm-3 col-form-label">Date range:</label>
<div class="col-sm-9">
<select name="date_range" id="date_range" class="form-control" onchange="toggleDateRangeInputs()">
<option value="all" @if($configuration->getDateRange() === 'all') selected @endif>All time</option>
<option value="dynamic" @if($configuration->getDateRange() === 'dynamic') selected @endif>Dynamic range</option>
<option value="specific" @if($configuration->getDateRange() === 'specific') selected @endif>Specific dates</option>
</select>
</div>
</div>
<div id="dynamic_range_inputs" style="display: {{ $configuration->getDateRange() === 'dynamic' ? 'block' : 'none' }};">
<div class="form-group row mb-3">
<label for="date_range_number" class="col-sm-3 col-form-label">Range:</label>
<div class="col-sm-5">
<input type="number" name="date_range_number" id="date_range_number" class="form-control" value="{{ $configuration->getDateRangeNumber() ?? 30 }}" min="1">
</div>
<div class="col-sm-4">
<select name="date_range_unit" id="date_range_unit" class="form-control">
<option value="d" @if($configuration->getDateRangeUnit() === 'd') selected @endif>Days</option>
<option value="w" @if($configuration->getDateRangeUnit() === 'w') selected @endif>Weeks</option>
<option value="m" @if($configuration->getDateRangeUnit() === 'm') selected @endif>Months</option>
<option value="y" @if($configuration->getDateRangeUnit() === 'y') selected @endif>Years</option>
</select>
</div>
</div>
</div>
<div id="specific_dates_inputs" style="display: {{ $configuration->getDateRange() === 'specific' ? 'block' : 'none' }};">
<div class="form-group row mb-3">
<label for="date_not_before" class="col-sm-3 col-form-label">Start date:</label>
<div class="col-sm-9">
<input type="date" name="date_not_before" id="date_not_before" class="form-control" value="{{ $configuration->getDateNotBefore() ?? '' }}">
</div>
</div>
<div class="form-group row mb-3">
<label for="date_not_after" class="col-sm-3 col-form-label">End date:</label>
<div class="col-sm-9">
<input type="date" name="date_not_after" id="date_not_after" class="form-control" value="{{ $configuration->getDateNotAfter() ?? '' }}">
</div>
</div>
</div>
<!-- Pending Transactions Configuration -->
<div class="form-group row mb-3">
<div class="col-sm-3">Pending transactions</div>
<div class="col-sm-9">
<div class="form-check">
<input class="form-check-input"
@if($configuration->getPendingTransactions()) checked @endif
type="checkbox" id="pending_transactions" name="pending_transactions" value="1"
aria-describedby="pendingTransactionsHelp">
<label class="form-check-label" for="pending_transactions">
Include pending transactions
</label>
<small id="pendingTransactionsHelp" class="form-text text-muted">
<br>Select to include pending (unposted) transactions in addition to posted transactions.
</small>
</div>
</div>
</div>
<!-- De-duplication Configuration -->
<div class="form-group row mb-3">
<div class="col-sm-3">De-duplication</div>
<div class="col-sm-9">
<div class="form-check">
<input class="form-check-input"
@if($configuration->getDuplicateDetectionMethod() !== 'none') checked @endif
type="checkbox" id="enable_deduplication" name="enable_deduplication" value="1"
aria-describedby="deduplicationHelp">
<label class="form-check-label" for="enable_deduplication">
Enable content-based de-duplication
</label>
<small id="deduplicationHelp" class="form-text text-muted">
<br>Prevent importing duplicate transactions based on transaction content.
</small>
</div>
</div>
</div>
<!-- Rules Configuration -->
<div class="form-group row mb-3">
<div class="col-sm-3">Rules</div>
<div class="col-sm-9">
<div class="form-check">
<input class="form-check-input"
@if($configuration->isRules()) checked @endif
type="checkbox" id="rules" name="rules" value="1"
aria-describedby="rulesHelp">
<label class="form-check-label" for="rules">
Apply Firefly III rules
</label>
<small id="rulesHelp" class="form-text text-muted">
<br>Apply your Firefly III rules to imported transactions.
</small>
</div>
</div>
</div>
<!-- Map Data Configuration -->
<div class="form-group row mb-3">
<div class="col-sm-3">Map data</div>
<div class="col-sm-9">
<div class="form-check">
<input class="form-check-input"
@if($configuration->isMapAllData()) checked @endif
type="checkbox" id="map_all_data" name="map_all_data" value="1"
aria-describedby="mapAllDataHelp">
<label class="form-check-label" for="map_all_data">
Map transaction data
</label>
<small id="mapAllDataHelp" class="form-text text-muted">
<br>Map expense and revenue account names for imported transactions.
</small>
</div>
</div>
</div>
<!-- Import Tag Configuration -->
<div class="form-group row mb-3">
<div class="col-sm-3">Import tag</div>
<div class="col-sm-9">
<div class="form-check">
<input class="form-check-input"
@if($configuration->isAddImportTag()) checked @endif
type="checkbox" id="add_import_tag" name="add_import_tag" value="1"
aria-describedby="add_import_tagHelp">
<label class="form-check-label" for="add_import_tag">
Add import tag
</label>
<small id="add_import_tagHelp" class="form-text text-muted">
<br>Add a tag to each imported transaction to group your import.
</small>
</div>
</div>
</div>
<!-- Custom Tag Configuration -->
<div class="form-group row mb-3">
<label for="custom_tag" class="col-sm-3 col-form-label">Custom tag</label>
<div class="col-sm-9">
<input type="text" name="custom_tag" id="custom_tag" class="form-control"
value="{{ $configuration->getCustomTag() ?? '' }}"
aria-describedby="customTagHelp">
<small id="customTagHelp" class="form-text text-muted">
Optional custom tag to add to all imported transactions.
</small>
</div>
</div>
</div>
</div>
</div>
</div>
<script>
function toggleDateRangeInputs() {
const dateRangeType = document.getElementById('date_range').value;
const dynamicInputs = document.getElementById('dynamic_range_inputs');
const specificInputs = document.getElementById('specific_dates_inputs');
dynamicInputs.style.display = (dateRangeType === 'dynamic') ? 'block' : 'none';
specificInputs.style.display = (dateRangeType === 'specific') ? 'block' : 'none';
}
// Initialize on page load
document.addEventListener('DOMContentLoaded', toggleDateRangeInputs);
</script>
@endif
<!-- End of SimpleFIN Import Options -->
<!-- Account selection for Gocardless and Spectre -->
<!-- also date range settings -->
@if('nordigen' === $flow || 'spectre' === $flow)
@ -213,7 +440,9 @@
@endif
<!-- end of update variables -->
<x-importer-account :account="$information"
:configuration="$configuration"/>
:configuration="$configuration"
:currencies="$currencies"
:flow="$flow"/>
@endforeach
</tbody>
<caption>Select and match the
@ -573,7 +802,10 @@
@endif
<!-- end of CSV options -->
<!-- duplicate detection options -->
<!-- generic import options -->
@if('simplefin' !== $flow)
<div class="row mt-3">
<div class="col-lg-10 offset-lg-1">
<div class="card">
@ -681,121 +913,8 @@
</div>
<!-- end of generic import options -->
@endif
<!-- duplicate detection options -->
<div class="row mt-3">
<div class="col-lg-10 offset-lg-1">
<div class="card">
<div class="card-header">
Duplicate transaction detection
</div>
<div class="card-body">
<div class="col-sm-9 offset-sm-3">
<p class="text-muted">
Firefly III can automatically detect duplicate transactions. This is pretty
foolproof. In some special cases however,
you want more control over this process. Read more about the options below in <a
href="https://docs.firefly-iii.org/how-to/data-importer/import/csv/"
target="_blank">the documentation</a>.
</p>
</div>
@if('file' === $flow)
<div class="form-group row mb-3">
<label for="X" class="col-sm-3 col-form-label">General detection options</label>
<div class="col-sm-9">
<div class="form-check">
<input class="form-check-input"
@if($configuration->isIgnoreDuplicateLines()) checked @endif
type="checkbox" value="1" id="ignore_duplicate_lines"
name="ignore_duplicate_lines" aria-describedby="duplicateHelp">
<label class="form-check-label" for="ignore_duplicate_lines">
Do not import duplicate lines or entries in the importable file.
</label>
<br>
<small class="form-text text-muted" id="duplicateHelp">
Whatever method you choose ahead, it's smart to make the importer ignore
any
duplicated lines or entries in your importable file.
</small>
</div>
</div>
</div>
@endif
<div class="form-group row mb-3">
<label for="duplicate_detection_method" class="col-sm-3 col-form-label">Detection
method</label>
<div class="col-sm-9">
<select id="duplicate_detection_method" name="duplicate_detection_method" x-model="detectionMethod"
class="form-control" aria-describedby="duplicate_detection_method_help">
<option label="No duplicate detection"
@if('none' === $configuration->getDuplicateDetectionMethod()) selected @endif
value="none">No duplicate detection
</option>
<option label="Content-based"
@if('classic' === $configuration->getDuplicateDetectionMethod()) selected @endif
value="classic">Content-based detection
</option>
<option label="Identifier-based"
@if('cell' === $configuration->getDuplicateDetectionMethod()) selected @endif
value="cell">Identifier-based detection
</option>
</select>
<small id="duplicate_detection_method_help" class="form-text text-muted">
For more details on these detection method see <a
href="https://docs.firefly-iii.org/references/data-importer/duplicate-detection/"
target="_blank">the documentation</a>. If you're not sure, select
"content-based" detection.
</small>
</div>
</div>
@if('file' === $flow)
<div class="form-group row mb-3" id="unique_column_index_holder" x-show="'cell' === detectionMethod">
<label for="unique_column_index" class="col-sm-3 col-form-label">Unique column
index</label>
<div class="col-sm-9">
<input type="number" step="1" name="unique_column_index" class="form-control"
id="unique_column_index" placeholder="Column index"
value="{{ $configuration->getUniqueColumnIndex() }}"
aria-describedby="unique_column_index_help">
<small id="unique_column_index_help" class="form-text text-muted">
This field is only relevant for the "identifier-based" detection option.
Indicate which column / field contains the unique identifier. Start counting from
zero!
</small>
</div>
</div>
@endif
<div class="form-group row" id="unique_column_type_holder" x-show="'cell' === detectionMethod">
<label for="unique_column_type" class="col-sm-3 col-form-label">Unique column
type</label>
<div class="col-sm-9">
<select id="unique_column_type" name="unique_column_type" class="form-control"
aria-describedby="unique_column_type_help">
@foreach($uniqueColumns as $columnType => $columnName)
<option label="{{ $columnName }}"
@if($configuration->getUniqueColumnType() === $columnType) selected @endif
value="{{ $columnType }}">{{ $columnName }}</option>
@endforeach
</select>
<small id="unique_column_type_help" class="form-text text-muted">
This field is only relevant for the "identifier-based" detection option.
Select
the type of value you expect in
the unique identifier. What must Firefly III search for?
</small>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- end of duplicate detection options -->
<!-- other options -->
<div class="row mt-3">

90
resources/views/v2/import/007-convert/index.blade.php

@ -77,14 +77,94 @@
</div>
</div>
</div>
</div>
<!-- New Account Creation Section (SimpleFIN only) -->
@if('simplefin' === $flow && !empty($newAccountsToCreate))
<div class="row mt-3">
<div class="col-lg-10 offset-lg-1">
<div class="card border-warning">
<div class="card-header bg-warning text-dark">
<h5 class="mb-0"><span class="fas fa-plus-circle"></span> New Accounts to Create</h5>
</div>
<div class="card-body">
<p class="text-muted">
The following accounts will be created in Firefly III before importing transactions.
You can customize their settings below or proceed with the smart defaults.
</p>
@foreach($newAccountsToCreate as $simplefinAccountId => $accountData)
<div class="card mb-3">
<div class="card-body">
<div class="row">
<div class="col-md-6">
<h6 class="card-title">{{ $accountData['name'] ?? 'New Account' }}</h6>
<small class="text-muted">SimpleFIN Account: {{ $simplefinAccountId }}</small>
</div>
<div class="col-md-6">
<form class="new-account-form" data-account-id="{{ $simplefinAccountId }}">
<div class="form-group mb-2">
<label class="form-label">Account Name:</label>
<input type="text"
class="form-control form-control-sm"
name="account_name"
value="{{ $accountData['name'] ?? '' }}"
required>
</div>
<div class="form-group mb-2">
<label class="form-label">Account Type:</label>
<select class="form-control form-control-sm" name="account_type" required>
<option value="asset" selected>Asset Account</option>
<option value="liability">Liability Account</option>
</select>
<small class="form-text text-muted">Smart default: Asset (recommended for most accounts)</small>
</div>
<div class="form-group mb-2">
<label class="form-label">Currency:</label>
<input type="text"
class="form-control form-control-sm"
name="account_currency"
value="USD"
maxlength="3"
required>
<small class="form-text text-muted">3-letter currency code</small>
</div>
<div class="form-group mb-2">
<label class="form-label">Opening Balance (optional):</label>
<input type="number"
step="0.01"
class="form-control form-control-sm"
name="opening_balance"
placeholder="0.00">
</div>
</form>
</div>
</div>
</div>
</div>
@endforeach
<div class="alert alert-info">
<small>
<span class="fas fa-info-circle"></span>
These accounts will be created automatically when you start the conversion process.
You can modify the details above or proceed with the defaults.
</small>
</div>
</div>
</div>
</div>
</div>
@endif
<div class="row mt-3">
<div class="col-lg-10 offset-lg-1">
<div class="card">
<div class="card-body">
<div class="btn-group btn-group-sm">
<div class="row mt-3">
<div class="col-lg-10 offset-lg-1">
<div class="card">
<div class="card-body">
<div class="btn-group btn-group-sm">
<a href="{{ $jobBackUrl }}" class="btn btn-secondary"><span class="fas fa-arrow-left"></span>
Go back to the previous step</a>
<a class="btn btn-danger text-white btn-sm" href="{{ route('flush') }}" data-bs-toggle="tooltip"

28
resources/views/v2/import/008-submit/index.blade.php

@ -33,8 +33,23 @@
</div>
<div x-show="showTooManyChecks()" class="card-body">
<p>
<em class="fa-solid fa-face-dizzy"></em>
The data importer has been polling for more than <span x-text="checkCount"></span> seconds. It has stopped, to prevent eternal loops.</p>
<em class="fa-solid fa-clock"></em>
<strong>Job Still Running</strong> - The import submission is taking longer than expected (<span x-text="checkCount"></span> seconds) but is likely still processing in the background.
</p>
<p>
Large imports with many transactions can take 20+ minutes to complete. The automatic status checking has been paused to prevent system overload.
</p>
<div class="alert alert-info">
<strong>What you can do:</strong>
<ul class="mb-2">
<li>Click "Refresh Status" below to check if the job has completed</li>
<li>Check your Firefly III installation directly to see if transactions are appearing</li>
<li>Wait a few more minutes and try refreshing this page</li>
</ul>
<button x-show="manualRefreshAvailable" @click="refreshStatus()" class="btn btn-primary btn-sm">
<span class="fas fa-sync-alt"></span> Refresh Status
</button>
</div>
</div>
<div x-show="showPostError()" class="card-body">
<p class="text-danger">
@ -51,8 +66,13 @@
</p>
<div class="progress">
<div aria-valuemax="100" aria-valuemin="0"
aria-valuenow="100" class="progress-bar progress-bar-striped progress-bar-animated"
role="progressbar" style="width: 100%"></div>
:aria-valuenow="getProgressPercentage()"
:class="hasProgressData() ? 'progress-bar' : 'progress-bar progress-bar-striped progress-bar-animated'"
role="progressbar"
:style="'width: ' + getProgressWidth()"></div>
</div>
<div x-show="hasProgressData()" class="text-center mt-2">
<small class="text-muted" x-text="getProgressDisplay()"></small>
</div>
<x-conversion-messages />
</div>

382
resources/views/v2/layout/v2.blade.php

@ -97,5 +97,387 @@
</p></noscript>
<!-- End Matomo Code -->
@endif
<script>
// Global SimpleFIN account management functions
// Moved from component to prevent timing/conflict issues
// Initialize global state
window.accountEditStates = window.accountEditStates || {};
// Account name inline editing functions
window.toggleAccountNameEdit = function(accountId, isEditing, retryCount = 0) {
try {
// Check if widget is visible first
const widget = document.getElementById('create-account-widget-' + accountId);
if (!widget || !widget.classList.contains('show')) {
console.warn('Widget not visible for account:', accountId, 'Attempting to show widget first');
// Try to show widget if it exists but is hidden
if (widget && typeof bootstrap !== 'undefined' && bootstrap.Collapse) {
const collapse = new bootstrap.Collapse(widget, { show: true });
// Retry after widget is shown
setTimeout(() => window.toggleAccountNameEdit(accountId, isEditing, retryCount + 1), 200);
return true;
}
}
const display = document.getElementById('account-name-display-' + accountId);
const input = document.getElementById('account-name-edit-' + accountId);
const editBtn = document.getElementById('edit-name-btn-' + accountId);
const commitBtn = document.getElementById('commit-name-btn-' + accountId);
const cancelBtn = document.getElementById('cancel-name-btn-' + accountId);
if (!display || !input || !editBtn || !commitBtn || !cancelBtn) {
console.error('Name edit elements not found for account:', accountId, {
display: !!display,
input: !!input,
editBtn: !!editBtn,
commitBtn: !!commitBtn,
cancelBtn: !!cancelBtn,
widgetExists: !!widget,
widgetVisible: widget?.classList.contains('show'),
retryCount: retryCount
});
// Retry mechanism for timing issues
if (retryCount < 3) {
console.log('Retrying toggleAccountNameEdit for account:', accountId, 'attempt:', retryCount + 1);
setTimeout(() => window.toggleAccountNameEdit(accountId, isEditing, retryCount + 1), 100);
return true;
}
// Additional debugging: log all elements with this account ID
const allElements = document.querySelectorAll('[id*="' + accountId + '"]');
console.error('All elements with account ID ' + accountId + ':', Array.from(allElements).map(el => el.id));
return false;
}
window.accountEditStates[accountId] = window.accountEditStates[accountId] || {};
window.accountEditStates[accountId].nameEditing = isEditing;
console.log('toggleAccountNameEdit called:', {
accountId: accountId,
isEditing: isEditing,
timestamp: new Date().toISOString()
});
if (isEditing) {
display.classList.add('d-none');
input.classList.remove('d-none');
editBtn.classList.add('d-none');
commitBtn.classList.remove('d-none');
cancelBtn.classList.remove('d-none');
input.focus();
input.select();
// Add keyboard event handler for Enter/Escape
input.addEventListener('keydown', function(event) {
console.log('Key pressed in account name edit:', event.key, 'for account:', accountId);
if (event.key === 'Enter') {
event.preventDefault();
event.stopPropagation();
console.log('Enter key - committing edit for account:', accountId);
window.commitAccountNameEdit(accountId);
} else if (event.key === 'Escape') {
event.preventDefault();
event.stopPropagation();
console.log('Escape key - cancelling edit for account:', accountId);
window.toggleAccountNameEdit(accountId, false);
}
}, { once: true });
} else {
display.classList.remove('d-none');
input.classList.add('d-none');
editBtn.classList.remove('d-none');
commitBtn.classList.add('d-none');
cancelBtn.classList.add('d-none');
// Reset input to original value on cancel
input.value = display.textContent.trim();
}
return true;
} catch (error) {
console.error('Error toggling name edit for account:', accountId, error);
return false;
}
}
window.commitAccountNameEdit = function(accountId) {
try {
const display = document.getElementById('account-name-display-' + accountId);
const input = document.getElementById('account-name-edit-' + accountId);
if (!display || !input) {
console.error('Name edit elements not found for account:', accountId);
return false;
}
const newName = input.value.trim();
if (!newName) {
// Use placeholder default if empty
const defaultName = 'New Account';
input.value = defaultName;
display.textContent = defaultName;
} else {
display.textContent = newName;
}
window.toggleAccountNameEdit(accountId, false);
// Trigger duplicate check after name commit
window.updateDuplicateStatus(accountId);
return true;
} catch (error) {
console.error('Error committing name edit for account:', accountId, error);
return false;
}
}
// Real-time duplicate account validation
window.updateDuplicateStatus = function(accountId) {
try {
const nameInput = document.getElementById('account-name-edit-' + accountId);
const nameDisplay = document.getElementById('account-name-display-' + accountId);
const typeSelect = document.getElementById('new-account-type-' + accountId);
const statusElement = document.getElementById('widget-status-' + accountId);
if (!statusElement) {
console.log('DUPLICATE_CHECK: Status element not found for account:', accountId);
return false;
}
// Get current name value (from input if editing, display if not)
let accountName = '';
if (nameInput && !nameInput.classList.contains('d-none')) {
accountName = nameInput.value.trim();
} else if (nameDisplay) {
accountName = nameDisplay.textContent.trim();
}
// Get current type value
const accountType = typeSelect ? typeSelect.value : '';
console.log('DUPLICATE_CHECK: Checking account', {
accountId: accountId,
name: accountName,
type: accountType
});
// Clear validation if name or type is empty
if (!accountName || !accountType) {
statusElement.innerHTML = '<i class="fas fa-info-circle me-1"></i>Complete name and type';
statusElement.className = 'text-muted';
return true;
}
// Show checking status
statusElement.innerHTML = '<i class="fas fa-spinner fa-spin me-1"></i>Checking for duplicates...';
statusElement.className = 'text-info';
// Make AJAX request to duplicate check endpoint
fetch('/import/check-duplicate', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-CSRF-TOKEN': document.querySelector('meta[name="csrf-token"]').getAttribute('content')
},
body: JSON.stringify({
name: accountName,
type: accountType
})
})
.then(response => {
if (!response.ok) {
throw new Error('Network response was not ok');
}
return response.json();
})
.then(data => {
console.log('DUPLICATE_CHECK: Response received', data);
if (data.isDuplicate) {
statusElement.innerHTML = '<i class="fas fa-exclamation-triangle me-1"></i>' + data.message;
statusElement.className = 'text-warning';
} else {
statusElement.innerHTML = '<i class="fas fa-check-circle me-1"></i>Ready for import';
statusElement.className = 'text-success';
}
})
.catch(error => {
console.error('DUPLICATE_CHECK: Error during duplicate check', error);
// Graceful degradation - show ready status on error
statusElement.innerHTML = '<i class="fas fa-check-circle me-1"></i>Ready for import';
statusElement.className = 'text-muted';
});
return true;
} catch (error) {
console.error('DUPLICATE_CHECK: Exception in updateDuplicateStatus', error);
return false;
}
}
// Enhanced global function for dropdown coordination
window.toggleAccountNameEditing = function(accountId, isCreateNew) {
try {
const widget = document.getElementById('create-account-widget-' + accountId);
if (!widget) {
console.error('toggleAccountNameEditing: Widget not found for account:', accountId);
return false;
}
console.log('toggleAccountNameEditing called for account:', accountId, 'createNew:', isCreateNew);
if (isCreateNew) {
widget.classList.add('show');
// Trigger validation check when widget is shown for "Create New Account"
if (window.updateDuplicateStatus) {
setTimeout(() => window.updateDuplicateStatus(accountId), 100);
}
} else {
widget.classList.remove('show');
}
return true;
} catch (error) {
console.error('Error in toggleAccountNameEditing for account:', accountId, error);
return false;
}
}
// Balance and currency inline editing functions
window.toggleBalanceCurrencyEdit = function(accountId, isEditing) {
try {
const balanceDisplay = document.getElementById('balance-display-' + accountId);
const currencyDisplay = document.getElementById('currency-display-' + accountId);
const balanceInput = document.getElementById('balance-edit-' + accountId);
const currencySelect = document.getElementById('currency-edit-' + accountId);
const editBalanceBtn = document.getElementById('edit-balance-btn-' + accountId);
const commitBalanceBtn = document.getElementById('commit-balance-btn-' + accountId);
const cancelBalanceBtn = document.getElementById('cancel-balance-btn-' + accountId);
if (!balanceDisplay || !currencyDisplay || !balanceInput || !currencySelect || !editBalanceBtn || !commitBalanceBtn || !cancelBalanceBtn) {
console.error('Balance/currency edit elements not found for account:', accountId);
return false;
}
window.accountEditStates[accountId] = window.accountEditStates[accountId] || {};
window.accountEditStates[accountId].balanceEditing = isEditing;
if (isEditing) {
balanceDisplay.classList.add('d-none');
currencyDisplay.classList.add('d-none');
balanceInput.classList.remove('d-none');
currencySelect.classList.remove('d-none');
editBalanceBtn.classList.add('d-none');
commitBalanceBtn.classList.remove('d-none');
cancelBalanceBtn.classList.remove('d-none');
balanceInput.focus();
balanceInput.select();
// Add keyboard event handlers for Enter/Escape
balanceInput.addEventListener('keydown', function(event) {
if (event.key === 'Enter') {
event.preventDefault();
event.stopPropagation();
window.commitBalanceCurrencyEdit(accountId);
} else if (event.key === 'Escape') {
event.preventDefault();
event.stopPropagation();
window.toggleBalanceCurrencyEdit(accountId, false);
}
}, { once: true });
// Also add handler to currency select
currencySelect.addEventListener('keydown', function(event) {
if (event.key === 'Enter') {
event.preventDefault();
event.stopPropagation();
window.commitBalanceCurrencyEdit(accountId);
} else if (event.key === 'Escape') {
event.preventDefault();
event.stopPropagation();
window.toggleBalanceCurrencyEdit(accountId, false);
}
}, { once: true });
} else {
balanceDisplay.classList.remove('d-none');
currencyDisplay.classList.remove('d-none');
balanceInput.classList.add('d-none');
currencySelect.classList.add('d-none');
editBalanceBtn.classList.remove('d-none');
commitBalanceBtn.classList.add('d-none');
cancelBalanceBtn.classList.add('d-none');
}
return true;
} catch (error) {
console.error('Error toggling balance/currency edit for account:', accountId, error);
return false;
}
}
window.commitBalanceCurrencyEdit = function(accountId) {
try {
const balanceDisplay = document.getElementById('balance-display-' + accountId);
const currencyDisplay = document.getElementById('currency-display-' + accountId);
const balanceInput = document.getElementById('balance-edit-' + accountId);
const currencySelect = document.getElementById('currency-edit-' + accountId);
if (!balanceDisplay || !currencyDisplay || !balanceInput || !currencySelect) {
console.error('Balance/currency edit elements not found for account:', accountId);
return false;
}
const newBalance = parseFloat(balanceInput.value);
const currencyCode = currencySelect.value;
if (!isNaN(newBalance)) {
balanceDisplay.textContent = newBalance.toFixed(2);
}
if (currencyCode && currencyCode !== '') {
currencyDisplay.textContent = currencyCode;
}
window.toggleBalanceCurrencyEdit(accountId, false);
return true;
} catch (error) {
console.error('Error committing balance/currency edit for account:', accountId, error);
return false;
}
}
// Show/hide account role section based on account type
window.updateAccountRoleVisibility = function(accountId) {
try {
const typeSelect = document.getElementById(`new-account-type-${accountId}`);
const roleSection = document.getElementById(`account-role-section-${accountId}`);
if (typeSelect && roleSection) {
if (typeSelect.value === 'asset') {
roleSection.style.display = 'block';
} else {
roleSection.style.display = 'none';
}
}
// Trigger duplicate check when account type changes
if (window.updateDuplicateStatus) {
window.updateDuplicateStatus(accountId);
}
return true;
} catch (error) {
console.error('Error in updateAccountRoleVisibility for account:', accountId, error);
return false;
}
}
console.log('Global SimpleFIN account management functions loaded');
</script>
</body>
</html>

3
routes/web.php

@ -59,6 +59,7 @@ Route::get('/import/configure', ['uses' => 'Import\ConfigurationController@index
Route::post('/import/configure', ['uses' => 'Import\ConfigurationController@postIndex', 'as' => '004-configure.post']);
Route::get('/import/configure/download', ['uses' => 'Import\DownloadController@download', 'as' => '004-configure.download']);
Route::get('/import/php_date', ['uses' => 'Import\ConfigurationController@phpDate', 'as' => '004-configure.php_date']);
Route::post('/import/check-duplicate', ['uses' => 'Import\DuplicateCheckController@checkDuplicate', 'as' => 'import.check-duplicate']);
// step 5: Set column roles (CSV or other file types)
// check : must be CSV and not config complete otherwise redirect to mapping.
@ -94,6 +95,8 @@ Route::get('/import/spectre-connections', ['uses' => 'Import\Spectre\ConnectionC
Route::post('/import/spectre-connections/submit', ['uses' => 'Import\Spectre\ConnectionController@post', 'as' => '011-connections.post']);
Route::get('/import/spectre-connections/callback', ['uses' => 'Import\Spectre\CallbackController@index', 'as' => '011-connections.callback']);
// routes to go back to other steps (also takes care of session vars)
Route::get('/back/start', 'NavController@toStart')->name('back.start');
Route::get('/back/upload', 'NavController@toUpload')->name('back.upload');

114
tests/Feature/SimpleFIN/DemoModeTest.php

@ -0,0 +1,114 @@
<?php
namespace Tests\Feature\SimpleFIN;
use Tests\TestCase;
use App\Support\Constants;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Config;
class DemoModeTest extends TestCase
{
use RefreshDatabase;
protected function setUp(): void
{
parent::setUp();
// Set up SimpleFIN demo configuration
Config::set('importer.simplefin.demo_token', 'demo_token_123');
Config::set('importer.simplefin.demo_url', 'https://demo:demo@beta-bridge.simplefin.org/simplefin');
}
public function test_demo_mode_checkbox_submission()
{
// Set the flow cookie to SimpleFIN
$response = $this->withCookie(Constants::FLOW_COOKIE, 'simplefin')
->post(route('003-upload.upload'), [
'use_demo' => '1'
]);
// Check that we don't redirect back to upload (which indicates validation failure)
$this->assertNotEquals(route('003-upload.index'), $response->getTargetUrl());
// If demo mode works correctly, we should redirect to configuration
$response->assertRedirect(route('004-configure.index'));
// Verify session data was set correctly
$this->assertEquals('demo_token_123', session(Constants::SIMPLEFIN_TOKEN));
$this->assertEquals('https://demo:demo@beta-bridge.simplefin.org/simplefin', session(Constants::SIMPLEFIN_BRIDGE_URL));
$this->assertTrue(session(Constants::SIMPLEFIN_IS_DEMO));
$this->assertTrue(session(Constants::HAS_UPLOAD));
}
public function test_demo_mode_checkbox_value_interpretation()
{
// Test various ways the checkbox might be submitted
$testCases = [
['use_demo' => '1'],
['use_demo' => 'on'],
['use_demo' => true],
['use_demo' => 1],
];
foreach ($testCases as $case) {
$response = $this->withCookie(Constants::FLOW_COOKIE, 'simplefin')
->post(route('003-upload.upload'), $case);
$this->assertNotEquals(route('003-upload.index'), $response->getTargetUrl(),
'Failed for case: ' . json_encode($case));
}
}
public function test_manual_mode_requires_token_and_url()
{
// Test that manual mode (no demo checkbox) requires token and URL
$response = $this->withCookie(Constants::FLOW_COOKIE, 'simplefin')
->post(route('003-upload.upload'), [
// No use_demo, no token, no URL
]);
// Should redirect back to upload with validation errors
$response->assertRedirect(route('003-upload.index'));
$response->assertSessionHasErrors(['simplefin_token', 'bridge_url']);
}
public function test_manual_mode_with_valid_credentials()
{
$response = $this->withCookie(Constants::FLOW_COOKIE, 'simplefin')
->post(route('003-upload.upload'), [
'simplefin_token' => 'valid_token_123',
'bridge_url' => 'https://bridge.example.com'
]);
// Should attempt to connect (may fail due to invalid credentials, but shouldn't fail validation)
// The exact behavior depends on whether SimpleFINService is mocked
$this->assertNotEquals(route('003-upload.index'), $response->getTargetUrl());
}
public function test_request_data_logging()
{
// Enable debug logging to capture the request data
Log::shouldReceive('debug')
->with('UploadController::upload() - Request All:', \Mockery::type('array'))
->once();
Log::shouldReceive('debug')
->with('handleSimpleFINFlow() - Request All:', \Mockery::type('array'))
->once();
Log::shouldReceive('debug')
->with('handleSimpleFINFlow() - Raw use_demo input:', \Mockery::type('array'))
->once();
Log::shouldReceive('debug')
->with('handleSimpleFINFlow() - Evaluated $isDemo:', [true])
->once();
$this->withCookie(Constants::FLOW_COOKIE, 'simplefin')
->post(route('003-upload.upload'), [
'use_demo' => '1'
]);
}
}

384
tests/Unit/Services/SimpleFIN/Validation/ConfigurationContractValidatorTest.php

@ -0,0 +1,384 @@
<?php
/*
* ConfigurationContractValidatorTest.php
* Copyright (c) 2021 james@firefly-iii.org
*
* This file is part of the Firefly III Data Importer
* (https://github.com/firefly-iii/data-importer).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace Tests\Unit\Services\SimpleFIN\Validation;
use App\Services\Shared\Configuration\Configuration;
use App\Services\SimpleFIN\Validation\ConfigurationContractValidator;
use App\Services\SimpleFIN\Validation\ValidationResult;
use Illuminate\Foundation\Testing\WithFaker;
use Illuminate\Support\Facades\Session;
use Tests\TestCase;
/**
* Class ConfigurationContractValidatorTest
*/
class ConfigurationContractValidatorTest extends TestCase
{
use WithFaker;
private ConfigurationContractValidator $validator;
private Configuration $mockConfiguration;
protected function setUp(): void
{
parent::setUp();
$this->validator = new ConfigurationContractValidator();
// Create a basic valid configuration
$this->mockConfiguration = $this->createMockConfiguration();
}
protected function tearDown(): void
{
Session::flush();
parent::tearDown();
}
/**
* Test successful validation with complete valid configuration
*/
public function testValidateConfigurationContractSuccess(): void
{
$this->setupValidSessionData();
$result = $this->validator->validateConfigurationContract($this->mockConfiguration);
$this->assertTrue($result->isValid());
$this->assertEmpty($result->getErrors());
}
/**
* Test validation failure with invalid flow
*/
public function testValidateConfigurationContractInvalidFlow(): void
{
$invalidConfig = $this->createMockConfiguration('nordigen');
$result = $this->validator->validateConfigurationContract($invalidConfig);
$this->assertFalse($result->isValid());
$this->assertNotEmpty($result->getErrors());
$this->assertStringContainsString('SimpleFIN flow', $result->getErrorMessages()[0]);
}
/**
* Test validation failure with missing session data
*/
public function testValidateConfigurationContractMissingSessionData(): void
{
// Don't set up session data
$result = $this->validator->validateConfigurationContract($this->mockConfiguration);
$this->assertFalse($result->isValid());
$this->assertStringContainsString('SimpleFIN accounts data missing', $result->getErrorMessages()[0]);
}
/**
* Test validation failure with invalid SimpleFIN account structure
*/
public function testValidateConfigurationContractInvalidAccountStructure(): void
{
Session::put('simplefin_accounts_data', [
[
'id' => 'acc1',
'name' => 'Test Account',
// Missing required fields: currency, balance, balance-date, org
]
]);
$result = $this->validator->validateConfigurationContract($this->mockConfiguration);
$this->assertFalse($result->isValid());
$this->assertTrue(count($result->getErrors()) >= 4); // Missing 4 required fields
}
/**
* Test validation failure with invalid account mappings
*/
public function testValidateConfigurationContractInvalidAccountMappings(): void
{
$this->setupValidSessionData();
// Create configuration with invalid account mappings
$config = $this->createMockConfiguration();
$config->setAccounts([
'acc1' => -1, // Invalid negative ID
'' => 0, // Invalid empty account ID
]);
$result = $this->validator->validateConfigurationContract($config);
$this->assertFalse($result->isValid());
$this->assertNotEmpty($result->getErrors());
}
/**
* Test validation failure with missing new account configuration
*/
public function testValidateConfigurationContractMissingNewAccountConfig(): void
{
$this->setupValidSessionData();
// Create configuration with account marked for creation but no new account config
$config = $this->createMockConfiguration();
$config->setAccounts(['acc1' => 0]); // Mark for creation
$config->setNewAccounts([]); // But no configuration
$result = $this->validator->validateConfigurationContract($config);
$this->assertFalse($result->isValid());
$this->assertStringContainsString('New account configuration missing', $result->getErrorMessages()[0]);
}
/**
* Test validation failure with invalid new account configuration
*/
public function testValidateConfigurationContractInvalidNewAccountConfig(): void
{
$this->setupValidSessionData();
$config = $this->createMockConfiguration();
$config->setAccounts(['acc1' => 0]);
$config->setNewAccounts([
'acc1' => [
'name' => '', // Empty name
'type' => 'invalid', // Invalid type
'currency' => 'USDD', // Invalid currency format
'opening_balance' => 'not_numeric', // Invalid balance
]
]);
$result = $this->validator->validateConfigurationContract($config);
$this->assertFalse($result->isValid());
$this->assertTrue(count($result->getErrors()) >= 4);
}
/**
* Test validation of liability account requirements
*/
public function testValidateConfigurationContractLiabilityAccountRequirements(): void
{
$this->setupValidSessionData();
$config = $this->createMockConfiguration();
$config->setAccounts(['acc1' => 0]);
$config->setNewAccounts([
'acc1' => [
'name' => 'Credit Card',
'type' => 'liability',
'currency' => 'USD',
'opening_balance' => '1000.00',
// Missing liability_type and liability_direction
]
]);
$result = $this->validator->validateConfigurationContract($config);
$this->assertFalse($result->isValid());
$errors = $result->getErrors();
$this->assertTrue(
collect($errors)->contains(fn($error) => str_contains($error['message'], 'Liability type required'))
);
$this->assertTrue(
collect($errors)->contains(fn($error) => str_contains($error['message'], 'Liability direction required'))
);
}
/**
* Test form field structure validation success
*/
public function testValidateFormFieldStructureSuccess(): void
{
$validFormData = [
'do_import' => ['acc1' => '1'],
'accounts' => ['acc1' => 0],
'new_account' => [
'acc1' => [
'name' => 'Test Account',
'type' => 'asset',
'currency' => 'USD',
'opening_balance' => '1000.00',
]
]
];
$result = $this->validator->validateFormFieldStructure($validFormData);
$this->assertTrue($result->isValid());
$this->assertEmpty($result->getErrors());
}
/**
* Test form field structure validation failure
*/
public function testValidateFormFieldStructureFailure(): void
{
$invalidFormData = [
'do_import' => 'not_array', // Should be array
// Missing 'accounts' and 'new_account'
];
$result = $this->validator->validateFormFieldStructure($invalidFormData);
$this->assertFalse($result->isValid());
$this->assertTrue(count($result->getErrors()) >= 3); // Missing/invalid fields
}
/**
* Test ValidationResult class functionality
*/
public function testValidationResultClass(): void
{
$errors = [
['field' => 'test', 'message' => 'Test error', 'value' => null]
];
$warnings = [
['field' => 'test', 'message' => 'Test warning', 'value' => null]
];
// Test invalid result
$invalidResult = new ValidationResult(false, $errors, $warnings);
$this->assertFalse($invalidResult->isValid());
$this->assertTrue($invalidResult->hasErrors());
$this->assertTrue($invalidResult->hasWarnings());
$this->assertEquals(['Test error'], $invalidResult->getErrorMessages());
$this->assertEquals(['Test warning'], $invalidResult->getWarningMessages());
// Test valid result
$validResult = new ValidationResult(true);
$this->assertTrue($validResult->isValid());
$this->assertFalse($validResult->hasErrors());
$this->assertFalse($validResult->hasWarnings());
$this->assertEmpty($validResult->getErrors());
$this->assertEmpty($validResult->getWarnings());
// Test toArray
$array = $invalidResult->toArray();
$this->assertArrayHasKey('valid', $array);
$this->assertArrayHasKey('errors', $array);
$this->assertArrayHasKey('warnings', $array);
$this->assertFalse($array['valid']);
}
/**
* Test asset account role validation
*/
public function testAssetAccountRoleValidation(): void
{
$this->setupValidSessionData();
$config = $this->createMockConfiguration();
$config->setAccounts(['acc1' => 0]);
$config->setNewAccounts([
'acc1' => [
'name' => 'Savings Account',
'type' => 'asset',
'currency' => 'USD',
'opening_balance' => '1000.00',
'account_role' => 'invalidRole', // Invalid role
]
]);
$result = $this->validator->validateConfigurationContract($config);
$this->assertFalse($result->isValid());
$this->assertTrue(
collect($result->getErrors())->contains(fn($error) => str_contains($error['message'], 'Invalid account role'))
);
}
/**
* Test import selection validation
*/
public function testImportSelectionValidation(): void
{
$this->setupValidSessionData();
Session::put('do_import', ['nonexistent_acc' => '1']);
$result = $this->validator->validateConfigurationContract($this->mockConfiguration);
$this->assertFalse($result->isValid());
$this->assertTrue(
collect($result->getErrors())->contains(fn($error) => str_contains($error['message'], 'selected for import but not in account mappings'))
);
}
/**
* Create a mock Configuration object for testing
*/
private function createMockConfiguration(string $flow = 'simplefin'): Configuration
{
$config = Configuration::fromArray([
'flow' => $flow,
'accounts' => ['acc1' => 1, 'acc2' => 0],
'new_account' => [
'acc2' => [
'name' => 'New Account',
'type' => 'asset',
'currency' => 'USD',
'opening_balance' => '1000.00',
'account_role' => 'defaultAsset',
]
]
]);
return $config;
}
/**
* Set up valid session data for testing
*/
private function setupValidSessionData(): void
{
Session::put('simplefin_accounts_data', [
[
'id' => 'acc1',
'name' => 'Test Account 1',
'currency' => 'USD',
'balance' => '1000.00',
'balance-date' => time(),
'org' => ['name' => 'Test Bank'],
'extra' => []
],
[
'id' => 'acc2',
'name' => 'Test Account 2',
'currency' => 'USD',
'balance' => '2000.00',
'balance-date' => time(),
'org' => ['name' => 'Test Bank'],
'extra' => []
]
]);
Session::put('do_import', [
'acc1' => '1',
'acc2' => '1'
]);
}
}
Loading…
Cancel
Save