Browse Source

Add commands to verify JSON and make the schema better.

pull/972/head
James Cole 3 weeks ago
parent
commit
e9152a4f8d
  1. 45
      app/Console/Commands/ValidateJsonFile.php
  2. 56
      app/Console/Commands/ValidateJsonFiles.php
  3. 19
      app/Console/VerifyJSON.php
  4. 8
      app/Http/Controllers/Import/DownloadController.php
  5. 64
      app/Http/Controllers/Import/UploadController.php
  6. 3
      composer.json
  7. 141
      composer.lock
  8. 41
      resources/schemas/v3.json

45
app/Console/Commands/ValidateJsonFile.php

@ -0,0 +1,45 @@
<?php
namespace App\Console\Commands;
use App\Console\VerifyJSON;
use Illuminate\Console\Command;
use Symfony\Component\Console\Command\Command as CommandAlias;
class ValidateJsonFile extends Command
{
use VerifyJSON;
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'import:validate-json {file : The JSON file to validate}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Checks if a JSON file is valid according to the v3 import configuration file standard.';
/**
* Execute the console command.
*/
public function handle(): int
{
$file = (string)$this->argument('file');
if (!is_file($file) || !is_readable($file)) {
$this->error(sprintf('File %s does not exist or is not readable.', $file));
return CommandAlias::FAILURE;
}
$result = $this->verifyJSON($file);
if(false === $result) {
$this->error('File is not valid JSON.');
return CommandAlias::FAILURE;
}
$this->info('File is valid JSON.');
return CommandAlias::SUCCESS;
}
}

56
app/Console/Commands/ValidateJsonFiles.php

@ -0,0 +1,56 @@
<?php
namespace App\Console\Commands;
use App\Console\VerifyJSON;
use Illuminate\Console\Command;
use Symfony\Component\Console\Command\Command as CommandAlias;
class ValidateJsonFiles extends Command
{
use VerifyJSON;
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'import:validate-json-directory {directory : The directory with JSON files to validate}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Recursively validate all JSON files in a directory. Stops after 100 files.';
/**
* Execute the console command.
*/
public function handle(): int
{
$directory = (string)$this->argument('directory');
if (!is_dir($directory) || !is_readable($directory)) {
$this->error(sprintf('Cannot read directory %s.', $directory));
return CommandAlias::FAILURE;
}
// check each file in the directory and see if it needs action.
// collect recursively:
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($directory, \RecursiveDirectoryIterator::SKIP_DOTS));
$Regex = new \RegexIterator($it, '/^.+\.json$/i', \RecursiveRegexIterator::GET_MATCH);
$fullPaths = [];
foreach ($Regex as $item) {
$path = $item[0];
$fullPaths[] = $path;
}
foreach ($fullPaths as $file) {
$result = $this->verifyJSON($file);
if (false === $result) {
$this->error(sprintf('File "%s" is not valid JSON.', $file));
return CommandAlias::FAILURE;
}
$this->info(sprintf('File "%s" is valid JSON.', $file));
}
return CommandAlias::SUCCESS;
}
}

19
app/Console/VerifyJSON.php

@ -35,31 +35,40 @@ use Swaggest\JsonSchema\Schema;
*/
trait VerifyJSON
{
protected string $errorMessage = '';
private function verifyJSON(string $file): bool
{
// basic check on the JSON.
$json = (string)file_get_contents($file);
$json = (string)file_get_contents($file);
try {
$config = json_decode($json, null, 512, JSON_THROW_ON_ERROR);
} catch (JsonException $e) {
Log::error(sprintf('The importer can\'t import: could not decode the JSON in the config file: %s', $e->getMessage()));
$message = sprintf('The importer can\'t import: could not decode the JSON in the config file: %s', $e->getMessage());
Log::error($message);
$this->errorMessage = $message;
return false;
}
// validate JSON schema.
$schemaFile = resource_path('schemas/v3.json');
if (!file_exists($schemaFile)) {
Log::error(sprintf('The schema file "%s" does not exist.', $schemaFile));
$message = sprintf('The schema file "%s" does not exist.', $schemaFile);
Log::error($message);
$this->errorMessage = $message;
return false;
}
$schema = json_decode(file_get_contents($schemaFile));
$schema = json_decode(file_get_contents($schemaFile));
try {
Schema::import($schema)->in($config);
} catch (Exception|\Exception $e) {
Log::error(sprintf('Configuration file "%s" does not adhere to the v3 schema: %s', $file, $e->getMessage()));
$message = sprintf('Configuration file "%s" does not adhere to the v3 schema: %s', $file, $e->getMessage());
Log::error($message);
$this->errorMessage = $message;
return false;
}

8
app/Http/Controllers/Import/DownloadController.php

@ -53,6 +53,14 @@ class DownloadController extends Controller
if(is_array($array['mapping']) && 0 === count($array['mapping'])) {
$array['mapping'] = new \stdClass();
}
// same for "accounts"
if(is_array($array['accounts']) && 0 === count($array['accounts'])) {
$array['accounts'] = new \stdClass();
}
// same for "accounts"
if(is_array($array['nordigen_requisitions']) && 0 === count($array['nordigen_requisitions'])) {
$array['nordigen_requisitions'] = new \stdClass();
}
$result = json_encode($array, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR);

64
app/Http/Controllers/Import/UploadController.php

@ -25,6 +25,7 @@ declare(strict_types=1);
namespace App\Http\Controllers\Import;
use App\Console\VerifyJSON;
use App\Exceptions\ImporterErrorException;
use App\Http\Controllers\Controller;
use App\Http\Middleware\UploadControllerMiddleware;
@ -53,6 +54,7 @@ use Storage;
class UploadController extends Controller
{
use RestoresConfiguration;
use VerifyJSON;
private string $configFileName;
private string $contentType;
@ -76,22 +78,22 @@ class UploadController extends Controller
public function index(Request $request)
{
Log::debug(sprintf('Now at %s', __METHOD__));
$mainTitle = 'Upload your file(s)';
$subTitle = 'Start page and instructions';
$flow = $request->cookie(Constants::FLOW_COOKIE);
$mainTitle = 'Upload your file(s)';
$subTitle = 'Start page and instructions';
$flow = $request->cookie(Constants::FLOW_COOKIE);
// simplefin settings.
$simpleFinToken = config('simplefin.token');
$simpleFinOriginUrl = config('simplefin.origin_url');
// get existing configs.
$disk = Storage::disk('configurations');
$disk = Storage::disk('configurations');
Log::debug(sprintf('Going to check directory for config files: %s', config('filesystems.disks.configurations.root')));
$all = $disk->files();
$all = $disk->files();
// remove files from list
$list = [];
$ignored = config('importer.ignored_files');
$list = [];
$ignored = config('importer.ignored_files');
foreach ($all as $entry) {
if (!in_array($entry, $ignored, true)) {
$list[] = $entry;
@ -113,13 +115,13 @@ class UploadController extends Controller
public function upload(Request $request)
{
Log::debug(sprintf('Now at %s', __METHOD__));
$importedFile = $request->file('importable_file');
$configFile = $request->file('config_file');
$flow = $request->cookie(Constants::FLOW_COOKIE);
$errors = new MessageBag();
$importedFile = $request->file('importable_file');
$configFile = $request->file('config_file');
$flow = $request->cookie(Constants::FLOW_COOKIE);
$errors = new MessageBag();
// process uploaded file (if present)
$errors = $this->processUploadedFile($flow, $errors, $importedFile);
$errors = $this->processUploadedFile($flow, $errors, $importedFile);
// process config file (if present)
if (0 === count($errors) && null !== $configFile) {
@ -127,7 +129,7 @@ class UploadController extends Controller
}
// process pre-selected file (if present):
$errors = $this->processSelection($errors, (string)$request->get('existing_config'), $configFile);
$errors = $this->processSelection($errors, (string)$request->get('existing_config'), $configFile);
if ($errors->count() > 0) {
return redirect(route('003-upload.index'))->withErrors($errors)->withInput();
@ -185,7 +187,7 @@ class UploadController extends Controller
// https://stackoverflow.com/questions/11066857/detect-eol-type-using-php
// because apparently there are banks that use "\r" as newline. Looking at the morons of KBC Bank, Belgium.
// This one is for you: 🤦‍♀️
$eol = $this->detectEOL($content);
$eol = $this->detectEOL($content);
if ("\r" === $eol) {
Log::error('Your bank is dumb. Tell them to fix their CSV files.');
$content = str_replace("\r", "\n", $content);
@ -195,7 +197,7 @@ class UploadController extends Controller
if ('camt' === $this->contentType) {
$content = (string)file_get_contents($file->getPathname());
}
$fileName = StorageService::storeContent($content);
$fileName = StorageService::storeContent($content);
session()->put(Constants::UPLOAD_DATA_FILE, $fileName);
session()->put(Constants::HAS_UPLOAD, true);
}
@ -215,9 +217,9 @@ class UploadController extends Controller
private function detectEOL(string $string): string
{
$eols = ['\n\r' => "\n\r", // 0x0A - 0x0D - acorn BBC
'\r\n' => "\r\n", // 0x0D - 0x0A - Windows, DOS OS/2
'\n' => "\n", // 0x0A - - Unix, OSX
'\r' => "\r", // 0x0D - - Apple ][, TRS80
'\r\n' => "\r\n", // 0x0D - 0x0A - Windows, DOS OS/2
'\n' => "\n", // 0x0A - - Unix, OSX
'\r' => "\r", // 0x0D - - Apple ][, TRS80
];
$curCount = 0;
$curEol = '';
@ -247,19 +249,27 @@ class UploadController extends Controller
// upload the file to a temp directory and use it from there.
if (0 === $errorNumber) {
Log::debug('Config file uploaded.');
$this->configFileName = StorageService::storeContent((string)file_get_contents($file->getPathname()));
$path = $file->getPathname();
$validation = $this->verifyJSON($path);
if (false === $validation) {
$errors->add('config_file', $this->errorMessage);
return $errors;
}
$content = (string)file_get_contents($path);
$this->configFileName = StorageService::storeContent($content);
session()->put(Constants::UPLOAD_CONFIG_FILE, $this->configFileName);
// process the config file
$success = false;
$configuration = null;
$success = false;
$configuration = null;
try {
$configuration = ConfigFileProcessor::convertConfigFile($this->configFileName);
$configuration->setContentType($this->contentType);
session()->put(Constants::CONFIGURATION, $configuration->toSessionArray());
$success = true;
$success = true;
} catch (ImporterErrorException $e) {
$errors->add('config_file', $e->getMessage());
}
@ -303,12 +313,12 @@ class UploadController extends Controller
*/
private function handleSimpleFINFlow(Request $request, Configuration $configuration): RedirectResponse
{
$errors = new MessageBag();
$errors = new MessageBag();
Log::debug('UploadController::handleSimpleFINFlow() INVOKED'); // Unique entry marker
$setupToken = (string)$request->get('simplefin_token');
$isDemo = $request->boolean('use_demo');
$accessToken = $configuration->getAccessToken();
$setupToken = (string)$request->get('simplefin_token');
$isDemo = $request->boolean('use_demo');
$accessToken = $configuration->getAccessToken();
Log::debug(sprintf('handleSimpleFINFlow("%s")', $setupToken));
if ($isDemo) {
@ -348,7 +358,7 @@ class UploadController extends Controller
$configuration->setAccessToken($accessToken);
try {
$accountsData = $simpleFINService->fetchAccounts();
$accountsData = $simpleFINService->fetchAccounts();
// save configuration in session and on disk: TODO needs a trait.
Log::debug('Save config to disk after setting access token.');

3
composer.json

@ -83,7 +83,8 @@
"spatie/enum": "^3.10",
"swaggest/json-schema": "^0.12.43",
"symfony/http-client": "^7.3",
"symfony/mailgun-mailer": "^7.3"
"symfony/mailgun-mailer": "^7.3",
"thecodingmachine/safe": "^3.3"
},
"require-dev": {
"barryvdh/laravel-debugbar": "^3.16",

141
composer.lock

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "90550ddfcbcb4c8043368ef25abd0c9d",
"content-hash": "b7cf2032b370512926f502bdd4038b70",
"packages": [
{
"name": "brick/math",
@ -6316,6 +6316,145 @@
],
"time": "2025-08-13T11:49:31+00:00"
},
{
"name": "thecodingmachine/safe",
"version": "v3.3.0",
"source": {
"type": "git",
"url": "https://github.com/thecodingmachine/safe.git",
"reference": "2cdd579eeaa2e78e51c7509b50cc9fb89a956236"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/thecodingmachine/safe/zipball/2cdd579eeaa2e78e51c7509b50cc9fb89a956236",
"reference": "2cdd579eeaa2e78e51c7509b50cc9fb89a956236",
"shasum": ""
},
"require": {
"php": "^8.1"
},
"require-dev": {
"php-parallel-lint/php-parallel-lint": "^1.4",
"phpstan/phpstan": "^2",
"phpunit/phpunit": "^10",
"squizlabs/php_codesniffer": "^3.2"
},
"type": "library",
"autoload": {
"files": [
"lib/special_cases.php",
"generated/apache.php",
"generated/apcu.php",
"generated/array.php",
"generated/bzip2.php",
"generated/calendar.php",
"generated/classobj.php",
"generated/com.php",
"generated/cubrid.php",
"generated/curl.php",
"generated/datetime.php",
"generated/dir.php",
"generated/eio.php",
"generated/errorfunc.php",
"generated/exec.php",
"generated/fileinfo.php",
"generated/filesystem.php",
"generated/filter.php",
"generated/fpm.php",
"generated/ftp.php",
"generated/funchand.php",
"generated/gettext.php",
"generated/gmp.php",
"generated/gnupg.php",
"generated/hash.php",
"generated/ibase.php",
"generated/ibmDb2.php",
"generated/iconv.php",
"generated/image.php",
"generated/imap.php",
"generated/info.php",
"generated/inotify.php",
"generated/json.php",
"generated/ldap.php",
"generated/libxml.php",
"generated/lzf.php",
"generated/mailparse.php",
"generated/mbstring.php",
"generated/misc.php",
"generated/mysql.php",
"generated/mysqli.php",
"generated/network.php",
"generated/oci8.php",
"generated/opcache.php",
"generated/openssl.php",
"generated/outcontrol.php",
"generated/pcntl.php",
"generated/pcre.php",
"generated/pgsql.php",
"generated/posix.php",
"generated/ps.php",
"generated/pspell.php",
"generated/readline.php",
"generated/rnp.php",
"generated/rpminfo.php",
"generated/rrd.php",
"generated/sem.php",
"generated/session.php",
"generated/shmop.php",
"generated/sockets.php",
"generated/sodium.php",
"generated/solr.php",
"generated/spl.php",
"generated/sqlsrv.php",
"generated/ssdeep.php",
"generated/ssh2.php",
"generated/stream.php",
"generated/strings.php",
"generated/swoole.php",
"generated/uodbc.php",
"generated/uopz.php",
"generated/url.php",
"generated/var.php",
"generated/xdiff.php",
"generated/xml.php",
"generated/xmlrpc.php",
"generated/yaml.php",
"generated/yaz.php",
"generated/zip.php",
"generated/zlib.php"
],
"classmap": [
"lib/DateTime.php",
"lib/DateTimeImmutable.php",
"lib/Exceptions/",
"generated/Exceptions/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"description": "PHP core functions that throw exceptions instead of returning FALSE on error",
"support": {
"issues": "https://github.com/thecodingmachine/safe/issues",
"source": "https://github.com/thecodingmachine/safe/tree/v3.3.0"
},
"funding": [
{
"url": "https://github.com/OskarStark",
"type": "github"
},
{
"url": "https://github.com/shish",
"type": "github"
},
{
"url": "https://github.com/staabm",
"type": "github"
}
],
"time": "2025-05-14T06:15:44+00:00"
},
{
"name": "tijsverkoyen/css-to-inline-styles",
"version": "v2.3.0",

41
resources/schemas/v3.json

@ -26,7 +26,7 @@
},
"default_account": {
"type": "integer",
"minimum": 1
"minimum": 0
},
"delimiter": {
"type": "string",
@ -55,14 +55,27 @@
}
},
"do_mapping": {
"type": "array",
"items":
"anyOf": [
{
"type": "boolean"
"type": "object"
},
{
"type": "array",
"items": {
"type": "boolean"
}
}
]
},
"mapping": {
"type": "object"
"anyOf": [
{
"type": "object"
},
{
"type": "array"
}
]
},
"duplicate_detection_method": {
"type": "string",
@ -107,7 +120,14 @@
"type": "boolean"
},
"accounts": {
"type": "object"
"anyOf": [
{
"type": "object"
},
{
"type": "array"
}
]
},
"date_range": {
"type": "string"
@ -131,7 +151,14 @@
"type": "string"
},
"nordigen_requisitions": {
"type": "object"
"anyOf": [
{
"type": "object"
},
{
"type": "array"
}
]
},
"conversion": {
"type": "boolean"

Loading…
Cancel
Save