diff --git a/app/Console/Commands/ValidateJsonFile.php b/app/Console/Commands/ValidateJsonFile.php new file mode 100644 index 00000000..c7db73a2 --- /dev/null +++ b/app/Console/Commands/ValidateJsonFile.php @@ -0,0 +1,45 @@ +argument('file'); + if (!is_file($file) || !is_readable($file)) { + $this->error(sprintf('File %s does not exist or is not readable.', $file)); + return CommandAlias::FAILURE; + } + $result = $this->verifyJSON($file); + if(false === $result) { + $this->error('File is not valid JSON.'); + return CommandAlias::FAILURE; + } + + $this->info('File is valid JSON.'); + return CommandAlias::SUCCESS; + } +} diff --git a/app/Console/Commands/ValidateJsonFiles.php b/app/Console/Commands/ValidateJsonFiles.php new file mode 100644 index 00000000..82ac94c6 --- /dev/null +++ b/app/Console/Commands/ValidateJsonFiles.php @@ -0,0 +1,56 @@ +argument('directory'); + if (!is_dir($directory) || !is_readable($directory)) { + $this->error(sprintf('Cannot read directory %s.', $directory)); + return CommandAlias::FAILURE; + } + + // check each file in the directory and see if it needs action. + // collect recursively: + $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($directory, \RecursiveDirectoryIterator::SKIP_DOTS)); + $Regex = new \RegexIterator($it, '/^.+\.json$/i', \RecursiveRegexIterator::GET_MATCH); + $fullPaths = []; + foreach ($Regex as $item) { + $path = $item[0]; + $fullPaths[] = $path; + } + foreach ($fullPaths as $file) { + $result = $this->verifyJSON($file); + if (false === $result) { + $this->error(sprintf('File "%s" is not valid JSON.', $file)); + return CommandAlias::FAILURE; + } + $this->info(sprintf('File "%s" is valid JSON.', $file)); + } + return CommandAlias::SUCCESS; + } +} diff --git a/app/Console/VerifyJSON.php b/app/Console/VerifyJSON.php index 5712eeb6..3d4e7bd9 100644 --- a/app/Console/VerifyJSON.php +++ b/app/Console/VerifyJSON.php @@ -35,31 +35,40 @@ use Swaggest\JsonSchema\Schema; */ trait VerifyJSON { + protected string $errorMessage = ''; + private function verifyJSON(string $file): bool { // basic check on the JSON. - $json = (string)file_get_contents($file); + $json = (string)file_get_contents($file); try { $config = json_decode($json, null, 512, JSON_THROW_ON_ERROR); } catch (JsonException $e) { - Log::error(sprintf('The importer can\'t import: could not decode the JSON in the config file: %s', $e->getMessage())); + $message = sprintf('The importer can\'t import: could not decode the JSON in the config file: %s', $e->getMessage()); + Log::error($message); + $this->errorMessage = $message; return false; } // validate JSON schema. $schemaFile = resource_path('schemas/v3.json'); if (!file_exists($schemaFile)) { - Log::error(sprintf('The schema file "%s" does not exist.', $schemaFile)); + $message = sprintf('The schema file "%s" does not exist.', $schemaFile); + Log::error($message); + $this->errorMessage = $message; return false; } - $schema = json_decode(file_get_contents($schemaFile)); + $schema = json_decode(file_get_contents($schemaFile)); try { Schema::import($schema)->in($config); } catch (Exception|\Exception $e) { - Log::error(sprintf('Configuration file "%s" does not adhere to the v3 schema: %s', $file, $e->getMessage())); + $message = sprintf('Configuration file "%s" does not adhere to the v3 schema: %s', $file, $e->getMessage()); + + Log::error($message); + $this->errorMessage = $message; return false; } diff --git a/app/Http/Controllers/Import/DownloadController.php b/app/Http/Controllers/Import/DownloadController.php index 9c24a40f..58652695 100644 --- a/app/Http/Controllers/Import/DownloadController.php +++ b/app/Http/Controllers/Import/DownloadController.php @@ -53,6 +53,14 @@ class DownloadController extends Controller if(is_array($array['mapping']) && 0 === count($array['mapping'])) { $array['mapping'] = new \stdClass(); } + // same for "accounts" + if(is_array($array['accounts']) && 0 === count($array['accounts'])) { + $array['accounts'] = new \stdClass(); + } + // same for "accounts" + if(is_array($array['nordigen_requisitions']) && 0 === count($array['nordigen_requisitions'])) { + $array['nordigen_requisitions'] = new \stdClass(); + } $result = json_encode($array, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR); diff --git a/app/Http/Controllers/Import/UploadController.php b/app/Http/Controllers/Import/UploadController.php index 0b59edb5..edfc2a44 100644 --- a/app/Http/Controllers/Import/UploadController.php +++ b/app/Http/Controllers/Import/UploadController.php @@ -25,6 +25,7 @@ declare(strict_types=1); namespace App\Http\Controllers\Import; +use App\Console\VerifyJSON; use App\Exceptions\ImporterErrorException; use App\Http\Controllers\Controller; use App\Http\Middleware\UploadControllerMiddleware; @@ -53,6 +54,7 @@ use Storage; class UploadController extends Controller { use RestoresConfiguration; + use VerifyJSON; private string $configFileName; private string $contentType; @@ -76,22 +78,22 @@ class UploadController extends Controller public function index(Request $request) { Log::debug(sprintf('Now at %s', __METHOD__)); - $mainTitle = 'Upload your file(s)'; - $subTitle = 'Start page and instructions'; - $flow = $request->cookie(Constants::FLOW_COOKIE); + $mainTitle = 'Upload your file(s)'; + $subTitle = 'Start page and instructions'; + $flow = $request->cookie(Constants::FLOW_COOKIE); // simplefin settings. $simpleFinToken = config('simplefin.token'); $simpleFinOriginUrl = config('simplefin.origin_url'); // get existing configs. - $disk = Storage::disk('configurations'); + $disk = Storage::disk('configurations'); Log::debug(sprintf('Going to check directory for config files: %s', config('filesystems.disks.configurations.root'))); - $all = $disk->files(); + $all = $disk->files(); // remove files from list - $list = []; - $ignored = config('importer.ignored_files'); + $list = []; + $ignored = config('importer.ignored_files'); foreach ($all as $entry) { if (!in_array($entry, $ignored, true)) { $list[] = $entry; @@ -113,13 +115,13 @@ class UploadController extends Controller public function upload(Request $request) { Log::debug(sprintf('Now at %s', __METHOD__)); - $importedFile = $request->file('importable_file'); - $configFile = $request->file('config_file'); - $flow = $request->cookie(Constants::FLOW_COOKIE); - $errors = new MessageBag(); + $importedFile = $request->file('importable_file'); + $configFile = $request->file('config_file'); + $flow = $request->cookie(Constants::FLOW_COOKIE); + $errors = new MessageBag(); // process uploaded file (if present) - $errors = $this->processUploadedFile($flow, $errors, $importedFile); + $errors = $this->processUploadedFile($flow, $errors, $importedFile); // process config file (if present) if (0 === count($errors) && null !== $configFile) { @@ -127,7 +129,7 @@ class UploadController extends Controller } // process pre-selected file (if present): - $errors = $this->processSelection($errors, (string)$request->get('existing_config'), $configFile); + $errors = $this->processSelection($errors, (string)$request->get('existing_config'), $configFile); if ($errors->count() > 0) { return redirect(route('003-upload.index'))->withErrors($errors)->withInput(); @@ -185,7 +187,7 @@ class UploadController extends Controller // https://stackoverflow.com/questions/11066857/detect-eol-type-using-php // because apparently there are banks that use "\r" as newline. Looking at the morons of KBC Bank, Belgium. // This one is for you: 🤦‍♀️ - $eol = $this->detectEOL($content); + $eol = $this->detectEOL($content); if ("\r" === $eol) { Log::error('Your bank is dumb. Tell them to fix their CSV files.'); $content = str_replace("\r", "\n", $content); @@ -195,7 +197,7 @@ class UploadController extends Controller if ('camt' === $this->contentType) { $content = (string)file_get_contents($file->getPathname()); } - $fileName = StorageService::storeContent($content); + $fileName = StorageService::storeContent($content); session()->put(Constants::UPLOAD_DATA_FILE, $fileName); session()->put(Constants::HAS_UPLOAD, true); } @@ -215,9 +217,9 @@ class UploadController extends Controller private function detectEOL(string $string): string { $eols = ['\n\r' => "\n\r", // 0x0A - 0x0D - acorn BBC - '\r\n' => "\r\n", // 0x0D - 0x0A - Windows, DOS OS/2 - '\n' => "\n", // 0x0A - - Unix, OSX - '\r' => "\r", // 0x0D - - Apple ][, TRS80 + '\r\n' => "\r\n", // 0x0D - 0x0A - Windows, DOS OS/2 + '\n' => "\n", // 0x0A - - Unix, OSX + '\r' => "\r", // 0x0D - - Apple ][, TRS80 ]; $curCount = 0; $curEol = ''; @@ -247,19 +249,27 @@ class UploadController extends Controller // upload the file to a temp directory and use it from there. if (0 === $errorNumber) { Log::debug('Config file uploaded.'); - $this->configFileName = StorageService::storeContent((string)file_get_contents($file->getPathname())); + $path = $file->getPathname(); + $validation = $this->verifyJSON($path); + if (false === $validation) { + $errors->add('config_file', $this->errorMessage); + return $errors; + } + + $content = (string)file_get_contents($path); + $this->configFileName = StorageService::storeContent($content); session()->put(Constants::UPLOAD_CONFIG_FILE, $this->configFileName); // process the config file - $success = false; - $configuration = null; + $success = false; + $configuration = null; try { $configuration = ConfigFileProcessor::convertConfigFile($this->configFileName); $configuration->setContentType($this->contentType); session()->put(Constants::CONFIGURATION, $configuration->toSessionArray()); - $success = true; + $success = true; } catch (ImporterErrorException $e) { $errors->add('config_file', $e->getMessage()); } @@ -303,12 +313,12 @@ class UploadController extends Controller */ private function handleSimpleFINFlow(Request $request, Configuration $configuration): RedirectResponse { - $errors = new MessageBag(); + $errors = new MessageBag(); Log::debug('UploadController::handleSimpleFINFlow() INVOKED'); // Unique entry marker - $setupToken = (string)$request->get('simplefin_token'); - $isDemo = $request->boolean('use_demo'); - $accessToken = $configuration->getAccessToken(); + $setupToken = (string)$request->get('simplefin_token'); + $isDemo = $request->boolean('use_demo'); + $accessToken = $configuration->getAccessToken(); Log::debug(sprintf('handleSimpleFINFlow("%s")', $setupToken)); if ($isDemo) { @@ -348,7 +358,7 @@ class UploadController extends Controller $configuration->setAccessToken($accessToken); try { - $accountsData = $simpleFINService->fetchAccounts(); + $accountsData = $simpleFINService->fetchAccounts(); // save configuration in session and on disk: TODO needs a trait. Log::debug('Save config to disk after setting access token.'); diff --git a/composer.json b/composer.json index 0185ab7d..a8e642f7 100644 --- a/composer.json +++ b/composer.json @@ -83,7 +83,8 @@ "spatie/enum": "^3.10", "swaggest/json-schema": "^0.12.43", "symfony/http-client": "^7.3", - "symfony/mailgun-mailer": "^7.3" + "symfony/mailgun-mailer": "^7.3", + "thecodingmachine/safe": "^3.3" }, "require-dev": { "barryvdh/laravel-debugbar": "^3.16", diff --git a/composer.lock b/composer.lock index 07b0ca87..778f2618 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "90550ddfcbcb4c8043368ef25abd0c9d", + "content-hash": "b7cf2032b370512926f502bdd4038b70", "packages": [ { "name": "brick/math", @@ -6316,6 +6316,145 @@ ], "time": "2025-08-13T11:49:31+00:00" }, + { + "name": "thecodingmachine/safe", + "version": "v3.3.0", + "source": { + "type": "git", + "url": "https://github.com/thecodingmachine/safe.git", + "reference": "2cdd579eeaa2e78e51c7509b50cc9fb89a956236" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thecodingmachine/safe/zipball/2cdd579eeaa2e78e51c7509b50cc9fb89a956236", + "reference": "2cdd579eeaa2e78e51c7509b50cc9fb89a956236", + "shasum": "" + }, + "require": { + "php": "^8.1" + }, + "require-dev": { + "php-parallel-lint/php-parallel-lint": "^1.4", + "phpstan/phpstan": "^2", + "phpunit/phpunit": "^10", + "squizlabs/php_codesniffer": "^3.2" + }, + "type": "library", + "autoload": { + "files": [ + "lib/special_cases.php", + "generated/apache.php", + "generated/apcu.php", + "generated/array.php", + "generated/bzip2.php", + "generated/calendar.php", + "generated/classobj.php", + "generated/com.php", + "generated/cubrid.php", + "generated/curl.php", + "generated/datetime.php", + "generated/dir.php", + "generated/eio.php", + "generated/errorfunc.php", + "generated/exec.php", + "generated/fileinfo.php", + "generated/filesystem.php", + "generated/filter.php", + "generated/fpm.php", + "generated/ftp.php", + "generated/funchand.php", + "generated/gettext.php", + "generated/gmp.php", + "generated/gnupg.php", + "generated/hash.php", + "generated/ibase.php", + "generated/ibmDb2.php", + "generated/iconv.php", + "generated/image.php", + "generated/imap.php", + "generated/info.php", + "generated/inotify.php", + "generated/json.php", + "generated/ldap.php", + "generated/libxml.php", + "generated/lzf.php", + "generated/mailparse.php", + "generated/mbstring.php", + "generated/misc.php", + "generated/mysql.php", + "generated/mysqli.php", + "generated/network.php", + "generated/oci8.php", + "generated/opcache.php", + "generated/openssl.php", + "generated/outcontrol.php", + "generated/pcntl.php", + "generated/pcre.php", + "generated/pgsql.php", + "generated/posix.php", + "generated/ps.php", + "generated/pspell.php", + "generated/readline.php", + "generated/rnp.php", + "generated/rpminfo.php", + "generated/rrd.php", + "generated/sem.php", + "generated/session.php", + "generated/shmop.php", + "generated/sockets.php", + "generated/sodium.php", + "generated/solr.php", + "generated/spl.php", + "generated/sqlsrv.php", + "generated/ssdeep.php", + "generated/ssh2.php", + "generated/stream.php", + "generated/strings.php", + "generated/swoole.php", + "generated/uodbc.php", + "generated/uopz.php", + "generated/url.php", + "generated/var.php", + "generated/xdiff.php", + "generated/xml.php", + "generated/xmlrpc.php", + "generated/yaml.php", + "generated/yaz.php", + "generated/zip.php", + "generated/zlib.php" + ], + "classmap": [ + "lib/DateTime.php", + "lib/DateTimeImmutable.php", + "lib/Exceptions/", + "generated/Exceptions/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "PHP core functions that throw exceptions instead of returning FALSE on error", + "support": { + "issues": "https://github.com/thecodingmachine/safe/issues", + "source": "https://github.com/thecodingmachine/safe/tree/v3.3.0" + }, + "funding": [ + { + "url": "https://github.com/OskarStark", + "type": "github" + }, + { + "url": "https://github.com/shish", + "type": "github" + }, + { + "url": "https://github.com/staabm", + "type": "github" + } + ], + "time": "2025-05-14T06:15:44+00:00" + }, { "name": "tijsverkoyen/css-to-inline-styles", "version": "v2.3.0", diff --git a/resources/schemas/v3.json b/resources/schemas/v3.json index cda80c6e..7e23f513 100644 --- a/resources/schemas/v3.json +++ b/resources/schemas/v3.json @@ -26,7 +26,7 @@ }, "default_account": { "type": "integer", - "minimum": 1 + "minimum": 0 }, "delimiter": { "type": "string", @@ -55,14 +55,27 @@ } }, "do_mapping": { - "type": "array", - "items": + "anyOf": [ { - "type": "boolean" + "type": "object" + }, + { + "type": "array", + "items": { + "type": "boolean" + } } + ] }, "mapping": { - "type": "object" + "anyOf": [ + { + "type": "object" + }, + { + "type": "array" + } + ] }, "duplicate_detection_method": { "type": "string", @@ -107,7 +120,14 @@ "type": "boolean" }, "accounts": { - "type": "object" + "anyOf": [ + { + "type": "object" + }, + { + "type": "array" + } + ] }, "date_range": { "type": "string" @@ -131,7 +151,14 @@ "type": "string" }, "nordigen_requisitions": { - "type": "object" + "anyOf": [ + { + "type": "object" + }, + { + "type": "array" + } + ] }, "conversion": { "type": "boolean"