Browse Source

Add commands to verify JSON and make the schema better.

pull/972/head
James Cole 4 weeks ago
parent
commit
e9152a4f8d
  1. 45
      app/Console/Commands/ValidateJsonFile.php
  2. 56
      app/Console/Commands/ValidateJsonFiles.php
  3. 15
      app/Console/VerifyJSON.php
  4. 8
      app/Http/Controllers/Import/DownloadController.php
  5. 12
      app/Http/Controllers/Import/UploadController.php
  6. 3
      composer.json
  7. 141
      composer.lock
  8. 33
      resources/schemas/v3.json

45
app/Console/Commands/ValidateJsonFile.php

@ -0,0 +1,45 @@
<?php
namespace App\Console\Commands;
use App\Console\VerifyJSON;
use Illuminate\Console\Command;
use Symfony\Component\Console\Command\Command as CommandAlias;
class ValidateJsonFile extends Command
{
use VerifyJSON;
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'import:validate-json {file : The JSON file to validate}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Checks if a JSON file is valid according to the v3 import configuration file standard.';
/**
* Execute the console command.
*/
public function handle(): int
{
$file = (string)$this->argument('file');
if (!is_file($file) || !is_readable($file)) {
$this->error(sprintf('File %s does not exist or is not readable.', $file));
return CommandAlias::FAILURE;
}
$result = $this->verifyJSON($file);
if(false === $result) {
$this->error('File is not valid JSON.');
return CommandAlias::FAILURE;
}
$this->info('File is valid JSON.');
return CommandAlias::SUCCESS;
}
}

56
app/Console/Commands/ValidateJsonFiles.php

@ -0,0 +1,56 @@
<?php
namespace App\Console\Commands;
use App\Console\VerifyJSON;
use Illuminate\Console\Command;
use Symfony\Component\Console\Command\Command as CommandAlias;
class ValidateJsonFiles extends Command
{
use VerifyJSON;
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'import:validate-json-directory {directory : The directory with JSON files to validate}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Recursively validate all JSON files in a directory. Stops after 100 files.';
/**
* Execute the console command.
*/
public function handle(): int
{
$directory = (string)$this->argument('directory');
if (!is_dir($directory) || !is_readable($directory)) {
$this->error(sprintf('Cannot read directory %s.', $directory));
return CommandAlias::FAILURE;
}
// check each file in the directory and see if it needs action.
// collect recursively:
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($directory, \RecursiveDirectoryIterator::SKIP_DOTS));
$Regex = new \RegexIterator($it, '/^.+\.json$/i', \RecursiveRegexIterator::GET_MATCH);
$fullPaths = [];
foreach ($Regex as $item) {
$path = $item[0];
$fullPaths[] = $path;
}
foreach ($fullPaths as $file) {
$result = $this->verifyJSON($file);
if (false === $result) {
$this->error(sprintf('File "%s" is not valid JSON.', $file));
return CommandAlias::FAILURE;
}
$this->info(sprintf('File "%s" is valid JSON.', $file));
}
return CommandAlias::SUCCESS;
}
}

15
app/Console/VerifyJSON.php

@ -35,6 +35,8 @@ use Swaggest\JsonSchema\Schema;
*/
trait VerifyJSON
{
protected string $errorMessage = '';
private function verifyJSON(string $file): bool
{
// basic check on the JSON.
@ -43,14 +45,18 @@ trait VerifyJSON
try {
$config = json_decode($json, null, 512, JSON_THROW_ON_ERROR);
} catch (JsonException $e) {
Log::error(sprintf('The importer can\'t import: could not decode the JSON in the config file: %s', $e->getMessage()));
$message = sprintf('The importer can\'t import: could not decode the JSON in the config file: %s', $e->getMessage());
Log::error($message);
$this->errorMessage = $message;
return false;
}
// validate JSON schema.
$schemaFile = resource_path('schemas/v3.json');
if (!file_exists($schemaFile)) {
Log::error(sprintf('The schema file "%s" does not exist.', $schemaFile));
$message = sprintf('The schema file "%s" does not exist.', $schemaFile);
Log::error($message);
$this->errorMessage = $message;
return false;
}
@ -59,7 +65,10 @@ trait VerifyJSON
try {
Schema::import($schema)->in($config);
} catch (Exception|\Exception $e) {
Log::error(sprintf('Configuration file "%s" does not adhere to the v3 schema: %s', $file, $e->getMessage()));
$message = sprintf('Configuration file "%s" does not adhere to the v3 schema: %s', $file, $e->getMessage());
Log::error($message);
$this->errorMessage = $message;
return false;
}

8
app/Http/Controllers/Import/DownloadController.php

@ -53,6 +53,14 @@ class DownloadController extends Controller
if(is_array($array['mapping']) && 0 === count($array['mapping'])) {
$array['mapping'] = new \stdClass();
}
// same for "accounts"
if(is_array($array['accounts']) && 0 === count($array['accounts'])) {
$array['accounts'] = new \stdClass();
}
// same for "accounts"
if(is_array($array['nordigen_requisitions']) && 0 === count($array['nordigen_requisitions'])) {
$array['nordigen_requisitions'] = new \stdClass();
}
$result = json_encode($array, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR);

12
app/Http/Controllers/Import/UploadController.php

@ -25,6 +25,7 @@ declare(strict_types=1);
namespace App\Http\Controllers\Import;
use App\Console\VerifyJSON;
use App\Exceptions\ImporterErrorException;
use App\Http\Controllers\Controller;
use App\Http\Middleware\UploadControllerMiddleware;
@ -53,6 +54,7 @@ use Storage;
class UploadController extends Controller
{
use RestoresConfiguration;
use VerifyJSON;
private string $configFileName;
private string $contentType;
@ -247,7 +249,15 @@ class UploadController extends Controller
// upload the file to a temp directory and use it from there.
if (0 === $errorNumber) {
Log::debug('Config file uploaded.');
$this->configFileName = StorageService::storeContent((string)file_get_contents($file->getPathname()));
$path = $file->getPathname();
$validation = $this->verifyJSON($path);
if (false === $validation) {
$errors->add('config_file', $this->errorMessage);
return $errors;
}
$content = (string)file_get_contents($path);
$this->configFileName = StorageService::storeContent($content);
session()->put(Constants::UPLOAD_CONFIG_FILE, $this->configFileName);

3
composer.json

@ -83,7 +83,8 @@
"spatie/enum": "^3.10",
"swaggest/json-schema": "^0.12.43",
"symfony/http-client": "^7.3",
"symfony/mailgun-mailer": "^7.3"
"symfony/mailgun-mailer": "^7.3",
"thecodingmachine/safe": "^3.3"
},
"require-dev": {
"barryvdh/laravel-debugbar": "^3.16",

141
composer.lock

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "90550ddfcbcb4c8043368ef25abd0c9d",
"content-hash": "b7cf2032b370512926f502bdd4038b70",
"packages": [
{
"name": "brick/math",
@ -6316,6 +6316,145 @@
],
"time": "2025-08-13T11:49:31+00:00"
},
{
"name": "thecodingmachine/safe",
"version": "v3.3.0",
"source": {
"type": "git",
"url": "https://github.com/thecodingmachine/safe.git",
"reference": "2cdd579eeaa2e78e51c7509b50cc9fb89a956236"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/thecodingmachine/safe/zipball/2cdd579eeaa2e78e51c7509b50cc9fb89a956236",
"reference": "2cdd579eeaa2e78e51c7509b50cc9fb89a956236",
"shasum": ""
},
"require": {
"php": "^8.1"
},
"require-dev": {
"php-parallel-lint/php-parallel-lint": "^1.4",
"phpstan/phpstan": "^2",
"phpunit/phpunit": "^10",
"squizlabs/php_codesniffer": "^3.2"
},
"type": "library",
"autoload": {
"files": [
"lib/special_cases.php",
"generated/apache.php",
"generated/apcu.php",
"generated/array.php",
"generated/bzip2.php",
"generated/calendar.php",
"generated/classobj.php",
"generated/com.php",
"generated/cubrid.php",
"generated/curl.php",
"generated/datetime.php",
"generated/dir.php",
"generated/eio.php",
"generated/errorfunc.php",
"generated/exec.php",
"generated/fileinfo.php",
"generated/filesystem.php",
"generated/filter.php",
"generated/fpm.php",
"generated/ftp.php",
"generated/funchand.php",
"generated/gettext.php",
"generated/gmp.php",
"generated/gnupg.php",
"generated/hash.php",
"generated/ibase.php",
"generated/ibmDb2.php",
"generated/iconv.php",
"generated/image.php",
"generated/imap.php",
"generated/info.php",
"generated/inotify.php",
"generated/json.php",
"generated/ldap.php",
"generated/libxml.php",
"generated/lzf.php",
"generated/mailparse.php",
"generated/mbstring.php",
"generated/misc.php",
"generated/mysql.php",
"generated/mysqli.php",
"generated/network.php",
"generated/oci8.php",
"generated/opcache.php",
"generated/openssl.php",
"generated/outcontrol.php",
"generated/pcntl.php",
"generated/pcre.php",
"generated/pgsql.php",
"generated/posix.php",
"generated/ps.php",
"generated/pspell.php",
"generated/readline.php",
"generated/rnp.php",
"generated/rpminfo.php",
"generated/rrd.php",
"generated/sem.php",
"generated/session.php",
"generated/shmop.php",
"generated/sockets.php",
"generated/sodium.php",
"generated/solr.php",
"generated/spl.php",
"generated/sqlsrv.php",
"generated/ssdeep.php",
"generated/ssh2.php",
"generated/stream.php",
"generated/strings.php",
"generated/swoole.php",
"generated/uodbc.php",
"generated/uopz.php",
"generated/url.php",
"generated/var.php",
"generated/xdiff.php",
"generated/xml.php",
"generated/xmlrpc.php",
"generated/yaml.php",
"generated/yaz.php",
"generated/zip.php",
"generated/zlib.php"
],
"classmap": [
"lib/DateTime.php",
"lib/DateTimeImmutable.php",
"lib/Exceptions/",
"generated/Exceptions/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"description": "PHP core functions that throw exceptions instead of returning FALSE on error",
"support": {
"issues": "https://github.com/thecodingmachine/safe/issues",
"source": "https://github.com/thecodingmachine/safe/tree/v3.3.0"
},
"funding": [
{
"url": "https://github.com/OskarStark",
"type": "github"
},
{
"url": "https://github.com/shish",
"type": "github"
},
{
"url": "https://github.com/staabm",
"type": "github"
}
],
"time": "2025-05-14T06:15:44+00:00"
},
{
"name": "tijsverkoyen/css-to-inline-styles",
"version": "v2.3.0",

33
resources/schemas/v3.json

@ -26,7 +26,7 @@
},
"default_account": {
"type": "integer",
"minimum": 1
"minimum": 0
},
"delimiter": {
"type": "string",
@ -55,15 +55,28 @@
}
},
"do_mapping": {
"type": "array",
"items":
"anyOf": [
{
"type": "object"
},
{
"type": "array",
"items": {
"type": "boolean"
}
}
]
},
"mapping": {
"anyOf": [
{
"type": "object"
},
{
"type": "array"
}
]
},
"duplicate_detection_method": {
"type": "string",
"enum": [
@ -107,8 +120,15 @@
"type": "boolean"
},
"accounts": {
"anyOf": [
{
"type": "object"
},
{
"type": "array"
}
]
},
"date_range": {
"type": "string"
},
@ -131,8 +151,15 @@
"type": "string"
},
"nordigen_requisitions": {
"anyOf": [
{
"type": "object"
},
{
"type": "array"
}
]
},
"conversion": {
"type": "boolean"
}

Loading…
Cancel
Save