kicad/common/settings/json_settings.cpp

/*
 * This program source code file is part of KiCad, a free EDA CAD application.
 *
 * Copyright (C) 2020 Jon Evans <jon@craftyjon.com>
 * Copyright (C) 2020-2022 KiCad Developers, see AUTHORS.txt for contributors.
 *
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <algorithm>
#include <fstream>
#include <iomanip>
#include <utility>
#include <sstream>

#include <locale_io.h>
#include <gal/color4d.h>
#include <settings/json_settings.h>
#include <settings/json_settings_internals.h>
#include <settings/nested_settings.h>
#include <settings/parameters.h>
#include <wx/config.h>
#include <wx/debug.h>
#include <wx/fileconf.h>
#include <wx/filename.h>
#include <wx/log.h>
#include <wx/stdstream.h>
#include <wx/wfstream.h>

const wxChar* const traceSettings = wxT( "KICAD_SETTINGS" );


nlohmann::json::json_pointer JSON_SETTINGS_INTERNALS::PointerFromString( std::string aPath )
{
    std::replace( aPath.begin(), aPath.end(), '.', '/' );
    aPath.insert( 0, "/" );

    nlohmann::json::json_pointer p;

    try
    {
        p = nlohmann::json::json_pointer( aPath );
    }
    catch( ... )
    {
        wxASSERT_MSG( false, wxT( "Invalid pointer path in PointerFromString!" ) );
    }

    return p;
}


JSON_SETTINGS::JSON_SETTINGS( const wxString& aFilename, SETTINGS_LOC aLocation,
                              int aSchemaVersion, bool aCreateIfMissing, bool aCreateIfDefault,
                              bool aWriteFile ) :
        m_filename( aFilename ),
        m_legacy_filename( "" ),
        m_location( aLocation ),
        m_createIfMissing( aCreateIfMissing ),
        m_createIfDefault( aCreateIfDefault ),
        m_writeFile( aWriteFile ),
        m_deleteLegacyAfterMigration( true ),
        m_resetParamsIfMissing( true ),
        m_schemaVersion( aSchemaVersion ),
        m_manager( nullptr )
{
    m_internals = std::make_unique<JSON_SETTINGS_INTERNALS>();

    try
    {
        m_internals->SetFromString( "meta.filename", GetFullFilename() );
    }
    catch( ... )
    {
        wxLogTrace( traceSettings, wxT( "Error: Could not create filename field for %s" ),
                    GetFullFilename() );
    }


    m_params.emplace_back(
            new PARAM<int>( "meta.version", &m_schemaVersion, m_schemaVersion, true ) );
}


JSON_SETTINGS::~JSON_SETTINGS()
{
    for( auto param: m_params )
        delete param;

    m_params.clear();
}


wxString JSON_SETTINGS::GetFullFilename() const
{
    return wxString( m_filename + "." + getFileExt() );
}


nlohmann::json& JSON_SETTINGS::At( const std::string& aPath )
{
    return m_internals->At( aPath );
}


bool JSON_SETTINGS::Contains( const std::string& aPath ) const
{
    return m_internals->contains( JSON_SETTINGS_INTERNALS::PointerFromString( aPath ) );
}


JSON_SETTINGS_INTERNALS* JSON_SETTINGS::Internals()
{
    return m_internals.get();
}


void JSON_SETTINGS::Load()
{
    for( auto param : m_params )
    {
        try
        {
            param->Load( this, m_resetParamsIfMissing );
        }
        catch( ... )
        {
            // Skip unreadable parameters in file
            wxLogTrace( traceSettings, wxT( "param '%s' load err" ), param->GetJsonPath().c_str() );
        }
    }
}


bool JSON_SETTINGS::LoadFromFile( const wxString& aDirectory )
{
    // First, load all params to default values
    m_internals->clear();
    Load();

    bool success         = true;
    bool migrated        = false;
    bool legacy_migrated = false;

    LOCALE_IO locale;

    auto migrateFromLegacy = [&] ( wxFileName& aPath ) {
        // Backup and restore during migration so that the original can be mutated if convenient
        bool backed_up = false;
        wxFileName temp;

        if( aPath.IsDirWritable() )
        {
            temp.AssignTempFileName( aPath.GetFullPath() );

            if( !wxCopyFile( aPath.GetFullPath(), temp.GetFullPath() ) )
            {
                wxLogTrace( traceSettings, wxT( "%s: could not create temp file for migration" ),
                        GetFullFilename() );
            }
            else
                backed_up = true;
        }

        // Silence popups if legacy file is read-only
        wxLogNull doNotLog;

        wxConfigBase::DontCreateOnDemand();
        auto cfg = std::make_unique<wxFileConfig>( wxT( "" ), wxT( "" ), aPath.GetFullPath() );

        // If migrate fails or is not implemented, fall back to built-in defaults that were
        // already loaded above
        if( !MigrateFromLegacy( cfg.get() ) )
        {
            success = false;
            wxLogTrace( traceSettings,
                        wxT( "%s: migrated; not all settings were found in legacy file" ),
                        GetFullFilename() );
        }
        else
        {
            success = true;
            wxLogTrace( traceSettings, wxT( "%s: migrated from legacy format" ), GetFullFilename() );
        }

        if( backed_up )
        {
            cfg.reset();

            if( !wxCopyFile( temp.GetFullPath(), aPath.GetFullPath() ) )
            {
                wxLogTrace( traceSettings,
                            wxT( "migrate; copy temp file %s to %s failed" ),
                            temp.GetFullPath(), aPath.GetFullPath() );
            }

            if( !wxRemoveFile( temp.GetFullPath() ) )
            {
                wxLogTrace( traceSettings,
                            wxT( "migrate; failed to remove temp file %s" ),
                            temp.GetFullPath() );
            }
         }

        // Either way, we want to clean up the old file afterwards
        legacy_migrated = true;
    };

    wxFileName path;

    if( aDirectory.empty() )
    {
        path.Assign( m_filename );
        path.SetExt( getFileExt() );
    }
    else
    {
        wxString dir( aDirectory );
        path.Assign( dir, m_filename, getFileExt() );
    }

    if( !path.Exists() )
    {
        // Case 1: legacy migration, no .json extension yet
        path.SetExt( getLegacyFileExt() );

        if( path.Exists() )
        {
            migrateFromLegacy( path );
        }
        // Case 2: legacy filename is different from new one
        else if( !m_legacy_filename.empty() )
        {
            path.SetName( m_legacy_filename );

            if( path.Exists() )
                migrateFromLegacy( path );
        }
        else
        {
            success = false;
        }
    }
    else
    {
        if( !path.IsFileWritable() )
            m_writeFile = false;

        try
        {
            wxFFileInputStream fp( path.GetFullPath(), wxT( "rt" ) );
            wxStdInputStream fstream( fp );

            if( fp.IsOk() )
            {
                *static_cast<nlohmann::json*>( m_internals.get() ) =
                        nlohmann::json::parse( fstream, nullptr,
                                               /* allow_exceptions = */ true,
                                               /* ignore_comments  = */ true );

                // If parse succeeds, check if schema migration is required
                int filever = -1;

                try
                {
                    filever = m_internals->Get<int>( "meta.version" );
                }
                catch( ... )
                {
                    wxLogTrace( traceSettings, wxT( "%s: file version could not be read!" ),
                                GetFullFilename() );
                    success = false;
                }

                if( filever >= 0 && filever < m_schemaVersion )
                {
                    wxLogTrace( traceSettings, wxT( "%s: attempting migration from version %d to %d" ),
                                GetFullFilename(), filever, m_schemaVersion );

                    if( Migrate() )
                    {
                        migrated = true;
                    }
                    else
                    {
                        wxLogTrace( traceSettings, wxT( "%s: migration failed!" ), GetFullFilename() );
                    }
                }
                else if( filever > m_schemaVersion )
                {
                    wxLogTrace( traceSettings,
                                wxT( "%s: warning: file version %d is newer than latest (%d)" ),
                                GetFullFilename(), filever, m_schemaVersion );
                }
            }
            else
            {
                wxLogTrace( traceSettings, wxT( "%s exists but can't be opened for read" ),
                            GetFullFilename() );
            }
        }
        catch( nlohmann::json::parse_error& error )
        {
            success = false;
            wxLogTrace( traceSettings, wxT( "Json parse error reading %s: %s" ),
                        path.GetFullPath(), error.what() );
            wxLogTrace( traceSettings, wxT( "Attempting migration in case file is in legacy format" ) );
            migrateFromLegacy( path );
        }
    }

    // Now that we have new data in the JSON structure, load the params again
    Load();

    // And finally load any nested settings
    for( auto settings : m_nested_settings )
        settings->LoadFromFile();

    wxLogTrace( traceSettings, wxT( "Loaded <%s> with schema %d" ), GetFullFilename(), m_schemaVersion );

    // If we migrated, clean up the legacy file (with no extension)
    if( m_writeFile && ( legacy_migrated || migrated ) )
    {
        if( legacy_migrated && m_deleteLegacyAfterMigration && !wxRemoveFile( path.GetFullPath() ) )
        {
            wxLogTrace( traceSettings, wxT( "Warning: could not remove legacy file %s" ),
                        path.GetFullPath() );
        }

        // And write-out immediately so that we don't lose data if the program later crashes.
        if( m_deleteLegacyAfterMigration )
            SaveToFile( aDirectory, true );
    }

    return success;
}


bool JSON_SETTINGS::Store()
{
    bool modified = false;

    for( auto param : m_params )
    {
        modified |= !param->MatchesFile( this );
        param->Store( this );
    }

    return modified;
}


void JSON_SETTINGS::ResetToDefaults()
{
    for( auto param : m_params )
        param->SetDefault();
}


bool JSON_SETTINGS::SaveToFile( const wxString& aDirectory, bool aForce )
{
    if( !m_writeFile )
        return false;

    // Default PROJECT won't have a filename set
    if( m_filename.IsEmpty() )
        return false;

    wxFileName path;

    if( aDirectory.empty() )
    {
        path.Assign( m_filename );
        path.SetExt( getFileExt() );
    }
    else
    {
        wxString dir( aDirectory );
        path.Assign( dir, m_filename, getFileExt() );
    }

    if( !m_createIfMissing && !path.FileExists() )
    {
        wxLogTrace( traceSettings,
                wxT( "File for %s doesn't exist and m_createIfMissing == false; not saving" ),
                GetFullFilename() );
        return false;
    }

    // Ensure the path exists, and create it if not.
    if( !path.DirExists() && !path.Mkdir() )
    {
        wxLogTrace( traceSettings, wxT( "Warning: could not create path %s, can't save %s" ),
                    path.GetPath(), GetFullFilename() );
        return false;
    }

    if( ( path.FileExists() && !path.IsFileWritable() ) ||
        ( !path.FileExists() && !path.IsDirWritable() ) )
    {
        wxLogTrace( traceSettings, wxT( "File for %s is read-only; not saving" ), GetFullFilename() );
        return false;
    }

    bool modified = false;

    for( auto settings : m_nested_settings )
        modified |= settings->SaveToFile();

    modified |= Store();

    if( !modified && !aForce && path.FileExists() )
    {
        wxLogTrace( traceSettings, wxT( "%s contents not modified, skipping save" ), GetFullFilename() );
        return false;
    }
    else if( !modified && !aForce && !m_createIfDefault )
    {
        wxLogTrace( traceSettings,
                wxT( "%s contents still default and m_createIfDefault == false; not saving" ),
                    GetFullFilename() );
        return false;
    }

    wxLogTrace( traceSettings, wxT( "Saving %s" ), GetFullFilename() );

    LOCALE_IO dummy;
    bool success = true;

    try
    {
        std::stringstream buffer;
        buffer << std::setw( 2 ) << *m_internals << std::endl;

        wxFFileOutputStream fileStream( path.GetFullPath(), "wb" );

        if( !fileStream.IsOk()
                || !fileStream.WriteAll( buffer.str().c_str(), buffer.str().size() ) )
        {
            wxLogTrace( traceSettings, wxT( "Warning: could not save %s" ), GetFullFilename() );
            success = false;
        }
    }
    catch( nlohmann::json::exception& error )
    {
        wxLogTrace( traceSettings, wxT( "Catch error: could not save %s. Json error %s" ),
                    GetFullFilename(), error.what() );
        success = false;
    }
    catch( ... )
    {
        wxLogTrace( traceSettings, wxT( "Error: could not save %s." ) );
        success = false;
    }

    return success;
}


const std::string JSON_SETTINGS::FormatAsString() const
{
    LOCALE_IO dummy;

    std::stringstream buffer;
    buffer << std::setw( 2 ) << *m_internals << std::endl;

    return buffer.str();
}


bool JSON_SETTINGS::LoadFromRawFile( const wxString& aPath )
{
    try
    {
        wxFFileInputStream fp( aPath, wxT( "rt" ) );
        wxStdInputStream   fstream( fp );

        if( fp.IsOk() )
        {
            *static_cast<nlohmann::json*>( m_internals.get() ) =
                    nlohmann::json::parse( fstream, nullptr,
                                           /* allow_exceptions = */ true,
                                           /* ignore_comments  = */ true );
        }
        else
        {
            return false;
        }
    }
    catch( nlohmann::json::parse_error& error )
    {
        wxLogTrace( traceSettings, wxT( "Json parse error reading %s: %s" ), aPath, error.what() );

        return false;
    }

    // Now that we have new data in the JSON structure, load the params again
    Load();
    return true;
}


std::optional<nlohmann::json> JSON_SETTINGS::GetJson( const std::string& aPath ) const
{
    nlohmann::json::json_pointer ptr = m_internals->PointerFromString( aPath );

    if( m_internals->contains( ptr ) )
    {
        try
        {
            return std::optional<nlohmann::json>{ m_internals->at( ptr ) };
        }
        catch( ... )
        {
        }
    }

    return std::optional<nlohmann::json>{};
}


template<typename ValueType>
std::optional<ValueType> JSON_SETTINGS::Get( const std::string& aPath ) const
{
    if( std::optional<nlohmann::json> ret = GetJson( aPath ) )
    {
        try
        {
            return ret->get<ValueType>();
        }
        catch( ... )
        {
        }
    }

    return std::nullopt;
}


// Instantiate all required templates here to allow reducing scope of json.hpp
template std::optional<bool> JSON_SETTINGS::Get<bool>( const std::string& aPath ) const;
template std::optional<double> JSON_SETTINGS::Get<double>( const std::string& aPath ) const;
template std::optional<float> JSON_SETTINGS::Get<float>( const std::string& aPath ) const;
template std::optional<int> JSON_SETTINGS::Get<int>( const std::string& aPath ) const;
template std::optional<unsigned int> JSON_SETTINGS::Get<unsigned int>( const std::string& aPath ) const;
template std::optional<unsigned long long> JSON_SETTINGS::Get<unsigned long long>( const std::string& aPath ) const;
template std::optional<std::string> JSON_SETTINGS::Get<std::string>( const std::string& aPath ) const;
template std::optional<nlohmann::json> JSON_SETTINGS::Get<nlohmann::json>( const std::string& aPath ) const;
template std::optional<KIGFX::COLOR4D> JSON_SETTINGS::Get<KIGFX::COLOR4D>( const std::string& aPath ) const;


template<typename ValueType>
void JSON_SETTINGS::Set( const std::string& aPath, ValueType aVal )
{
    m_internals->SetFromString( aPath, aVal );
}


// Instantiate all required templates here to allow reducing scope of json.hpp
template void JSON_SETTINGS::Set<bool>( const std::string& aPath, bool aValue );
template void JSON_SETTINGS::Set<double>( const std::string& aPath, double aValue );
template void JSON_SETTINGS::Set<float>( const std::string& aPath, float aValue );
template void JSON_SETTINGS::Set<int>( const std::string& aPath, int aValue );
template void JSON_SETTINGS::Set<unsigned int>( const std::string& aPath, unsigned int aValue );
template void JSON_SETTINGS::Set<unsigned long long>( const std::string& aPath, unsigned long long aValue );
template void JSON_SETTINGS::Set<const char*>( const std::string& aPath, const char* aValue );
template void JSON_SETTINGS::Set<std::string>( const std::string& aPath, std::string aValue );
template void JSON_SETTINGS::Set<nlohmann::json>( const std::string& aPath, nlohmann::json aValue );
template void JSON_SETTINGS::Set<KIGFX::COLOR4D>( const std::string& aPath, KIGFX::COLOR4D aValue );


void JSON_SETTINGS::registerMigration( int aOldSchemaVersion, int aNewSchemaVersion,
                                       std::function<bool()> aMigrator )
{
    wxASSERT( aNewSchemaVersion > aOldSchemaVersion );
    wxASSERT( aNewSchemaVersion <= m_schemaVersion );
    m_migrators[aOldSchemaVersion] = std::make_pair( aNewSchemaVersion, aMigrator );
}


bool JSON_SETTINGS::Migrate()
{
    int filever = m_internals->Get<int>( "meta.version" );

    while( filever < m_schemaVersion )
    {
        if( !m_migrators.count( filever ) )
        {
            wxLogTrace( traceSettings, wxT( "Migrator missing for %s version %d!" ),
                        typeid( *this ).name(), filever );
            return false;
        }

        std::pair<int, std::function<bool()>> pair = m_migrators.at( filever );

        if( pair.second() )
        {
            wxLogTrace( traceSettings, wxT( "Migrated %s from %d to %d" ), typeid( *this ).name(),
                        filever, pair.first );
            filever = pair.first;
            m_internals->At( "meta.version" ) = filever;
        }
        else
        {
            wxLogTrace( traceSettings, wxT( "Migration failed for %s from %d to %d" ),
                        typeid( *this ).name(), filever, pair.first );
            return false;
        }
    }

    return true;
}


bool JSON_SETTINGS::MigrateFromLegacy( wxConfigBase* aLegacyConfig )
{
    wxLogTrace( traceSettings,
            wxT( "MigrateFromLegacy() not implemented for %s" ), typeid( *this ).name() );
    return false;
}


bool JSON_SETTINGS::SetIfPresent( const nlohmann::json& aObj, const std::string& aPath,
                                  wxString& aTarget )
{
    nlohmann::json::json_pointer ptr = JSON_SETTINGS_INTERNALS::PointerFromString( aPath );

    if( aObj.contains( ptr ) && aObj.at( ptr ).is_string() )
    {
        aTarget = aObj.at( ptr ).get<wxString>();
        return true;
    }

    return false;
}


bool JSON_SETTINGS::SetIfPresent( const nlohmann::json& aObj, const std::string& aPath,
                                  bool& aTarget )
{
    nlohmann::json::json_pointer ptr = JSON_SETTINGS_INTERNALS::PointerFromString( aPath );

    if( aObj.contains( ptr ) && aObj.at( ptr ).is_boolean() )
    {
        aTarget = aObj.at( ptr ).get<bool>();
        return true;
    }

    return false;
}


bool JSON_SETTINGS::SetIfPresent( const nlohmann::json& aObj, const std::string& aPath,
                                  int& aTarget )
{
    nlohmann::json::json_pointer ptr = JSON_SETTINGS_INTERNALS::PointerFromString( aPath );

    if( aObj.contains( ptr ) && aObj.at( ptr ).is_number_integer() )
    {
        aTarget = aObj.at( ptr ).get<int>();
        return true;
    }

    return false;
}


bool JSON_SETTINGS::SetIfPresent( const nlohmann::json& aObj, const std::string& aPath,
                                  unsigned int& aTarget )
{
    nlohmann::json::json_pointer ptr = JSON_SETTINGS_INTERNALS::PointerFromString( aPath );

    if( aObj.contains( ptr ) && aObj.at( ptr ).is_number_unsigned() )
    {
        aTarget = aObj.at( ptr ).get<unsigned int>();
        return true;
    }

    return false;
}


template<typename ValueType>
bool JSON_SETTINGS::fromLegacy( wxConfigBase* aConfig, const std::string& aKey,
                             const std::string& aDest )
{
    ValueType val;

    if( aConfig->Read( aKey, &val ) )
    {
        try
        {
            ( *m_internals )[aDest] = val;
        }
        catch( ... )
        {
            wxASSERT_MSG( false, wxT( "Could not write value in fromLegacy!" ) );
            return false;
        }

        return true;
    }

    return false;
}


// Explicitly declare these because we only support a few types anyway, and it means we can keep
// wxConfig detail out of the header file
template bool JSON_SETTINGS::fromLegacy<int>( wxConfigBase*, const std::string&,
                                              const std::string& );

template bool JSON_SETTINGS::fromLegacy<double>( wxConfigBase*, const std::string&,
                                                 const std::string& );

template bool JSON_SETTINGS::fromLegacy<bool>( wxConfigBase*, const std::string&,
                                               const std::string& );


bool JSON_SETTINGS::fromLegacyString( wxConfigBase* aConfig, const std::string& aKey,
                                      const std::string& aDest )
{
    wxString str;

    if( aConfig->Read( aKey, &str ) )
    {
        try
        {
            ( *m_internals )[aDest] = str.ToUTF8();
        }
        catch( ... )
        {
            wxASSERT_MSG( false, wxT( "Could not write value in fromLegacyString!" ) );
            return false;
        }

        return true;
    }

    return false;
}


bool JSON_SETTINGS::fromLegacyColor( wxConfigBase* aConfig, const std::string& aKey,
    const std::string& aDest )
{
    wxString str;

    if( aConfig->Read( aKey, &str ) )
    {
        KIGFX::COLOR4D color;
        color.SetFromWxString( str );

        try
        {
            nlohmann::json js = nlohmann::json::array( { color.r, color.g, color.b, color.a } );
            ( *m_internals )[aDest] = js;
        }
        catch( ... )
        {
            wxASSERT_MSG( false, wxT( "Could not write value in fromLegacyColor!" ) );
            return false;
        }

        return true;
    }

    return false;
}


void JSON_SETTINGS::AddNestedSettings( NESTED_SETTINGS* aSettings )
{
    wxLogTrace( traceSettings, wxT( "AddNestedSettings %s" ), aSettings->GetFilename() );
    m_nested_settings.push_back( aSettings );
}


void JSON_SETTINGS::ReleaseNestedSettings( NESTED_SETTINGS* aSettings )
{
    if( !aSettings || !m_manager )
        return;

    auto it = std::find_if( m_nested_settings.begin(), m_nested_settings.end(),
                            [&aSettings]( const JSON_SETTINGS* aPtr ) {
                              return aPtr == aSettings;
                            } );

    if( it != m_nested_settings.end() )
    {
        wxLogTrace( traceSettings, wxT( "Flush and release %s" ), ( *it )->GetFilename() );
        ( *it )->SaveToFile();
        m_nested_settings.erase( it );
    }

    aSettings->SetParent( nullptr );
}


// Specializations to allow conversion between wxString and std::string via JSON_SETTINGS API

template<> std::optional<wxString> JSON_SETTINGS::Get( const std::string& aPath ) const
{
    if( std::optional<nlohmann::json> opt_json = GetJson( aPath ) )
        return wxString( opt_json->get<std::string>().c_str(), wxConvUTF8 );

    return std::nullopt;
}


template<> void JSON_SETTINGS::Set<wxString>( const std::string& aPath, wxString aVal )
{
    ( *m_internals )[aPath] = aVal.ToUTF8();
}

// Specializations to allow directly reading/writing wxStrings from JSON

void to_json( nlohmann::json& aJson, const wxString& aString )
{
    aJson = aString.ToUTF8();
}


void from_json( const nlohmann::json& aJson, wxString& aString )
{
    aString = wxString( aJson.get<std::string>().c_str(), wxConvUTF8 );
}


template<typename ResultType>
ResultType JSON_SETTINGS::fetchOrDefault( const nlohmann::json& aJson, const std::string& aKey,
                                          ResultType aDefault )
{
    ResultType ret = aDefault;

    try
    {
        if( aJson.contains( aKey ) )
            ret = aJson.at( aKey ).get<ResultType>();
    }
    catch( ... )
    {
    }

    return ret;
}


template std::string JSON_SETTINGS::fetchOrDefault( const nlohmann::json& aJson,
                                                    const std::string& aKey, std::string aDefault );

template bool JSON_SETTINGS::fetchOrDefault( const nlohmann::json& aJson, const std::string& aKey,
                                             bool aDefault );