classdef TrainingOptionsLBFGS < nnet.cnn.TrainingOptions
    % TrainingOptionsLBFGS   Training options for L-BFGS
    %
    %   This class holds the training options for L-BFGS
    %
    %   TrainingOptionsLBFGS properties:
    %       HistorySize                  - Maximum history size for the
    %                                      Hessian approximation
    %       InitialInverseHessianFactor  - Scalar used to construct the
    %                                      inverse Hessian approximation at
    %                                      the first iteration
    %       LineSearchMethod             - Line search method
    %       MaxNumLineSearchIterations   - Maximum number of line search
    %                                      iterations
    %       GradientTolerance            - Relative gradient tolerance
    %       StepTolerance                - Step size tolerance
    %       L2Regularization             - Factor for L2 regularization.
    %       GradientThresholdMethod      - Method for gradient thresholding
    %       GradientThreshold            - Gradient threshold
    %       MaxIterations                - Maximum number of iterations
    %       Verbose                      - Flag for printing information to
    %                                      the command window
    %       VerboseFrequency             - Number of iterations between
    %                                      printing verbose output to the
    %                                      command window
    %       ValidationData               - Data to use for validation during
    %                                      training
    %       ValidationFrequency          - Number of iterations between
    %                                      evaluations of validation
    %                                      metrics
    %       ValidationPatience           - Number of consecutive times that
    %                                      the validation loss must be
    %                                      greater than or equal to the
    %                                      smallest validation loss to stop
    %                                      training
    %       CheckpointPath               - Path where checkpoint networks
    %                                      are saved
    %       CheckpointFrequency          - Frequency of saving checkpoint
    %                                      networks
    %       ExecutionEnvironment         - Hardware to use for training
    %                                      network
    %       OutputFcn                    - Function to evaluate at each
    %                                      iteration
    %       Metrics                      - Metrics to calculate at each iteration
    %       Plots                        - Plots to display during training
    %       SequenceLength               - Sequence length of training data
    %       SequencePaddingValue         - Value used to pad mini-batches
    %                                      along the sequence dimension
    %       SequencePaddingDirection     - Direction of padding or
    %                                      truncation
    %       ResetInputNormalization      - Flag to reset input normalization
    %       BatchNormalizationStatistics - Method to calculate statistics
    %                                      in batch normalization layers
    %       OutputNetwork                - Network to return when training
    %                                      ends
    %
    %   Example:
    %       % Specify training options for training with L-BFGS:
    %       %   - Determine the learn rate using a line search method
    %       %     that satisfies the strong Wolfe conditions.
    %       %   - Display the training progress in a plot.
    %
    %       opts = trainingOptions("lbfgs", ...
    %           LineSearchMethod="strong-wolfe", ...
    %           Plots="training-progress");
    %
    %   See also trainingOptions, trainnet.
    
    % Copyright 2023-2024 The MathWorks, Inc.
    
    properties(Constant=true,Access=protected)
        SolverName = 'lbfgs'

        % Version   Number to identify the current version of this object
        %   This is used to ensure that objects from older versions are
        %   loaded correctly.
        Version = 2
    end

    properties
        % MaxIterations   Maximum number of iterations
        %   The maximum number of iterations that will be used for training.
        %   Training will stop once this number of iterations has passed.
        MaxIterations {iValidatePropertyValue(MaxIterations, 'MaxIterations')} = ...
                                                                    iGetDefaultValue('MaxIterations')

        % HistorySize   Maximum history size for the Hessian approximation
        %   A positive integer specifying the maximum number of gradient
        %   histories to store for calculating the Hessian approximation.
        HistorySize {iValidatePropertyValue(HistorySize, 'HistorySize')} = ...
                                                                    iGetDefaultValue('HistorySize')

        % InitialInverseHessianFactor   Inverse Hessian factor at first iteration
        %   A positive real scalar specifying the value to approximate the
        % inverse Hessian when applying the first iteration of L-BFGS.
        InitialInverseHessianFactor {iValidatePropertyValue(InitialInverseHessianFactor,'InitialInverseHessianFactor')} = ...
                                                iGetDefaultValue('InitialInverseHessianFactor')

        % LineSearchMethod   Line search method
        %   Method used for line search. Options are "backtracking", 
        % "weak-wolfe" (default) and "strong-wolfe".
        LineSearchMethod {iValidatePropertyValue(LineSearchMethod, 'LineSearchMethod')} = iGetDefaultValue('LineSearchMethod')

        % MaxNumLineSearchIterations   Maximum number of line search iterations
        %   A positive integer specifying the maximum number of line search
        %   iterations to perform when determining the learning rate.
        MaxNumLineSearchIterations {iValidatePropertyValue(MaxNumLineSearchIterations, 'MaxNumLineSearchIterations')} = iGetDefaultValue('MaxNumLineSearchIterations')

        % GradientTolerance   Relative gradient tolerance
        %   A positive real scalar specifying the relative gradient
        %   tolerance. Training stops when the relative gradient is less
        %   than or equal to this value.
        GradientTolerance {iValidatePropertyValue(GradientTolerance, 'GradientTolerance')} = iGetDefaultValue('GradientTolerance')

        % StepTolerance   Step size tolerance
        %   A positive real scalar specifying the step size tolerance.
        %   Training stops when the step taken is less than than or equal
        %   to this value.
        StepTolerance {iValidatePropertyValue(StepTolerance, 'StepTolerance')} = iGetDefaultValue('StepTolerance')

        % SequenceLength   Determines the strategy used to create
        % mini-batches of sequence data. Options are "longest" (default)
        % and "shortest".
        SequenceLength

        % CheckpointFrequency   Frequency for saving checkpoint networks
        CheckpointFrequency
    end

    % Certain properties only have a single option for full-batch training.
    % These properties may still be referred to during training, so provide
    % these properties as constant, hidden properties.
    properties(Hidden, Constant)
        Shuffle = 'never'

        CheckpointFrequencyUnit = 'iteration'

        DispatchInBackground = false

        MiniBatchSize = 'all'

        PreprocessingEnvironment = 'serial'
    end

    methods
        function this = TrainingOptionsLBFGS(varargin, args)
            arguments (Repeating)
                varargin
            end
            arguments
                args.Definition = nnet.internal.cnn.options.TrainingOptionsDefinitionLBFGS
            end

            this = this@nnet.cnn.TrainingOptions(args.Definition,varargin{:});          
        end

        function out = saveobj(this)
            out = saveobj@nnet.cnn.TrainingOptions(this);
            out.MaxIterations = this.MaxIterations;
            out.HistorySize = this.HistorySize;
            out.InitialInverseHessianFactor = this.InitialInverseHessianFactor;
            out.LineSearchMethod = this.LineSearchMethod;
            out.MaxNumLineSearchIterations = this.MaxNumLineSearchIterations;
            out.GradientTolerance = this.GradientTolerance;
            out.StepTolerance = this.StepTolerance;
        end
    end

    methods(Access=protected)
        function warnIfCheckpointFrequencyDoesNotMatchCheckpointPath(this)
            if isempty(this.CheckpointPath) ...
                    && ~(this.CheckpointFrequency == iGetDefaultValue('CheckpointFrequency'))
                % Disable backtrace so that warning does not produce stack
                % trace if backtrace is on, then restore backtrace to its
                % original state when exiting the method
                backtraceState = warning('query', 'backtrace');
                restoreBacktraceState = onCleanup(@()warning(backtraceState));
                warning off backtrace
                warning(message('nnet_cnn:TrainingOptionsSGDM:CheckpointFreqAndPathMismatch'));
            end
        end
    end

    methods(Static)
        function this = loadobj(in)
            if iTrainingOptionsAreFrom2023b(in)
                versionNumber24a = 2;
                in = iUpgradeTrainingOptionsFrom2023bTo2024a(in, versionNumber24a);
            end

            % remove the Version field to prevent the parser from throwing
            % an error for an unmatched option.
            in = rmfield(in,'Version');

            this = nnet.internal.cnn.options.loadTrainingOptions(...
                @nnet.cnn.TrainingOptionsLBFGS,...
                in);        
        end
    end

    % set methods for properties requiring nontrivial canonicalization
    methods
        function obj = set.MaxIterations(obj,newValue)
            obj.MaxIterations = iCanonicalizePropertyValue(newValue,'MaxIterations');
        end

        function obj = set.HistorySize(obj,newValue)
            obj.HistorySize = iCanonicalizePropertyValue(newValue,'HistorySize');
        end

        function obj = set.InitialInverseHessianFactor(obj,newValue)
            obj.InitialInverseHessianFactor = iCanonicalizePropertyValue(newValue,'InitialInverseHessianFactor');
        end

        function obj = set.MaxNumLineSearchIterations(obj,newValue)
            obj.MaxNumLineSearchIterations = iCanonicalizePropertyValue(newValue,'MaxNumLineSearchIterations');
        end

        function obj = set.GradientTolerance(obj,newValue)
            obj.GradientTolerance = iCanonicalizePropertyValue(newValue,'GradientTolerance');
        end

        function obj = set.StepTolerance(obj,newValue)
            obj.StepTolerance = iCanonicalizePropertyValue(newValue,'StepTolerance');
        end

        function obj = set.CheckpointFrequency(obj,newValue)
            iValidatePropertyValue(newValue,'CheckpointFrequency');
            obj.CheckpointFrequency = iCanonicalizePropertyValue(newValue,'CheckpointFrequency');
        end

        function this = set.SequenceLength(this,newValue)
            iValidatePropertyValue(newValue,'SequenceLength');
            this.SequenceLength = iCanonicalizePropertyValue(newValue,'SequenceLength');
        end
    end
end

function tf = iTrainingOptionsAreFrom2023b(in)
% For training options from 2023b, Version will be 1
tf = in.Version == 1;
end

function inStruct = iUpgradeTrainingOptionsFrom2023bTo2024a(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2023bTo2024a(inStruct, version);
end

function val = iGetDefaultValue(optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinitionLBFGS;
end
val = optionsDefinition.getDefaultValue(optname);
end

function iValidatePropertyValue(x,optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinitionLBFGS;
end
optionsDefinition.validateValue(x,optname);
end

function canonicalizedval = iCanonicalizePropertyValue(x,optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinitionLBFGS;
end
canonicalizedval = optionsDefinition.canonicalizeValue(x,optname);
end
