classdef TrainingOptionsRMSProp < nnet.cnn.TrainingOptionsMiniBatch
    % TrainingOptionsRMSProp   Training options for root mean square propagation (RMSProp)
    %
    %   This class holds the training options for root mean square
    %   propagation (RMSProp)
    %
    %   TrainingOptionsRMSProp properties:
    %       SquaredGradientDecayFactor   - Decay factor for moving average
    %                                      of squared gradients
    %       Epsilon                      - Offset for the denominator in the
    %                                      RMSProp update
    %       InitialLearnRate             - Initial learning rate
    %       LearnRateSchedule            - Method for lowering the global 
    %                                      learning rate during training
    %       LearnRateDropFactor          - A multiplicative factor applied 
    %                                      to the learning rate every time
    %                                      a certain number of epochs passed 
    %                                      if 'LearnRateSchedule' is set to 
    %                                      'piecewise'
    %       LearnRateDropPeriod          - The learning rate drop factor is 
    %                                      applied to the global learning 
    %                                      rate every time this number of 
    %                                      epochs is passed if  
    %                                      'LearnRateSchedule' is set to 
    %                                      'piecewise'
    %       L2Regularization             - Factor for L2 regularization
    %       GradientThresholdMethod      - Method for gradient thresholding
    %       GradientThreshold            - Gradient threshold
    %       MaxEpochs                    - Maximum number of epochs
    %       MiniBatchSize                - The size of a mini-batch for
    %                                      training
    %       Verbose                      - Flag for printing information to
    %                                      the command window
    %       VerboseFrequency             - Number of iterations between
    %                                      printing verbose output to the
    %                                      command window
    %       ValidationData               - Data to use for validation during
    %                                      training
    %       ValidationFrequency          - Number of iterations between
    %                                      evaluations of validation
    %                                      metrics
    %       ValidationPatience           - Number of consecutive times that
    %                                      the validation loss must be
    %                                      greater than or equal to the
    %                                      smallest validation loss to stop
    %                                      training
    %       Shuffle                      - This controls if the training
    %                                      data is shuffled
    %       CheckpointPath               - Path where checkpoint networks
    %                                      are saved
    %       CheckpointFrequency          - Frequency of saving checkpoint
    %                                      networks
    %       CheckpointFrequencyUnit      - Unit of CheckpointFrequency
    %       ExecutionEnvironment         - Hardware to use for training
    %                                      network
    %       WorkerLoad                   - Specify compute and prefetch
    %                                      workers and their relative load
    %                                      in a parallel pool
    %       OutputFcn                    - Function to evaluate at each
    %                                      iteration
    %       Metrics                      - Metrics to calculate at each iteration
    %       Plots                        - Plots to display during training
    %       SequenceLength               - Sequence length of a mini-batch
    %                                      during training
    %       SequencePaddingValue         - Value used to pad mini-batches
    %                                      along the sequence dimension
    %       SequencePaddingDirection     - Direction of padding or
    %                                      truncation
    %       DispatchInBackground         - Flag to use prefetch queueing
    %                                      when reading data
    %       ResetInputNormalization      - Flag to reset input normalization
    %       BatchNormalizationStatistics - Method to calculate statistics
    %                                      in batch normalization layers
    %       OutputNetwork                - Network to return when training
    %                                      ends
    %
    %   Example:
    %       Create a set of training options for training with RMSProp. The
    %       learning rate will be reduced by a factor of 0.2 every 5
    %       epochs. The training will last for 20 epochs, and each
    %       iteration will use a mini-batch with 300 observations.
    %
    %       opts = trainingOptions("rmsprop", ...
    %           Plots="training-progress", ...
    %           LearnRateSchedule="piecewise", ...
    %           LearnRateDropFactor=0.2, ...
    %           LearnRateDropPeriod=5, ...
    %           MaxEpochs=20, ...
    %           MiniBatchSize=300);
    %
    %   See also trainingOptions, trainNetwork.
    
    % Copyright 2017-2024 The MathWorks, Inc.

    properties(Constant=true,Access=protected)

        SolverName = 'rmsprop'
        
        % Version   Number to identify the current version of this object
        %   This is used to ensure that objects from older versions are
        %   loaded correctly.
        Version = 7
    end
    
    properties
        % SquaredGradientDecayFactor   Decay factor for moving average of squared gradients
        %   A real scalar in [0,1) specifying the exponential decay rate
        %   for the squared gradient moving average.
        SquaredGradientDecayFactor {iValidatePropertyValue(SquaredGradientDecayFactor, 'SquaredGradientDecayFactor')} = iGetDefaultValue('SquaredGradientDecayFactor')
        
        % Epsilon   Offset for the denominator in the RMSProp update
        %   A positive real scalar specifying the offset to use in the
        %   denominator for the RMSProp update to prevent divide-by-zero
        %   problems.
        Epsilon {iValidatePropertyValue(Epsilon, 'Epsilon')} = iGetDefaultValue('Epsilon')
        
        % InitialLearnRate   Initial learning rate
        %   The initial learning rate that is used for training. If the
        %   learning rate is too low, training will take a long time, but
        %   if it is too high, the training is likely to get stuck at a
        %   suboptimal result.
        InitialLearnRate
    end
    
    methods
        function this = TrainingOptionsRMSProp(varargin, args)
            arguments (Repeating)
                varargin
            end
            arguments
                args.Definition = nnet.internal.cnn.options.TrainingOptionsDefinitionRMSProp
            end

            this = this@nnet.cnn.TrainingOptionsMiniBatch(args.Definition,varargin{:});
        end
        
        function out = saveobj(this)
            out = saveobj@nnet.cnn.TrainingOptionsMiniBatch(this);
            out.SquaredGradientDecayFactor = this.SquaredGradientDecayFactor;
            out.Epsilon = this.Epsilon;
        end
    end
    
    methods(Static)
        function this = loadobj(in)
            if iTrainingOptionsAreFrom2018a(in)
                versionNumber19b = 2;
                in = iUpgradeTrainingOptionsFrom2018aTo2019b(in, versionNumber19b);
            end
            if iTrainingOptionsAreFrom2019b(in)
                versionNumber20a = 3;
                in = iUpgradeTrainingOptionsFrom2019bTo2020a(in, versionNumber20a);
            end
            if iTrainingOptionsAreFrom2020a(in)
                versionNumber21a = 4;
                in = iUpgradeTrainingOptionsFrom2020aTo2021a(in, versionNumber21a);
            end
            if iTrainingOptionsAreFrom2021a(in)
                versionNumber22a = 5;
                in = iUpgradeTrainingOptionsFrom2021aTo2022a(in, versionNumber22a);
            end
            if iTrainingOptionsAreFrom2022a(in)
                versionNumber23b = 6;
                in = iUpgradeTrainingOptionsFrom2022aTo2023b(in, versionNumber23b);
            end
            if iTrainingOptionsAreFrom2023b(in)
                versionNumber24a = 7;
                in = iUpgradeTrainingOptionsFrom2023bTo2024a(in, versionNumber24a);
            end
            % remove the Version field to prevent the parser from throwing
            % an error for an unmatched option.
            in = rmfield(in,'Version');

            this = nnet.internal.cnn.options.loadTrainingOptions(...
                @nnet.cnn.TrainingOptionsRMSProp,...
                in);        
        end
    end

    % set methods for properties requiring nontrivial canonicalization
    methods
        function obj = set.SquaredGradientDecayFactor(obj,newValue)
            obj.SquaredGradientDecayFactor = iCanonicalizePropertyValue(newValue,'SquaredGradientDecayFactor');
        end

        function obj = set.Epsilon(obj,newValue)
            obj.Epsilon = iCanonicalizePropertyValue(newValue,'Epsilon');
        end

        function obj = set.InitialLearnRate(obj,newValue)
            iValidatePropertyValue(newValue,'InitialLearnRate');
            obj.InitialLearnRate = iCanonicalizePropertyValue(newValue,'InitialLearnRate');
        end
    end

end

function tf = iTrainingOptionsAreFrom2018a(in)
% For training options from 2018a, Version will be 1
tf = in.Version == 1;
end

function tf = iTrainingOptionsAreFrom2019b(in)
% For training options from 2019b, Version will be 2
tf = in.Version == 2;
end

function tf = iTrainingOptionsAreFrom2020a(in)
% For training options from 2020a, Version will be 3
tf = in.Version == 3;
end

function tf = iTrainingOptionsAreFrom2021a(in)
% For training options from 2021a, Version will be 4
tf = in.Version == 4;
end

function tf = iTrainingOptionsAreFrom2022a(in)
% For training options from 2022a, Version will be 5
tf = in.Version == 5;
end

function tf = iTrainingOptionsAreFrom2023b(in)
% For training options from 2023b, Version will be 6
tf = in.Version == 6;
end

function inStruct = iUpgradeTrainingOptionsFrom2018aTo2019b(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2018aTo2019b(inStruct, version);
end

function inStruct = iUpgradeTrainingOptionsFrom2019bTo2020a(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2019bTo2020a(inStruct, version);
end

function inStruct = iUpgradeTrainingOptionsFrom2020aTo2021a(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2020aTo2021a(inStruct, version);
end

function inStruct = iUpgradeTrainingOptionsFrom2021aTo2022a(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2021aTo2022a(inStruct, version);
end

function inStruct = iUpgradeTrainingOptionsFrom2022aTo2023b(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2022aTo2023b(inStruct, version);
end

function inStruct = iUpgradeTrainingOptionsFrom2023bTo2024a(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2023bTo2024a(inStruct, version);
end

function val = iGetDefaultValue(optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinitionRMSProp;
end
val = optionsDefinition.getDefaultValue(optname);
end

function iValidatePropertyValue(x,optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinitionRMSProp;
end
optionsDefinition.validateValue(x,optname);
end

function canonicalizedval = iCanonicalizePropertyValue(x,optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinitionRMSProp;
end
canonicalizedval = optionsDefinition.canonicalizeValue(x,optname);
end
