classdef TrainingOptionsSGDM < nnet.cnn.TrainingOptionsMiniBatch
    % TrainingOptionsSGDM   Training options for stochastic gradient descent with momentum
    %
    %   This class holds the training options for stochastic gradient
    %   descent with momentum.
    %
    %   TrainingOptionsSGDM properties:
    %       Momentum                     - Momentum for learning
    %       InitialLearnRate             - Initial learning rate
    %       LearnRateSchedule            - Method for lowering the global 
    %                                      learning rate during training
    %       LearnRateDropFactor          - A multiplicative factor applied 
    %                                      to the learning rate every time
    %                                      a certain number of epochs passed 
    %                                      if 'LearnRateSchedule' is set to 
    %                                      'piecewise'
    %       LearnRateDropPeriod          - The learning rate drop factor is 
    %                                      applied to the global learning 
    %                                      rate every time this number of 
    %                                      epochs is passed if  
    %                                      'LearnRateSchedule' is set to 
    %                                      'piecewise'
    %       L2Regularization             - Factor for L2 regularization
    %       GradientThresholdMethod      - Method for gradient thresholding
    %       GradientThreshold            - Gradient threshold
    %       MaxEpochs                    - Maximum number of epochs
    %       MiniBatchSize                - The size of a mini-batch for
    %                                      training
    %       Verbose                      - Flag for printing information to
    %                                      the command window
    %       VerboseFrequency             - Number of iterations between
    %                                      printing verbose output to the
    %                                      command window
    %       ValidationData               - Data to use for validation during
    %                                      training
    %       ValidationFrequency          - Number of iterations between
    %                                      evaluations of validation
    %                                      metrics
    %       ValidationPatience           - Number of consecutive times that
    %                                      the validation loss must be
    %                                      greater than or equal to the
    %                                      smallest validation loss to stop
    %                                      training
    %       Shuffle                      - This controls if the training
    %                                      data is shuffled
    %       CheckpointPath               - Path where checkpoint networks
    %                                      are saved
    %       CheckpointFrequency          - Frequency of saving checkpoint
    %                                      networks
    %       CheckpointFrequencyUnit      - Unit of CheckpointFrequency
    %       ExecutionEnvironment         - Hardware to use for training
    %                                      network
    %       WorkerLoad                   - Specify compute and prefetch
    %                                      workers and their relative load
    %                                      in a parallel pool
    %       OutputFcn                    - Function to evaluate at each
    %                                      iteration
    %       Metrics                      - Metrics to calculate at each iteration
    %       Plots                        - Plots to display during training
    %       SequenceLength               - Sequence length of a mini-batch
    %                                      during training
    %       SequencePaddingValue         - Value used to pad mini-batches
    %                                      along the sequence dimension
    %       SequencePaddingDirection     - Direction of padding or
    %                                      truncation
    %       DispatchInBackground         - Flag to use prefetch queueing
    %                                      when reading data
    %       ResetInputNormalization      - Flag to reset input normalization
    %       BatchNormalizationStatistics - Method to calculate statistics
    %                                      in batch normalization layers
    %       OutputNetwork                - Network to return when training
    %                                      ends
    %
    %   Example:
    %       Create a set of training options for training with stochastic
    %       gradient descent with momentum. The learning rate will be
    %       reduced by a factor of 0.2 every 5 epochs. The training will
    %       last for 20 epochs, and each iteration will use a mini-batch
    %       with 300 observations.
    %
    %       opts = trainingOptions("sgdm", ...
    %           Plots="training-progress", ...
    %           LearnRateSchedule="piecewise", ...
    %           LearnRateDropFactor=0.2, ...
    %           LearnRateDropPeriod=5, ...
    %           MaxEpochs=20, ...
    %           MiniBatchSize=300);
    %
    %   See also trainingOptions, trainNetwork.
    
    % Copyright 2015-2024 The MathWorks, Inc.

    properties(Constant=true,Access=protected)
        SolverName = 'sgdm'

        % Version   Number to identify the current version of this object
        %   This is used to ensure that objects from older versions are
        %   loaded correctly.
        Version = 9
    end
    
    properties
        % Momentum   Momentum for learning
        %   The momentum determines the contribution of the gradient step
        %   from the previous iteration to the current iteration of
        %   training. It must be a value between 0 and 1, where 0 will give
        %   no contribution from the previous step, and 1 will give a
        %   maximal contribution from the previous step.
        Momentum double {iValidatePropertyValue(Momentum, 'Momentum')} = iGetDefaultValue('Momentum')

        % InitialLearnRate   Initial learning rate
        %   The initial learning rate that is used for training. If the
        %   learning rate is too low, training will take a long time, but
        %   if it is too high, the training is likely to get stuck at a
        %   suboptimal result.
        InitialLearnRate
    end
    
    methods
        function this = TrainingOptionsSGDM(varargin, args)
            arguments (Repeating)
                varargin
            end
            arguments
                args.Definition = nnet.internal.cnn.options.TrainingOptionsDefinitionSGDM
            end

            this = this@nnet.cnn.TrainingOptionsMiniBatch(args.Definition,varargin{:});
        end

        function out = saveobj(this)
            out = saveobj@nnet.cnn.TrainingOptionsMiniBatch(this);
            out.Momentum = this.Momentum;
        end
    end
    
    methods(Static)
        function this = loadobj(in)
            if iTrainingOptionsAreFrom2016aOr2016b(in)
                in = iUpgradeTrainingOptionsFrom2016aOr2016bTo2017a(in);
            end
            if iTrainingOptionsAreFrom2017a(in)
                in = iUpgradeTrainingOptionsFrom2017aTo2017b(in);
            end
            if iTrainingOptionsAreFrom2017b(in)
                in = iUpgradeTrainingOptionsFrom2017bTo2018a(in);
            end
            if iTrainingOptionsAreFrom2018a(in)
                versionNumber19b = 4;
                in = iUpgradeTrainingOptionsFrom2018aTo2019b(in, versionNumber19b);
            end
            if iTrainingOptionsAreFrom2019b(in)
                versionNumber20a = 5;
                in = iUpgradeTrainingOptionsFrom2019bTo2020a(in, versionNumber20a);
            end
            if iTrainingOptionsAreFrom2020a(in)
                versionNumber21a = 6;
                in = iUpgradeTrainingOptionsFrom2020aTo2021a(in, versionNumber21a);
            end
            if iTrainingOptionsAreFrom2021a(in)
                versionNumber22a = 7;
                in = iUpgradeTrainingOptionsFrom2021aTo2022a(in, versionNumber22a);
            end
            if iTrainingOptionsAreFrom2022a(in)
                versionNumber23b = 8;
                in = iUpgradeTrainingOptionsFrom2022aTo2023b(in, versionNumber23b);
            end
            if iTrainingOptionsAreFrom2023b(in)
                versionNumber24a = 9;
                in = iUpgradeTrainingOptionsFrom2023bTo2024a(in, versionNumber24a);
            end
            % remove the Version field to prevent the parser from throwing
            % an error for an unmatched option.
            in = rmfield(in,'Version');
            
            this = nnet.internal.cnn.options.loadTrainingOptions(...
                @nnet.cnn.TrainingOptionsSGDM,...
                in);        
        end
    end

    % set methods for properties requiring nontrivial canonicalization
    methods
        function obj = set.Momentum(obj,newValue)
            obj.Momentum = iCanonicalizePropertyValue(newValue,'Momentum');
        end

        function obj = set.InitialLearnRate(obj,newValue)
            iValidatePropertyValue(newValue,'InitialLearnRate');
            obj.InitialLearnRate = iCanonicalizePropertyValue(newValue,'InitialLearnRate');
        end
    end
end

function tf = iTrainingOptionsAreFrom2016aOr2016b(in)
% For training options from 2016a and 2016b, "in" will be an object
% instead of a struct.
tf = isa(in, 'nnet.cnn.TrainingOptionsSGDM');
end

function tf = iTrainingOptionsAreFrom2017a(in)
% For training options from 2017a, Version will be 1
tf = in.Version == 1;
end

function tf = iTrainingOptionsAreFrom2017b(in)
% For training options from 2017b, Version will be 2
tf = in.Version == 2;
end

function tf = iTrainingOptionsAreFrom2018a(in)
% For training options from 2018a, Version will be 3
tf = in.Version == 3;
end

function tf = iTrainingOptionsAreFrom2019b(in)
% For training options from 2019b, Version will be 4
tf = in.Version == 4;
end

function tf = iTrainingOptionsAreFrom2020a(in)
% For training options from 2020a, Version will be 5
tf = in.Version == 5;
end

function tf = iTrainingOptionsAreFrom2021a(in)
% For training options from 2021a, Version will be 6
tf = in.Version == 6;
end

function tf = iTrainingOptionsAreFrom2022a(in)
% For training options from 2022a, Version will be 7
tf = in.Version == 7;
end

function tf = iTrainingOptionsAreFrom2023b(in)
% For training options from 2023b, Version will be 8
tf = in.Version == 8;
end

function inStruct = iUpgradeTrainingOptionsFrom2016aOr2016bTo2017a(in)
% iUpgradeTrainingOptionsFrom2016aOr2016bTo2017a   Upgrade training options
% from R2016a or R2016b to R2017a

% Set properties that exist in 2016a and 2016b
inStruct = struct;
inStruct.Momentum = in.Momentum;
inStruct.InitialLearnRate = in.InitialLearnRate;
learnRateScheduleSettings = in.LearnRateScheduleSettings;
learnRateScheduleSettings.InitialLearnRate = in.InitialLearnRate;
inStruct = iFlattenLearnRateScheduleSettings(inStruct, learnRateScheduleSettings);
inStruct.L2Regularization = in.L2Regularization;
inStruct.MaxEpochs = in.MaxEpochs;
inStruct.MiniBatchSize = in.MiniBatchSize;
inStruct.Verbose = in.Verbose;
inStruct.Shuffle = in.Shuffle;
inStruct.CheckpointPath = in.CheckpointPath;

% Set properties that don't exist in 2016a or 2016b
inStruct.VerboseFrequency = 50;
inStruct.ExecutionEnvironment = 'auto';
inStruct.WorkerLoad = [];
inStruct.OutputFcn = [];
inStruct.Version = 1;
end

function inStruct = iUpgradeTrainingOptionsFrom2017aTo2017b(inStruct)
% iUpgradeTrainingOptionsFrom2017aTo2017b   Upgrade training options
% from R2017a to R2017b

% Set properties that exist in 2017a
inStruct.Version = 2;

% Set properties that don't exist in 2017a
inStruct.ValidationData = [];
inStruct.ValidationFrequency = 50;
inStruct.ValidationPatience = 5;
inStruct.Plots = 'none';
inStruct.SequenceLength = 'longest';
inStruct.SequencePaddingValue = 0;
end

function inStruct = iUpgradeTrainingOptionsFrom2017bTo2018a(inStruct)
% iUpgradeTrainingOptionsFrom2017bTo2018a   Upgrade training options
% from R2017b to R2018a

% Set properties that exist in 2017b
inStruct.Version = 3;

% Set properties that don't exist in 2017b
inStruct.GradientThresholdMethod = 'l2norm';
inStruct.GradientThreshold = Inf;
end

function inStruct = iUpgradeTrainingOptionsFrom2018aTo2019b(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2018aTo2019b(inStruct, version);
end

function inStruct = iUpgradeTrainingOptionsFrom2019bTo2020a(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2019bTo2020a(inStruct, version);
end

function inStruct = iUpgradeTrainingOptionsFrom2020aTo2021a(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2020aTo2021a(inStruct, version);
end

function inStruct = iUpgradeTrainingOptionsFrom2021aTo2022a(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2021aTo2022a(inStruct, version);
end

function inStruct = iUpgradeTrainingOptionsFrom2022aTo2023b(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2022aTo2023b(inStruct, version);
end

function inStruct = iUpgradeTrainingOptionsFrom2023bTo2024a(inStruct, version)
inStruct = nnet.internal.cnn.options.upgradeTrainingOptionsFrom2023bTo2024a(inStruct, version);
end

function val = iGetDefaultValue(optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinitionSGDM;
end
val = optionsDefinition.getDefaultValue(optname);
end

function iValidatePropertyValue(x,optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinitionSGDM;
end
optionsDefinition.validateValue(x,optname);
end

function canonicalizedval = iCanonicalizePropertyValue(x,optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinitionSGDM;
end
canonicalizedval = optionsDefinition.canonicalizeValue(x,optname);
end

function out = iFlattenLearnRateScheduleSettings(out, learnRateScheduleSettings)
learnRateScedule = learnRateScheduleSettings.Method;
out.LearnRateSchedule = learnRateScedule;
switch learnRateScedule
    case 'none'
        out.LearnRateDropFactor = [];
        out.LearnRateDropPeriod = [];
    case 'piecewise'
        out.LearnRateDropFactor = learnRateScheduleSettings.DropRateFactor;
        out.LearnRateDropPeriod = learnRateScheduleSettings.DropPeriod;
    otherwise
        error(message('nnet_cnn:TrainingOptionsSGDM:InvalidLearningRateScheduleMethod'));
end
end
