classdef(Abstract) TrainingOptions
    % TrainingOptions   Base training options for all solvers
    %
    %   This class holds the training options common to all solvers
    %
    %   TrainingOptions properties:
    %       L2Regularization             - Factor for L2 regularization
    %       GradientThresholdMethod      - Method for gradient thresholding
    %       GradientThreshold            - Gradient threshold
    %       Verbose                      - Flag for printing information to
    %                                      the command window
    %       VerboseFrequency             - Number of iterations between
    %                                      printing verbose output to the
    %                                      command window
    %       ValidationData               - Data to use for validation during
    %                                      training
    %       ValidationFrequency          - Number of iterations between
    %                                      evaluations of validation
    %                                      metrics
    %       ValidationPatience           - Number of consecutive times that
    %                                      the validation loss must be
    %                                      greater than or equal to the
    %                                      smallest validation loss to stop
    %                                      training
    %       ObjectiveMetricName          - Name of the metric to use for 
    %                                      early stopping and return the 
    %                                      best network
    %       CheckpointPath               - Path where checkpoint networks
    %                                      are saved
    %       CheckpointFrequency          - Frequency of saving checkpoint
    %                                      networks
    %       ExecutionEnvironment         - Hardware to use for training
    %                                      network
    %       OutputFcn                    - Function to evaluate at each
    %                                      iteration
    %       Metrics                      - Metrics to calculate at every iteration
    %       Plots                        - Plots to display during training
    %       SequenceLength               - Sequence length of a mini-batch
    %                                      during training
    %       SequencePaddingValue         - Value used to pad mini-batches
    %                                      along the sequence dimension
    %       SequencePaddingDirection     - Direction of padding or
    %                                      truncation
    %       InputDataFormats             - Input data formats
    %       TargetDataFormats            - Target data formats
    %       ResetInputNormalization      - Flag to reset input normalization
    %       BatchNormalizationStatistics - Method to calculate statistics
    %                                      in batch normalization layers
    %       OutputNetwork                - Network to return when training ends   
    %       Acceleration                 - Performance optimizations
    
    % Copyright 2017-2023 The MathWorks, Inc.
    
    properties(Abstract,Constant=true,Access=protected)
        % Version   Number to identify the current version of this object
        %   This is used to ensure that objects from older versions are
        %   loaded correctly.
        Version

        % SolverName TBD for every class derived from
        % TrainingOptions, e.g. TrainingOptionsADAM.SolverName =
        % 'adam'.
        SolverName
    end

    properties(Abstract)
        SequenceLength

        CheckpointFrequency
    end
    
    properties
        % L2Regularization   Factor for L2 regularization
        %   The factor for the L2 regularizer. It should be noted that each
        %   set of parameters in a layer can specify a multiplier for this
        %   L2 regularizer.
        L2Regularization {iValidatePropertyValue(L2Regularization,'L2Regularization')} = iGetDefaultValue('L2Regularization')

        % GradientThresholdMethod   Method used for gradient thresholding
        %   Method used for thresholding the gradient. Options are
        %   'global-l2norm','l2norm' (default) and 'absolute-value'.
        GradientThresholdMethod {iValidatePropertyValue(GradientThresholdMethod,'GradientThresholdMethod')} = iGetDefaultValue('GradientThresholdMethod')
    
        % GradientThreshold   Threshold used for gradients
        %   The threshold used to scale the gradients by the method
        %   specified in GradientThresholdMethod.
        GradientThreshold {iValidatePropertyValue(GradientThreshold,'GradientThreshold')} = iGetDefaultValue('GradientThreshold')
                                                            
        % Verbose   Flag for printing information to the command window
        %   If this is set to true, information on training progress will
        %   be printed to the command window. The default is true.
        Verbose logical {iValidatePropertyValue(Verbose,'Verbose')} = iGetDefaultValue('Verbose')

        % VerboseFrequency   Frequency for printing information
        %   This only has an effect if 'Verbose' is true. It specifies the
        %   number of iterations between printing to the command window.
        VerboseFrequency {iValidatePropertyValue(VerboseFrequency,'VerboseFrequency')} = iGetDefaultValue('VerboseFrequency')
 
        % ValidationData   Data to be used for validation purposes
        ValidationData {iValidatePropertyValue(ValidationData,'ValidationData')} = iGetDefaultValue('ValidationData')
        
        % ValidationFrequency   Frequency for computing validation metrics
        ValidationFrequency {iValidatePropertyValue(ValidationFrequency,'ValidationFrequency')} = iGetDefaultValue('ValidationFrequency')
        
        % ValidationPatience   Patience used to stop training
        ValidationPatience {iValidatePropertyValue(ValidationPatience,'ValidationPatience')} = iGetDefaultValue('ValidationPatience')

        % ObjectiveMetricName   Name of the metric to use for early stopping and return best network
        ObjectiveMetricName {iValidatePropertyValue(ObjectiveMetricName,'ObjectiveMetricName')} = iGetDefaultValue('ObjectiveMetricName')
        
        % CheckpointPath   This is the path where the checkpoint networks
        % will be saved. If empty, no checkpoint will be saved.
        CheckpointPath {iValidatePropertyValue(CheckpointPath,'CheckpointPath')} = iGetDefaultValue('CheckpointPath')
        
        % ExecutionEnvironment   Determines what hardware to use for
        % training the network.
        ExecutionEnvironment {iValidatePropertyValue(ExecutionEnvironment,'ExecutionEnvironment')} = iGetDefaultValue('ExecutionEnvironment')
        
        % OutputFcn   Functions to call after each iteration, passing
        % training info from the current iteration, and returning true to
        % terminate training early
        OutputFcn {iValidatePropertyValue(OutputFcn,'OutputFcn')} = iGetDefaultValue('OutputFcn')
        
        % Metrics  Metrics to calculate at every iteration
        Metrics {iValidatePropertyValue(Metrics,'Metrics')} = iGetDefaultValue('Metrics')
        
        % Plots   Plots to show during training
        Plots {iValidatePropertyValue(Plots,'Plots')} = iGetDefaultValue('Plots')
        
        % SequencePaddingValue   Scalar value used to pad mini-batches in
        % the along the sequence dimension
        SequencePaddingValue double {iValidatePropertyValue(SequencePaddingValue,'SequencePaddingValue')} = iGetDefaultValue('SequencePaddingValue')
        
        % SequencePaddingDirection   Specify direction of sequence padding
        % or truncation
        SequencePaddingDirection {iValidatePropertyValue(SequencePaddingDirection,'SequencePaddingDirection')} = iGetDefaultValue('SequencePaddingDirection')

        % InputDataFormats  Training input data dimension formats
        InputDataFormats {iValidatePropertyValue(InputDataFormats,'InputDataFormats')} = iGetDefaultValue('InputDataFormats')

        % TargetDataFormats  Training input data dimension formats
        TargetDataFormats {iValidatePropertyValue(TargetDataFormats,'TargetDataFormats')} = iGetDefaultValue('TargetDataFormats')
        
        % ResetInputNormalization  Flag for clearing statistics in input
        % layers. Empty statistics are derived during training.
        ResetInputNormalization logical {iValidatePropertyValue(ResetInputNormalization,'ResetInputNormalization')} = iGetDefaultValue('ResetInputNormalization')
        
        % BatchNormalizationStatistics  Method for selecting the approach
        % to evaluate the statistics in batch normalization layers
        BatchNormalizationStatistics {iValidatePropertyValue(BatchNormalizationStatistics,'BatchNormalizationStatistics')} = iGetDefaultValue('BatchNormalizationStatistics')
        
        % OutputNetwork  Specify network to return when training ends
        OutputNetwork {iValidatePropertyValue(OutputNetwork,'OutputNetwork')} = iGetDefaultValue('OutputNetwork')

        % Acceleration  Specify which performance optimization to enable
        Acceleration {iValidatePropertyValue(Acceleration,'Acceleration')} = iGetDefaultValue('Acceleration')
    end

    methods(Access = public)
        function this = TrainingOptions(defaultOptions,varargin)
            try
                keepUnmatched = true;
                [inputArguments, unmatched] = nnet.internal.cnn.options.readNVPsWithOptionsDefinitionParser(defaultOptions,keepUnmatched,varargin{:});
                this.errorForInvalidOptions(unmatched,this.SolverName);
                this = this.assignTrainingOptions(inputArguments);
            catch e
                % Reduce the stack trace of the error message by throwing as caller
                throwAsCaller(e)
            end
        end
        
        function out = saveobj(this)
            out.Version = this.Version;
            out.L2Regularization = this.L2Regularization;
            out.GradientThresholdMethod = this.GradientThresholdMethod;
            out.GradientThreshold = this.GradientThreshold;
            out.Verbose = this.Verbose;
            out.VerboseFrequency = this.VerboseFrequency;
            out.ValidationData = this.ValidationData;
            out.ValidationFrequency = this.ValidationFrequency;
            out.ValidationPatience = this.ValidationPatience;
            out.ObjectiveMetricName = this.ObjectiveMetricName;
            out.CheckpointPath = this.CheckpointPath;
            out.CheckpointFrequency = this.CheckpointFrequency;
            out.ExecutionEnvironment = this.ExecutionEnvironment;
            out.OutputFcn = this.OutputFcn;
            out.Metrics = this.Metrics;
            out.Plots = this.Plots;
            out.SequenceLength = this.SequenceLength;
            out.SequencePaddingValue = this.SequencePaddingValue;
            out.SequencePaddingDirection = this.SequencePaddingDirection;
            out.InputDataFormats = this.InputDataFormats;
            out.TargetDataFormats = this.TargetDataFormats;
            out.ResetInputNormalization = this.ResetInputNormalization;
            out.BatchNormalizationStatistics = this.BatchNormalizationStatistics;
            out.OutputNetwork = this.OutputNetwork;
            out.Acceleration = this.Acceleration;

            if isa(out.Plots,'nnet.internal.cnn.ui.TrainingPlotter')
                out.Plots = 'training-progress';
            end
        end
    end

    % set methods for properties requiring nontrivial canonicalization
    methods
        
        function obj = set.L2Regularization(obj,newValue)
            obj.L2Regularization = iCanonicalizePropertyValue(newValue,'L2Regularization');
        end

        function obj = set.GradientThreshold(obj,newValue)
            obj.GradientThreshold = iCanonicalizePropertyValue(newValue,'GradientThreshold');
        end

        function obj = set.Verbose(obj,newValue)
            obj.Verbose = iCanonicalizePropertyValue(newValue,'Verbose');
        end

        function obj = set.VerboseFrequency(obj,newValue)
            obj.VerboseFrequency = iCanonicalizePropertyValue(newValue,'VerboseFrequency');
        end

        function obj = set.ValidationFrequency(obj,newValue)
            obj.ValidationFrequency = iCanonicalizePropertyValue(newValue,'ValidationFrequency');
        end

        function obj = set.ValidationPatience(obj,newValue)
            obj.ValidationPatience = iCanonicalizePropertyValue(newValue,'ValidationPatience');
        end

        function obj = set.SequencePaddingValue(obj,newValue)
            obj.SequencePaddingValue = iCanonicalizePropertyValue(newValue,'SequencePaddingValue');
        end

        function obj = set.ResetInputNormalization(obj,newValue)
            obj.ResetInputNormalization = iCanonicalizePropertyValue(newValue,'ResetInputNormalization');
        end

        function obj = set.GradientThresholdMethod(obj,newValue)
            obj.GradientThresholdMethod = iCanonicalizePropertyValue(newValue,'GradientThresholdMethod');
        end

        function obj = set.CheckpointPath(obj,newValue)
            obj.CheckpointPath = iCanonicalizePropertyValue(newValue,'CheckpointPath');
        end

        function obj = set.ExecutionEnvironment(obj,newValue)
            obj.ExecutionEnvironment = iCanonicalizePropertyValue(newValue,'ExecutionEnvironment');
        end
        
        function obj = set.ObjectiveMetricName(obj,newValue)
            obj.ObjectiveMetricName = iCanonicalizePropertyValue(newValue,'ObjectiveMetricName');
        end

        function obj = set.Metrics(obj,newValue)
            obj.Metrics = iCanonicalizePropertyValue(newValue,'Metrics');
        end
        
        function obj = set.Plots(obj,newValue)
            obj.Plots = iCanonicalizePropertyValue(newValue,'Plots');
        end
        
        function obj = set.SequencePaddingDirection(obj,newValue)
            obj.SequencePaddingDirection = iCanonicalizePropertyValue(newValue,'SequencePaddingDirection');
        end

        function obj = set.InputDataFormats(obj,newValue)
            obj.InputDataFormats = iCanonicalizePropertyValue(newValue,'InputDataFormats');
        end

        function obj = set.TargetDataFormats(obj,newValue)
            obj.TargetDataFormats = iCanonicalizePropertyValue(newValue,'TargetDataFormats');
        end

        function obj = set.BatchNormalizationStatistics(obj,newValue)
            obj.BatchNormalizationStatistics = iCanonicalizePropertyValue(newValue,'BatchNormalizationStatistics');
        end

        function obj = set.OutputNetwork(obj,newValue)
            obj.OutputNetwork = iCanonicalizePropertyValue(newValue,'OutputNetwork');
        end

        function obj = set.Acceleration(obj,newValue)
            obj.Acceleration = iCanonicalizePropertyValue(newValue,'Acceleration');
        end
    end

    methods(Abstract, Access=protected)
        warnIfCheckpointFrequencyDoesNotMatchCheckpointPath(this);
    end

    methods (Access=protected)
        function fn = getPublicProperties(this)
            % Return the name of all the editable public properties.
            % Sub-classes can modify this list
            fn = fieldnames(this);
        end
    end

    methods(Access=private,Hidden=true)
        function this = assignTrainingOptions(this,inputArguments)
            fn = getPublicProperties(this);
            for ii = 1:numel(fn)
                this.(fn{ii}) = inputArguments.(fn{ii});
            end
        end

        function errorForInvalidOptions(~,s,solverName)
            fieldNames = fieldnames(s);
            numFields = numel(fieldNames);
            if ( numFields > 0 )
                firstInvalidFieldName = fieldNames{1};
                error(message('nnet_cnn:TrainingOptionsSGDM:InvalidTrainingOptionForSolver',firstInvalidFieldName,solverName));
            end
        end

        function assertOutputNetworkMatchesValidationData(this)
            validationOpts = ["best-validation-loss","best-validation"];
            [~,idx] = ismember(string(this.OutputNetwork), validationOpts);
            if idx && isempty(this.ValidationData)
                error(message('nnet_cnn:TrainingOptionsSGDM:InvalidOutputNetworkOption', validationOpts(idx)));
            end
        end

        function assertObjectiveMetricNameMatchesMetrics(this)
            metricNames = lower(nnet.internal.cnn.util.getMetricNames(this.Metrics));
            objectiveMetric = lower(string(this.ObjectiveMetricName));
            tf = ismember(objectiveMetric, metricNames);
            if ~tf && (objectiveMetric ~= "loss")
                error(message('nnet_cnn:TrainingOptionsSGDM:InvalidObjectiveMetricNameOption'));
            end
        end

        function assertOutputNetworkMatchesObjectiveMetricName(this)
            if (this.OutputNetwork == "best-validation-loss") && (lower(this.ObjectiveMetricName) ~= "loss")
                error(message('nnet_cnn:TrainingOptionsSGDM:OutputNetworkObjectiveMetricMismatch'));
            end
        end

        function assertNotTrainingOptionsLBFGS(this)
            if isa(this, 'nnet.cnn.TrainingOptionsLBFGS')
                error(message('nnet_cnn:TrainingOptionsLBFGS:TrainNetworkUnsupportedSolver'));
            end
        end

        function assertDataFormatsAreAuto(this)
            if ~isequal(this.InputDataFormats,"auto")
                error(message('nnet_cnn:TrainingOptionsSGDM:TrainNetworkUnsupportedOption','InputDataFormats'));
            end
            if ~isequal(this.TargetDataFormats,"auto")
                error(message('nnet_cnn:TrainingOptionsSGDM:TrainNetworkUnsupportedOption','TargetDataFormats'));
            end
        end

        function assertMetricsAreEmpty(this)
            if ~isempty(this.Metrics)
                error(message('nnet_cnn:TrainingOptionsSGDM:TrainNetworkUnsupportedOption','Metrics'));
            end
        end

        function assertAccelerationAuto(this)
            if isa(this.Acceleration, 'deep.internal.optimization.TrainingAccelerationProfile') || this.Acceleration ~= "auto"
                error(message('nnet_cnn:TrainingOptionsSGDM:TrainNetworkUnsupportedOption','Acceleration'));
            end
		end

        function assertValidationDataIsNotMinibatchqueue(this)
            if isa(this.ValidationData, 'minibatchqueue')
                error(message('nnet_cnn:TrainingOptionsSGDM:TrainNetworkMinibatchqueueValidationData'));
            end
        end
    end

    methods(Hidden=true)
        function validateRelations(this)
            %validateRelations   Validates relations between options
            %
            % This method is called inside trainNetwork and trainnet. 
            % Validation of relations, between dependent options, is 
            % reserved for when the training options object is passed to
            % the trainer. This is required because any of the related
            % options can be set first and doing the validation in the
            % trainer, and not trainingOptions, avoids error or warning for
            % potentially valid codes
            this.assertOutputNetworkMatchesValidationData();
            this.warnIfCheckpointFrequencyDoesNotMatchCheckpointPath();
            this.assertObjectiveMetricNameMatchesMetrics();
            this.assertOutputNetworkMatchesObjectiveMetricName();
        end
        
        function validateOptionsForTrainNetwork(this)
            assertDataFormatsAreAuto(this);
            assertMetricsAreEmpty(this);
            assertNotTrainingOptionsLBFGS(this);
            assertAccelerationAuto(this);
            assertValidationDataIsNotMinibatchqueue(this);
        end

        function validateDataFormatsAndMetricsAreEmpty(this, trainerName)
            if ~isequal(this.InputDataFormats,"auto")
                error(message('nnet_cnn:TrainingOptionsSGDM:TrainerUnsupportedOption', trainerName, 'InputDataFormats'));
            end
            if ~isequal(this.TargetDataFormats,"auto")
                error(message('nnet_cnn:TrainingOptionsSGDM:TrainerUnsupportedOption', trainerName, 'TargetDataFormats'));
            end
            if ~isempty(this.Metrics)
                error(message('nnet_cnn:TrainingOptionsSGDM:TrainerUnsupportedOption', trainerName, 'Metrics'));
            end
        end

        function assertMiniBatchSolver(this, trainerName)
            if isa(this, 'nnet.cnn.TrainingOptionsLBFGS')
                error(message('nnet_cnn:TrainingOptionsLBFGS:TrainerUnsupportedSolver', trainerName));
            end
        end
    end
end

function val = iGetDefaultValue(optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinition;
end
val = optionsDefinition.getDefaultValue(optname);
end

function iValidatePropertyValue(x,optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinition;
end
optionsDefinition.validateValue(x,optname);
end

function canonicalizedval = iCanonicalizePropertyValue(x,optname)
persistent optionsDefinition;
if isempty(optionsDefinition)
    optionsDefinition=nnet.internal.cnn.options.TrainingOptionsDefinition;
end
canonicalizedval = optionsDefinition.canonicalizeValue(x,optname);
end
