function opts = trainingOptions(solverName, varargin)
% trainingOptions   Options for training a neural network
%
%   options = trainingOptions(solverName) creates a set of training options
%   for the solver specified by solverName. Possible values for solverName
%   include:
%
%       'sgdm'    -   Stochastic gradient descent with momentum.
%       'adam'    -   Adaptive moment estimation (ADAM).
%       'rmsprop' -   Root mean square propagation (RMSProp).
%       'lbfgs'   -   Limited-memory BFGS (L-BFGS).
%
%   options = trainingOptions(solverName, 'PARAM1', VAL1, 'PARAM2', VAL2, ...)
%   specifies optional parameter name/value pairs for creating the training
%   options:
%
%       'Momentum'            - This parameter only applies if the solver
%                               is 'sgdm'. The momentum determines the
%                               contribution of the gradient step from the
%                               previous iteration to the current
%                               iteration of training. It must be a value
%                               between 0 and 1, where 0 will give no
%                               contribution from the previous step, and 1
%                               will give a maximal contribution from the
%                               previous step. The default value is 0.9.
%       'GradientDecayFactor' - This parameter only applies if the solver
%                               is 'adam'. It specifies the exponential
%                               decay rate for the gradient moving average
%                               in solver 'adam'. It must be a value >= 0
%                               and < 1. This parameter is denoted by the
%                               symbol 'beta1' in the original paper on
%                               'adam'. The default value is 0.9.
%       'SquaredGradientDecayFactor'
%                             - This parameter only applies if the solver
%                               is 'adam' or 'rmsprop'. It specifies the
%                               exponential decay rate for the squared
%                               gradient moving average in solver 'adam'
%                               and 'rmsprop'. It must be a value >= 0 and
%                               < 1. This parameter is denoted by the
%                               symbol 'beta2' in the original paper on
%                               'adam'. The default value is 0.999 for
%                               'adam' and 0.9 for 'rmsprop'.
%       'Epsilon'             - This parameter only applies if the solver
%                               is 'adam' or 'rmsprop'. It specifies the
%                               offset to use in the denominator for the
%                               update in solver 'adam' and 'rmsprop'. It
%                               must be a value > 0. The default value is
%                               1e-8.
%       'HistorySize'         - This parameter only applies if the solver
%                               is 'lbfgs'. It specifies the number of
%                               steps to store, specified as a positive
%                               integer. The default value is 10.
%       'InitialInverseHessianFactor'
%                             - This parameter only applies if the solver
%                               is 'lbfgs'. It specifies the value to
%                               approximate the inverse Hessian when
%                               applying the first L-BFGS update step,
%                               specified as a positive scalar. The default
%                               value is 1.
%       'LineSearchMethod'    - This parameter only applies if the solver
%                               is 'lbfgs'. It specifies the method to find
%                               suitable learning rate. Possible options
%                               include:
%                                 - "weak-wolfe" - Search for a learning
%                                   rate which satisfies the weak Wolfe
%                                   conditions. This method maintains a
%                                   positive definite approximation of the
%                                   inverse Hessian.
%                                 - "strong-wolfe" - Search for a learning
%                                   rate which satisfies the strong Wolfe
%                                   conditions. This method maintains a
%                                   positive definite approximation of the
%                                   inverse Hessian.
%                                 - "backtracking" - Search for a learning
%                                   rate which satisfies sufficient decrease
%                                   conditions. This method does not maintain
%                                   a positive definite approximation of the
%                                   inverse Hessian.
%                               The default value is "weak-wolfe".
%       'MaxNumLineSearchIterations'
%                             - This parameter only applies if the solver
%                               is 'lbfgs'. It specifies the maximum number
%                               of line search iterations, specified as a
%                               positive integer. The default value is 20.
%       'GradientTolerance'   - This parameter only applies if the solver
%                               is 'lbfgs'. It specifies the relative
%                               gradient tolerance, specified as a
%                               nonnegative scalar. The default value is
%                               1e-5.
%       'StepTolerance'       - This parameter only applies if the solver
%                               is 'lbfgs'. It specifies the step tolerance,
%                               specified as a nonnegative scalar. The
%                               default value is 1e-5.
%       'InitialLearnRate'    - The initial learning rate that is used for
%                               training. If the learning rate is too low,
%                               training will take a long time, but if it
%                               is too high, the training is likely to get
%                               stuck at a suboptimal result. The default
%                               is 0.01 for solver 'sgdm' and 0.001 for
%                               solvers 'adam' and 'rmsprop'. This
%                               parameter does not apply for 'lbfgs'.
%       'LearnRateSchedule'   - This option allows the user to specify a
%                               method for lowering the global learning
%                               rate during training. Possible options
%                               include:
%                                 - 'none' - The learning rate does not
%                                   change and remains constant.
%                                 - 'piecewise' - The learning rate is
%                                   multiplied by a factor every time a
%                                   certain number of epochs has passed.
%                                   The multiplicative factor is controlled
%                                   by the parameter 'LearnRateDropFactor',
%                                   and the number of epochs between
%                                   multiplications is controlled by
%                                   'LearnRateDropPeriod'.
%                               The default is 'none' for 'adam', 'sgdm',
%                               and 'rmsprop'. This parameter does not
%                               apply for 'lbfgs'.
%       'LearnRateDropFactor' - This parameter only applies if the
%                               'LearnRateSchedule' is set to 'piecewise'.
%                               It is a multiplicative factor that is
%                               applied to the learning rate every time a
%                               certain number of epochs has passed.
%                               The default is 0.1 for 'adam', 'sgdm', and
%                               'rmsprop'. This parameter does not apply
%                               for 'lbfgs'.
%       'LearnRateDropPeriod' - This parameter only applies if the
%                               'LearnRateSchedule' is set to 'piecewise'.
%                               The learning rate drop factor will be
%                               applied to the global learning rate every
%                               time this number of epochs is passed. The
%                               default is 10 for 'adam', 'sgdm', and
%                               'rmsprop'. This parameter does not apply
%                               for 'lbfgs'.
%       'L2Regularization'    - The factor for the L2 regularizer. It
%                               should be noted that each set of parameters
%                               in a layer can specify a multiplier for
%                               this L2 regularizer. The default is 0.0001.
%       'GradientThresholdMethod'
%                             - Method used for gradient thresholding
%                               Possible options are:
%                                 - 'global-l2norm' - If the global L2 norm
%                                   of the gradients, considering all
%                                   learnable parameters, is larger than
%                                   GradientThreshold, then scale all
%                                   gradients by a factor of
%                                   GradientThreshold/L where L is the
%                                   global L2 norm.
%                                 - 'l2norm' - If the L2 norm of the
%                                   gradient of a learnable parameter is
%                                   larger than GradientThreshold, then
%                                   scale the gradient so that its norm
%                                   equals GradientThreshold.
%                                 - 'absolute-value' - If the absolute value
%                                   of the individual partial derivatives
%                                   in the gradient of a learnable
%                                   parameter are larger than
%                                   GradientThreshold, then scale them to
%                                   have absolute value equal to
%                                   GradientThreshold and retain its sign.
%                               The default is 'l2norm'.
%       'GradientThreshold'   - Positive threshold for the gradient. The
%                               default is Inf.
%       'MaxEpochs'           - The maximum number of epochs that will be
%                               used for training. The default is 30 for
%                               'adam', 'sgdm', and 'rmsprop'. This
%                               parameter does not apply for 'lbfgs'.
%       'MaxIterations'       - Maximum number of iterations to use for
%                               training. This parameter only applies when
%                               the solver is 'lbfgs'.
%                               The default is 1000. 
%       'MiniBatchSize'       - The size of the mini-batch used for each
%                               training iteration. The default is 128 for
%                               'adam', 'sgdm', and 'rmsprop'. This
%                               parameter does not apply for 'lbfgs'.
%       'Verbose'             - If this is set to true, information on
%                               training progress will be printed to the
%                               command window. The default is true.
%       'VerboseFrequency'    - This only has an effect if 'Verbose' is set
%                               to true. It specifies the number of
%                               iterations between printing to the command
%                               window. The default is 50.
%       'ValidationData'      - Data to use for validation during training.
%                               This can be:
%                                 - A datastore with categorical labels or
%                                   numeric responses
%                                 - A table, where the first column
%                                   contains either image paths or images
%                                 - A cell array {X, Y}, where X is a
%                                   numeric array with the input data and Y
%                                   is an array of responses
%       'ValidationFrequency' - Number of iterations between evaluations of
%                               validation metrics. This only has an effect
%                               if you also specify 'ValidationData'. The
%                               default is 50.
%       'ValidationPatience'  - Number of times that the validation metric
%                               is allowed to be the same value or worse
%                               than the last best metric before network
%                               training is stopped, specified as a positive
%                               integer or Inf. The software aims to maximize
%                               or minimize the metric as specified by the 
%                               Maximize property of the metric. Use the 
%                               ObjectiveMetricName training option to specify
%                               which metric to use for early stopping.
%                               The default is Inf.
%       'Shuffle'             - This controls if the training data is
%                               shuffled. The options are:
%                                 - 'never'- No shuffling is applied.
%                                 - 'once' - The data will be shuffled once
%                                   before training.
%                                 - 'every-epoch' - The data will be
%                                   shuffled before every training epoch.
%                               The default is 'once' for 'adam', 'sgdm',
%                               and 'rmsprop'. This parameter does not
%                               apply for 'lbfgs'.
%       'CheckpointPath'      - Path to save checkpoint networks, specified
%                               as a character vector or string scalar. If
%                               CheckPointPath is '', then the software
%                               does not save checkpoints.
%                               The default is ''.
%       'CheckpointFrequency' - Frequency of saving checkpoint networks,
%                               specified as a positive integer. To save
%                               checkpoint networks, specify a path using
%                               the 'CheckpointPath' training option.
%                               The default is 1.
%       'CheckpointFrequencyUnit' 
%                             - Unit of 'CheckpointFrequency', specified
%                               as one of the following:
%                                 - 'epoch'     - Save checkpoints every
%                                                 CheckpointFrequency
%                                                 epochs.
%                                 - 'iteration' - Save checkpoints every 
%                                                 CheckpointFrequency
%                                                 iterations.
%                               The default is 'epoch' for 'adam', 'sgdm',
%                               and 'rmsprop'. This parameter does not
%                               apply for 'lbfgs'.
%       'ExecutionEnvironment'
%                             - The execution environment for the
%                               network. This determines what hardware
%                               resources will be used to train the
%                               network. To use a GPU, you must have 
%                               Parallel Computing Toolbox(TM) and a 
%                               supported GPU device. To use a compute 
%                               cluster, you must have Parallel Computing 
%                               Toolbox.
%                                 - 'auto' - Use a GPU if it is
%                                   available, otherwise use the CPU.
%                                 - 'gpu' - Use the GPU.
%                                 - 'cpu' - Use the CPU.
%                                 - 'multi-gpu' - Use multiple GPUs on one
%                                   machine, using a local parallel pool.
%                                   If no pool is open, one is opened with
%                                   one worker per supported GPU device.
%                                 - 'parallel' - Use a compute cluster. If
%                                   no pool is open, one is opened using
%                                   the default cluster profile. If the
%                                   pool has access to GPUs then they will
%                                   be used and excess workers will be
%                                   idle. If the pool has no GPUs then
%                                   training will take place on all cluster
%                                   CPUs.
%                                 - 'parallel-auto' - Same as 'parallel'.
%                                 - 'parallel-cpu' - Use a CPU compute
%                                   cluster. If no pool is open, one is
%                                   opened using the default cluster
%                                   profile. If the pool has access to
%                                   GPUs, they will not be used.
%                                 - 'parallel-gpu' - Use a GPU compute
%                                   cluster. If no pool is open, one is
%                                   opened using the default cluster
%                                   profile.
%                               The default is 'auto'.
%       'WorkerLoad'          - For the 'multi-gpu' and 'parallel'
%                               execution environments. Determines how to
%                               divide up computation between GPUs or CPUs,
%                               by specifying relative load on parallel
%                               workers. For MiniBatchable Datastore inputs
%                               to training, if the 'DispatchInBackground'
%                               value is set to true, then workers with a
%                               load of 0 prefetch data in the background.
%                               Specify as one of
%                                 - an integer specifying the number, or
%                                 fraction specifying the proportion, of
%                                 workers on each machine to use for
%                                 training computation. The remaining
%                                 workers perform background prefetch, if
%                                 enabled.
%                                 - a vector with one element per worker in
%                                 the parallel pool, specifying the load
%                                 for each worker. For a vector W, each
%                                 worker gets W/sum(W) of the work. A
%                                 worker with a WorkerLoad of 0 performs
%                                 background prefetch, if enabled.
%                               For GPU environments, the default is to use
%                               one worker per GPU for training
%                               computation, with the remaining available
%                               for background prefetch. For CPU
%                               environments, the default is to use all
%                               workers for computation, except when
%                               'DispatchInBackground' is enabled, in which
%                               case one worker on each machine is set
%                               aside for background prefetch. This
%                               parameter does not apply for 'lbfgs'.
%       'OutputFcn'           - Specifies one or more functions to be
%                               called during training at the end of each
%                               iteration. Typically, you might use an
%                               output function to display or plot progress
%                               information, or determine whether training
%                               should be terminated early. The function
%                               will be passed a struct containing
%                               information from the current iteration. It
%                               may also return true, which will trigger
%                               early termination.
%       'Metrics'             - Metrics to calculate at the end of each iteration, specified as: 
%                                 - Built-in metric name: "accuracy", "rmse",
%                                   "fscore", "recall", "precision", or 
%                                   "auc", or a string array of names. 
%                                 - Function handle
%                                 - DifferentiableFunction object
%                                 - Metric object 
%                                 - Cell array of names, function handles, 
%                                   DifferentiableFunction objects, or
%                                   metric objects.
%       'ObjectiveMetricName' - Name of metric to use for early stopping 
%                               and returning the best network, specified 
%                               as "loss" or the name of a metric specified
%                               in the 'Metrics' training option.
%                               The default is 'loss'.
%       'Plots'               - Plots to display during training, specified
%                               as 'training-progress' or 'none' (default)
%       'SequenceLength'      - Strategy to determine the length of the
%                               sequences used per mini-batch. Options are:
%                                 - 'longest' to pad all sequences in a
%                                   batch to the length of the longest
%                                   sequence.
%                                 - 'shortest' to truncate all sequences in
%                                   a batch to the length of the shortest
%                                   sequence.
%                                 - Positive integer - For each mini-batch,
%                                   pad the sequences to the nearest
%                                   multiple of the specified length that
%                                   is greater than the longest sequence
%                                   length in the mini-batch, and then
%                                   split the sequences into smaller
%                                   sequences of the specified length. If
%                                   splitting occurs, then the software
%                                   creates extra mini-batches.
%                                The default is 'longest'.
%       'SequencePaddingValue' - Scalar value used to pad sequences where
%                                necessary. The default is 0.
%       'SequencePaddingDirection'
%                              - Direction of padding or truncation,
%                                specified as:
%                                 - 'right' - Pad or truncate sequences
%                                 on the right.
%                                 - 'left' - Pad or truncate sequences on
%                                 the left.
%                                The default is 'right'.
%       'InputDataFormats'     - Formats of input data, specified as a 
%                                string array, cell array of character 
%                                vectors, or a character vector. Each
%                                format must contain the labels "S", 
%                                "C", "B", "T", and "U", only.
%                                The default is "auto".
%       'TargetDataFormats'    - Formats of targets passed to loss
%                                function, specified as a string array,
%                                cell array of character vectors, or a
%                                character vector. Each format must 
%                                contain the labels "S", "C", "B", "T",
%                                and "U", only.
%                                The default is "auto".
%       'DispatchInBackground' - Scalar logical used to control whether
%                                asynchronous pre-fetch queueing is
%                                used when reading training data. The
%                                default is false for 'adam', 'sgdm',
%                                and 'rmsprop'. This parameter does not
%                                apply for 'lbfgs'.
%                                Requires Parallel Computing Toolbox.
%       'ResetInputNormalization'
%                              - Flag to reset input layer normalization,
%                                specified as one of the following:
%                                  - true - Clear the input layer statistics
%                                    and recalculate them during training.
%                                  - false - Do not clear existing input
%                                    layer statistics and calculate values
%                                    for empty statistics only.
%                                The default is true.
%       'BatchNormalizationStatistics'
%                              - Mode to evaluate the statistics in batch
%                                normalization layers, specified as one of
%                                the following:
%                                  - 'population' - use the population
%                                    statistics. This requires a
%                                    finalization phase after training.
%                                  - 'moving' - approximate the statistics
%                                    using a running estimate accumulated
%                                    during training. To use this option,
%                                    'ExecutionEnviroment' must be 'auto',
%                                    'cpu', or 'gpu'.
%                                  - 'auto' - use 'population' in trainNetwork
%                                     and 'moving' in trainnet.
%                                The default is 'auto'.
%       'OutputNetwork'        - Network to return when training ends,
%                                specified as one of the following:
%                                  - 'last-iteration' - return the network
%                                    corresponding to the last training
%                                    iteration.
%                                  - 'best-validation' - return the network
%                                    at the iteration corresponding to the
%                                    best validation metric. The metric 
%                                    is specified by the ObjectiveMetricName.
%                                    this option, you must also specify
%                                    'ValidationData'.
%                                  - 'auto' - For trainnet, if ValidationData
%                                    is specified, use 'best-validation'. 
%                                    Otherwise, use 'last-iteration'. For 
%                                    trainNetwork, use 'last-iteration'.
%                                The default is 'auto'.
%
%
%   options can also be created using trainingOptions and setting the
%   parameters via dot notation. For example: 
%     % Create and set parameters using trainingOptions
%     options = trainingOptions('adam', 'MaxEpochs', 30, ...
%         'LearnRateSchedule', 'piecewise')
%   
%     % Create same options using trainingOptions and dot notation
%     options = trainingOptions('adam');
%     options.MaxEpochs = 30;
%     options.LearnRateSchedule = 'piecewise';
% 
%   Example:
%       % Create a set of training options for training with stochastic
%       % gradient descent with momentum. The learning rate will be reduced
%       % by a factor of 0.2 every 5 epochs. Each iteration will use a 
%       % mini-batch with 300 observations.
%
%       options = trainingOptions('sgdm', ...
%           'Plots', 'training-progress', ...
%           'LearnRateSchedule', 'piecewise', ...
%           'LearnRateDropFactor', 0.2, ...
%           'LearnRateDropPeriod', 5, ...
%           'MiniBatchSize', 300);
%
%       % Use dot notation to set the MaxEpochs value to 30 and the
%       % GradientThresholdMethod value to 'global-l2norm'.
% 
%       options.MaxEpochs = 30;
%       options.GradientThresholdMethod = 'global-l2norm';
% 
%   See also nnet.cnn.TrainingOptionsSGDM, trainNetwork.

%   Copyright 2015-2023 The MathWorks, Inc.

if (strcmp(solverName,'sgdm'))
    opts = nnet.cnn.TrainingOptionsSGDM(varargin{:});
elseif (strcmp(solverName,'adam'))
    opts = nnet.cnn.TrainingOptionsADAM(varargin{:});
elseif (strcmp(solverName,'rmsprop'))
    opts = nnet.cnn.TrainingOptionsRMSProp(varargin{:});
elseif (strcmp(solverName,'lbfgs'))
    opts = nnet.cnn.TrainingOptionsLBFGS(varargin{:});
else
    exception = iCreateExceptionFromErrorID('nnet_cnn:trainingOptions:InvalidSolverName');
    throwAsCaller(exception);
end
end

function exception = iCreateExceptionFromErrorID(errorID)
exception = MException(errorID, getString(message(errorID)));
end
