classdef RegressionNeuralNetwork < ...
       classreg.learning.regr.FullRegressionModel & classreg.learning.regr.CompactRegressionNeuralNetwork
%RegressionNeuralNetwork Neural network for regression.
%   RegressionNeuralNetwork is a neural network for regression. It
%   can predict response for new data. It also stores data used for
%   training and can compute resubstitution predictions.
%
%   An object of this class cannot be created by calling the constructor.
%   Use FITRNET to create a RegressionNeuralNetwork object by fitting the neural network to
%   training data.
%
%   This class is derived from CompactRegressionNeuralNetwork.
%
%   RegressionNeuralNetwork properties:
%       NumObservations               - Number of observations.
%       X                             - Matrix or table of predictors used to train this neural network.
%       Y                             - Observed response used to train this neural network.
%       W                             - Weights of observations used to train this neural network.
%       ModelParameters               - Neural network parameters.
%       PredictorNames                - Names of predictors used for this neural network.
%       ExpandedPredictorNames        - Names of expanded predictors used for this neural network.
%       CategoricalPredictors         - Indices of categorical predictors.
%       ResponseName                  - Name of the response variable.
%       ResponseTransform             - Transformation applied to predicted regression response.
%       ConvergenceInfo               - Information on the convergence of the neural network training.
%       Solver                        - Solver used for training.
%       TrainingHistory               - Table containing training record
%       LayerSizes                    - Size of the fully connected layers.
%       Activations                   - Name of the activation function used after each fully connected layer.
%       LayerWeights                  - Trained weights for all the layers in the neural network.
%       LayerBiases                   - Trained biases for all the layers in the neural networks.
%       OutputLayerActivation         - Name of the activation function used after the final fully connected layer.
%
%   RegressionNeuralNetwork methods:
%       compact               - Compact this neural network.
%       crossval              - Cross-validate this neural network.
%       loss                  - Regression loss.
%       predict               - Predicted response of this neural network.
%       partialDependence     - Partial Dependence values.
%       plotPartialDependence - Plot Partial Dependence.
%       resubLoss             - Resubstitution regression loss.
%       resubPredict          - Resubstitution predicted response.
%
%   Example: Grow a regression neural network for car data.
%       % Using matrices
%       load carsmall
%       t = fitrnet([Weight Horsepower],MPG,'PredictorNames',{'Weight' 'Horsepower'})
%       view(t)
%
%       % Using a table
%       tbl = table(Weight,Horsepower,MPG);
%       t2 = fitrnet(tbl,'MPG')
%       view(t,'mode','graph')
%
%   See also fitrnet, classreg.learning.regr.CompactRegressionNeuralNetwork.

%   Copyright 2020-2023 The MathWorks, Inc.

    properties(SetAccess = private,GetAccess = public)
        ConvergenceInfo = struct;
        Solver = 'LBFGS';
    end
    
     properties(SetAccess = private,GetAccess = private)
        LossFunction = @mlearnlib.internal.nnet.loss.mse;
        LossFunctionName = 'mse';
    end
    
    properties(Dependent, SetAccess = private, GetAccess = public)
        TrainingHistory;
    end
    
    methods
        function history = get.TrainingHistory(this)
            history = this.ConvergenceInfo.History;
        end
    end
    methods(Hidden)
        function [Weights,Biases] = getLearnableParams(this)
            Weights = this.NeuralNetworkImpl.LayerWeights;
            Biases = this.NeuralNetworkImpl.LayerBiases;
        end

        function this = setLearnableParams(this,Weights,Biases)
            this.NeuralNetworkImpl.LayerWeights = Weights;
            this.NeuralNetworkImpl.LayerBiases = Biases;
        end
    end
    
    methods(Hidden)
        function this = RegressionNeuralNetwork(X,Y,W,modelParams,...
                dataSummary,responseTransform)
            
            this = this@classreg.learning.regr.FullRegressionModel(...
                X,Y,W,modelParams,dataSummary,responseTransform);
            
            this = this@classreg.learning.regr.CompactRegressionNeuralNetwork(...
                dataSummary,responseTransform,[],[],[],[],[],[],[],[]);
            
            this.LayerSizes = this.ModelParameters.LayerSizes;
            this.Activations = this.ModelParameters.Activations;
            
            if ~isfloat(X)
                error(message('stats:fitnet:InvalidX'));
            end
            
            % Compute default response values
            this.DefaultResponse.Median = classreg.learning.internal.wnanmedian(this.PrivY,this.W);
            this.DefaultResponse.Mean   = classreg.learning.internal.wnanmean(this.PrivY,this.W);

            % Find rows with NaNs and don't use them for training.
            [this,X,Y,W] = processNaNX(this,'stats:fitnet:NoDataNaN');

            % Process categoricals
            [trainX,catPreds] = this.dummifyCategoricals(X);
            
            valX = [];
            
            % Missing values have been removed from validation data in
            % NeuralNetworkParams
            if ~isempty(this.ModelParams.ValidationX)
                [valX,~] = this.dummifyCategoricals(this.ModelParams.ValidationX);
            end
            
            if this.ModelParams.StandardizeData
                this.StandardizeData = true;
                [this,trainX] = this.standardizeTrainData(trainX,W,catPreds);
                if ~isempty(this.ModelParams.ValidationX)
                    valX = this.standardize(valX);
                end
            end
            
            if this.DataSummary.ObservationsInRows
                this.NumPredictors = size(trainX,2);
            else
                this.NumPredictors = size(trainX,1);
            end

            this.NeuralNetworkImpl = this.createNeuralNetwork();
            
            this.ConvergenceInfo = this.NeuralNetworkImpl.intializeConvergenceInfoAndHistory();
                        
            YVal = this.ModelParameters.ValidationY';
            
            this.ExpandedActivations = this.expandActivations(this.ModelParams);
            [this.NeuralNetworkImpl,this.ConvergenceInfo] =  this.NeuralNetworkImpl.trainNetwork(trainX,...
                valX,Y',YVal,this.ModelParameters,this.ConvergenceInfo,...
                this.DataSummary.ObservationsInRows,W,this.ExpandedActivations);
        end
    end
    
    methods(Access=protected)
        function s = propsForDisp(this,s)
            s = propsForDisp@classreg.learning.regr.FullRegressionModel(this,s);
            s = propsForDisp@classreg.learning.regr.CompactRegressionNeuralNetwork(this,s);
            s.Solver = this.Solver;
            s.ConvergenceInfo = this.ConvergenceInfo;
            if this.ModelParameters.StoreHistory
                s.TrainingHistory = this.TrainingHistory;
            end
        end
        
        function n = getExpandedPredictorNames(this)
            n = classreg.learning.internal.expandPredictorNamesWithReference(...
                   this.PredictorNames,this.CategoricalPredictors,...
                   this.OrdinalPredictors,this.VariableRange);
        end
    end

    methods(Static,Hidden)        
        function this = fit(X,Y,varargin)
            mdl = classreg.learning.FitTemplate.make(...
                'NeuralNetwork','type','regression',varargin{:});
            this = fit(mdl,X,Y);
        end

        function [X,Y,W,dataSummary,responseTransform] = prepareData(X,Y,varargin)
            [X,Y,vrange,wastable,varargin] = classreg.learning.internal.table2FitMatrix(X,Y,varargin{:});
            
            % Process input args
            args = {'responsetransform'};
            defs = {                 []};
            [transformer,~,crArgs] = ...
                internal.stats.parseArgs(args,defs,varargin{:});
            
            % Check Y type
            if ~isfloat(Y) || ~isvector(Y)
                error(message('stats:classreg:learning:regr:FullRegressionModel:prepareData:BadYType'));
            end
            internal.stats.checkSupportedNumeric('Y',Y,true,false,false,true);
            Y = Y(:);

            % Pre-process
            [X,Y,W,dataSummary] = ...
                classreg.learning.FullClassificationRegressionModel.prepareDataCR(...
                X,Y,crArgs{:},'VariableRange',vrange,'TableInput',wastable);

            if ~dataSummary.TableInput
                 X = classreg.learning.internal.encodeCategorical(X,dataSummary.VariableRange,dataSummary.ObservationsInRows);
            end
            
            if ~isfloat(X) && ~isUnderlyingType(X,'int32')
                error(message('stats:classreg:learning:regr:FullRegressionModel:prepareData:BadXType'));
            end

            [X,Y,W,dataSummary] = classreg.learning.regr.FullRegressionModel.removeNaNs(X,Y,W,dataSummary);
            
            % Renormalize weights
            W = W/sum(W);

            % Make output response transformation
           responseTransform = ...
                classreg.learning.regr.FullRegressionModel.processResponseTransform(transformer);
        end
                
    end
    
    methods
        function cmp = compact(this)
            %CMP = compact(MODEL) returns a CompactRegressionNeuralNetwork object
            %     holding the structure of the trained neural network model. The compact object does not
            %     contain X and Y used for training.
            %  
            %     See also fitrnet, RegressionNeuralNetwork,
            %     classreg.learning.regr.CompactRegressionNeuralNetwork.
            % 
            %     Documentation for RegressionNeuralNetwork/compact
            
            cmp = classreg.learning.regr.CompactRegressionNeuralNetwork(...
                this.DataSummary,this.PrivResponseTransform,...
                this.NeuralNetworkImpl,this.StandardizeData,this.Mu,this.Sigma,...
                this.NumPredictors,...
                this.ModelParams.LayerSizes,this.ModelParams.Activations,this.DefaultResponse);
        end
        
        function [varargout] = resubPredict(this,varargin)
        % resubPredict Predict resubstitution response.
        %     YFIT = resubPredict(MDL) returns the predicted response YFIT for the
        %     regression neural network MDL and training data MDL.X. YFIT is a
        %     numeric vector with NumObservations elements.
        %  
        %   YFIT = resubPredict(MDL,'NAME1',VALUE1,'NAME2',VALUE2,...)
        %   specifies optional name/value arguments:
        %       'PredictionForMissingValue' - Numeric scalar, character vector, or string scalar. 
        %                                     If a character vector or string scalar, must be one of: 
        %                                     'median' or 'mean'. If a numeric scalar, resubPredict uses
        %                                     this response for observations with missing predictor 
        %                                     values. If 'mean' or 'median', resubPredict uses the mean 
        %                                     or median of observed response values in the training data. 
        %                                     Default: 'median'
        %
        %     See also fitrnet, RegressionNeuralNetwork, predict.
       
            [varargout{1:nargout}] = ...
                resubPredict@classreg.learning.regr.FullRegressionModel(this,varargin{:});
        end
        
        function [varargout] = resubLoss(this,varargin)
        %resubLoss Regression error by resubstitution.
        %     L = resubLoss(MDL) returns the mean squared error for the regression
        %     neural network MDL, computed using the training data MDL.X and MDL.Y.
        %  
        %     L = resubLoss(MDL,'PARAM1',val1,'PARAM2',val2,...) specifies
        %     optional name/value arguments:
        %         'LossFun'          - Function handle for loss function, or
        %                              character vector or string scalar representing a built-in loss
        %                              function. The only available loss function for
        %                              regression is 'mse'. If you pass a
        %                              function handle FUN, LOSS calls it as
        %                              shown below:
        %                                 FUN(Y,Yfit,W)
        %                              where Y, Yfit and W are numeric
        %                              vectors of length N. Y is observed
        %                              response, Yfit is predicted response,
        %                              and W is observation weights. Default:
        %                              'mse'
        %       'PredictionForMissingValue' - Numeric scalar, character vector, or
        %                                     string scalar. If a character vector or
        %                                     string scalar, must be one of:
        %                                     'median', 'mean', or 'omitted'. If a
        %                                     numeric scalar, resubLoss uses this response 
        %                                     for observations with missing predictor 
        %                                     values. If 'mean' or 'median', resubLoss uses 
        %                                     the mean or median of observed response 
        %                                     values in the training data. If 'omitted',
        %                                     resubLoss excludes observations with missing 
        %                                     predictor values from the computation. 
        %                                     Default: 'median'
        %  
        %     See also RegressionNeuralNetwork, loss.

            [varargout{1:nargout}] = ...
                resubLoss@classreg.learning.regr.FullRegressionModel(this,varargin{:});
        end
    end

    methods(Hidden)
        function [YPRED,dX,dWEIGHTS,dBIASES] = predictionDerivatives(this,X)
            %PREDICTIONDERIVATIVES Compute derivatives of prediction of the neural network.
            %   YPRED = predictionDerivatives(MDL,X) returns the predicted response
            %   YPRED for regression neural network model MDL and predictors X. X is a 
            %   N by P matrix, where P is the number of predictors used for training.
            %   YPRED is a numeric vector with N elements, where N is the number of rows in X.
            %
            %   NOTE: This method only supports the case when MDL was
            %   originally created by training on a matrix X. If MDL was
            %   created by training on a table X then this method should
            %   not be used. Also, this method assumes there were no
            %   categorical predictors when MDL was trained. If X includes
            %   categorical predictors, this method should not be used.
            %   This method also assumes the data was not standardized as
            %   part of the training process. The function will also only
            %   work if the model was trained on data in rows.
            %
            %   [YPRED,dX,dWEIGHTS,dBIASES] = predictionDerivatives(MDL,X) also
            %   returns the derivatives of YPRED w.r.t WEIGHTS, BIASES and inputs to
            %   the neural network model MDL. If YPRED has N elements then dWEIGHTS and
            %   dBIASES are 1-by-M cell arrays such that dWEIGHTS{i} and dBIASES{i} are
            %   the derivatives of YPRED w.r.t WEIGHTS{i} and BIASES{i}. If WEIGHTS{i}
            %   is a Q1-by-Q2 array then dWEIGHTS{i} is a Q1-by-Q2-by-N array such that
            %   dWEIGHTS{i}(:,:,j) is the derivatives of YPRED(j) w.r.t WEIGHTS{i}. The
            %   same applies to dBIASES.
            % 
            %   Output dX is a N-by-P array where dX(i,:) is the
            %   derivative of YPRED(i) w.r.t X(i,:) where X(i,:) is the
            %   1-by-P vector of predictors.

            % Check that the model was not trained on any categorical predictors
            if numel(this.CategoricalPredictors) ~= 0
                error(message('stats:fitnet:CategoricalModelNotSupported'));
            end

            % Check that the model was not trained on tabular input
            if this.TableInput
                error(message('stats:fitnet:TabularModelNotSupported'));
            end

            % Check that the model's training data was not standardized as
            % part of the training.
            if this.StandardizeData
                error(message('stats:fitnet:StandarizedModelNotSupported'));
            end

            % Check that the model's data is in rows not columns
            if ~this.DataSummary.ObservationsInRows
                error(message('stats:fitnet:ColumnDataModelNotSupported'));
            end

            % Check that X is not a table
            if  istable(X)
                error(message('stats:fitnet:InvalidInputX'));
            end
            
            % Get parameters
            W = this.LayerWeights;
            B = this.LayerBiases;

            this.ExpandedActivations = this.expandActivations(this.ModelParams);
            numActivations = numel(this.ExpandedActivations);
            activationsInfo = cell(1,numActivations);
            for i = 1:numActivations
                activationsInfo{i} = mlearnlib.internal.nnet.activation.create(this.ExpandedActivations{i});
            end
            
            % Calculate derivatives
            if nargout > 1
               [YPRED,dX,dWEIGHTS,dBIASES] = mlearnlib.internal.nnet.feedForwardDerivatives(X',W,B,activationsInfo);
               YPRED = YPRED';
               dX = dX';
            else
               YPRED = mlearnlib.internal.nnet.feedForwardDerivatives(X',W,B,activationsInfo);
               YPRED = YPRED';
            end
        end
    end


    methods(Access = private)
        function impl = createNeuralNetwork(this)
           impl = classreg.learning.impl.NeuralNetworkImpl.make(...
            this.ModelParameters.LayerSizes,...
            this.ModelParameters.Activations,1,this.NumPredictors,...
            this.ModelParameters.LayerWeightsInitializer,...
            this.ModelParameters.LayerBiasesInitializer,...
            this.PrivX,this.LossFunction,this.LossFunctionName);
        end

        function [this,X] = standardizeTrainData(this,X,w,catPreds)
            if ~all(catPreds)
                if this.DataSummary.ObservationsInRows
                    this.Mu = zeros(1,size(X,2),'like',X);
                    this.Sigma = ones(1,size(X,2),'like',X);
                    [this.Sigma(~catPreds),this.Mu(~catPreds)] = classreg.learning.internal.wnanvar(X(:,~catPreds),w,1,this.DataSummary.ObservationsInRows);
                else
                    this.Mu = zeros(size(X,1),1,'like',X);
                    this.Sigma = ones(size(X,1),1,'like',X);
                    [this.Sigma(~catPreds),this.Mu(~catPreds)] = classreg.learning.internal.wnanvar(X(~catPreds,:),w,1,this.DataSummary.ObservationsInRows);
                end
                nonZeroSigma = this.Sigma>0;
                this.Sigma(~nonZeroSigma) = 0;
                this.Sigma = sqrt(this.Sigma);
                X = this.standardize(X);
            end   
        end

        function activations = expandActivations(this,modelParams)
            if ~iscell(modelParams.Activations)
                activations = {modelParams.Activations};
            else
                activations = modelParams.Activations;
            end
            
            numActivations = length(activations);
            numHiddenLayers = length(modelParams.LayerSizes);
            if numActivations == 1 && numHiddenLayers ~= numActivations
                activations = repmat(activations,[1 numHiddenLayers]);
            end
            
            activations{end + 1} = this.OutputLayerActivation;
            
            %LBFGS Solver uses 'linear'
            for i = 1:length(activations)
                if strcmp(activations{i},'none')
                    activations{i} = 'linear';
                end
            end
        end
    end
end

% LocalWords:  fitnet PREDICTIONDERIVATIVES Standarized
