classdef ClassificationNeuralNetwork < ...
        classreg.learning.classif.FullClassificationModel & ...
        classreg.learning.classif.CompactClassificationNeuralNetwork
    %ClassificationNeuralNetwork Neural network for classification.
    %   ClassificationNeuralNetwork is a neural network for classification.
    %   It also stores data used for training and can compute resubstitution predictions.
    %
    %   An object of this class cannot be created by calling the constructor.
    %   Use FITCNET to create a ClassificationNeuralNetwork object by fitting the
    %   neural network to training data.
    %
    %   This class is derived from CompactClassificationNeuralNetwork.
    %
    %   ClassificationNeuralNetwork properties:
    %       NumObservations               - Number of observations.
    %       X                             - Matrix or table of predictors used to train this neural network.
    %       Y                             - True class labels used to train this neural network.
    %       W                             - Weights of observations used to train this neural network.
    %       ModelParameters               - Neural network parameters.
    %       PredictorNames                - Names of predictors used for this neural network.
    %       ExpandedPredictorNames        - Names of expanded predictors used for this neural network.
    %       CategoricalPredictors         - Indices of categorical predictors.
    %       ResponseName                  - Name of the response variable.
    %       ClassNames                    - Names of classes in Y.
    %       Cost                          - Misclassification costs.
    %       Prior                         - Prior class probabilities.
    %       ScoreTransform                - Transformation applied to predicted classification scores.
    %       ConvergenceInfo               - Information on the convergence of the neural network training.
    %       Solver                        - Solver used for training.
    %       TrainingHistory               - Table containing training record
    %       LayerSizes                    - Size of the fully connected layers.
    %       Activations                   - Name of the activation function used after each fully connected layer.
    %       LayerWeights                  - Trained weights for all the layers in the neural network.
    %       LayerBiases                   - Trained biases for all the layers in the neural networks.
    %       OutputLayerActivation         - Name of the activation function used after the final fully connected layer.
    %
    %   ClassificationNeuralNetwork methods:
    %       compact               - Compact this neural network.
    %       compareHoldout        - Compare two models using test data.
    %       crossval              - Cross-validate this neural network.
    %       edge                  - Classification edge.
    %       loss                  - Classification loss.
    %       margin                - Classification margins.
    %       partialDependence     - Partial Dependence values.
    %       plotPartialDependence - Plot Partial Dependence.
    %       predict               - Predicted response of this neural network.
    %       resubEdge             - Resubstitution classification edge.
    %       resubLoss             - Resubstitution classification loss.
    %       resubMargin           - Resubstitution classification margins.
    %       resubPredict          - Resubstitution predicted response.
    %
    %   Example: Grow a classification neural network for Fisher's iris data.
    %       Tbl = readtable('fisheriris.csv');
    %       Mdl = fitcnet(Tbl,'Species');
    %       confusionchart(Tbl.Species,predict(Mdl,Tbl))
    %
    %   See also fitcnet,
    %   classreg.learning.classif.CompactClassificationNeuralNetwork.

    %   Copyright 2020-2023 The MathWorks, Inc.

    properties(SetAccess = private,GetAccess = public)
        ConvergenceInfo = struct;
        Solver = 'LBFGS';
    end

    properties(SetAccess = private,GetAccess = private)
        LossFunction = @mlearnlib.internal.nnet.loss.crossentropy;
        LossFunctionName = 'crossentropy';
    end

    properties(Dependent, SetAccess = private, GetAccess = public)
        TrainingHistory;
    end

    methods
        function history = get.TrainingHistory(this)
            history = this.ConvergenceInfo.History;
        end
    end

    methods(Hidden)
        function this = ClassificationNeuralNetwork(X,Y,W,modelParams,...
                dataSummary,classSummary,scoreTransform)

            this = this@classreg.learning.classif.FullClassificationModel(...
                X,Y,W,modelParams,dataSummary,classSummary,scoreTransform);

            this = this@classreg.learning.classif.CompactClassificationNeuralNetwork(...
                dataSummary,classSummary,scoreTransform,[],[],[],[],[],[],[],[]);

            this.LayerSizes = this.ModelParameters.LayerSizes;
            this.Activations = this.ModelParameters.Activations;

            if ~isfloat(X)
                error(message('stats:fitnet:InvalidX'));
            end

            % Make temp copies
            X = this.PrivX;
            Y = this.PrivY;
            W = this.W;
            classSummary = this.ClassSummary;
            nonzeroClassNames = this.ClassSummary.NonzeroProbClasses;
            dataSummary = this.DataSummary;

            % Find rows with NaNs and don't use them for training.
            obsInRows = dataSummary.ObservationsInRows;
            if obsInRows
                nanX = any(isnan(X),2);
            else
                nanX = any(isnan(X),1);
            end
            hasNaN = any(nanX);
            removeNaN = dataSummary.RemoveObservationsWithMissingValues;
            if hasNaN
                if obsInRows
                    X(nanX,:) = [];
                else
                    X(:,nanX) = [];
                end
                Y(nanX)   = [];
                W(nanX)   = [];
                W = W/sum(W);
                this.HasNaNX = nanX;
                if removeNaN
                    rowsused = dataSummary.RowsUsed;
                    if isempty(rowsused)
                        rowsused = ~nanX;
                    else
                        rowsused(rowsused) = ~nanX;
                    end
                    dataSummary.RowsUsed = rowsused;
                end
            end
            
            if isempty(X)
                error(message('stats:fitnet:NoDataNaN'));
            end

            if hasNaN && ~removeNaN % Operate on both XYW and object properties XYW
                [~,this.DataSummary,this.PrivX,this.PrivY,this.W] = adjustPrior(...
                    classSummary,dataSummary,nonzeroClassNames,this.PrivX,this.PrivY,this.W);
                [this.ClassSummary,~,X,~,W] = adjustPrior(...
                    classSummary,dataSummary,nonzeroClassNames,X,Y,W);
            else
                [classSummary,dataSummary,X,Y,W] = adjustPrior(...
                    classSummary,dataSummary,nonzeroClassNames,X,Y,W);
                this.PrivX = X;
                this.PrivY = Y;
                this.W     = W;
                this.ClassSummary = classSummary;
                this.DataSummary  = dataSummary;
                this.HasNaNX = false(size(this.PrivX,1),1);
            end

            [trainX,catPreds] = this.dummifyCategoricals(X);

            valX = [];

            if ~isempty(this.ModelParams.ValidationX)
                [valX,~] = this.dummifyCategoricals(this.ModelParams.ValidationX);
            end

            if this.ModelParams.StandardizeData
                this.StandardizeData = true;
                [this,trainX] = this.standardizeTrainData(trainX,W,catPreds);
                if ~isempty(this.ModelParams.ValidationX)
                    valX = this.standardize(valX);
                end
            end

            if this.DataSummary.ObservationsInRows
                this.NumPredictors = size(trainX,2);
            else
                this.NumPredictors = size(trainX,1);
            end

            this.NeuralNetworkImpl = this.createNeuralNetwork();

            this.ConvergenceInfo = this.NeuralNetworkImpl.intializeConvergenceInfoAndHistory();

            YTrain = classreg.learning.internal.classCount(this.ClassSummary.NonzeroProbClasses,Y)';
            YVal = [];
            if ~isempty(this.ModelParameters.ValidationY)
                YVal = classreg.learning.internal.ClassLabel(this.ModelParameters.ValidationY);
                YVal = classreg.learning.internal.classCount(this.ClassSummary.NonzeroProbClasses,YVal)';
            end
            this.ExpandedActivations = this.expandActivations(this.ModelParams);
            [this.NeuralNetworkImpl,this.ConvergenceInfo] =  this.NeuralNetworkImpl.trainNetwork(trainX,...
                valX,YTrain,YVal,this.ModelParameters,this.ConvergenceInfo,...
                this.DataSummary.ObservationsInRows,W,this.ExpandedActivations);
        end
    end

    methods(Access=protected)
        function s = propsForDisp(this,s)
            s = propsForDisp@classreg.learning.classif.FullClassificationModel(this,s);
            s = propsForDisp@classreg.learning.classif.CompactClassificationNeuralNetwork(this,s);
            s.Solver = this.Solver;
            s.ConvergenceInfo = this.ConvergenceInfo;
            if this.ModelParameters.StoreHistory
                s.TrainingHistory = this.TrainingHistory;
            end
        end

        function n = getExpandedPredictorNames(this)
            n = classreg.learning.internal.expandPredictorNamesWithReference(...
                this.PredictorNames,this.CategoricalPredictors,...
                this.OrdinalPredictors,this.VariableRange);
        end
    end

    methods(Static,Hidden)
        function this = fit(X,Y,varargin)

            args = {'cost'};
            defs = {    []};
            [cost,~,fitArgs] = internal.stats.parseArgs(args,defs,varargin{:});

            mdl = classreg.learning.FitTemplate.make(...
                'NeuralNetwork','type','classification',fitArgs{:});
            this = fit(mdl,X,Y);

            % Assign cost after fitting the model
            if ~isempty(cost)
                this.Cost = cost;
            end
        end

        function [X,Y,W,dataSummary,classSummary,scoreTransform] = prepareData(X,Y,varargin)
             [X,Y,vrange,wastable,varargin] = classreg.learning.internal.table2FitMatrix(X,Y,varargin{:});

            % Cast Y to the same storage type as X (for example, in memory
            % array or gpuArray). This ensures that the storage type of Y
            % does not affect how the model is fitted.
            Y = internal.stats.castToMatchingStorageTypeIfPossible(Y,X);

            % Process input args
            args = {'classnames' 'cost' 'prior' 'scoretransform', 'weights'};
            defs = {          []     []      []               [],        []};
            [userClassNames,cost,prior,transformer,W,~,crArgs] = ...
                internal.stats.parseArgs(args,defs,varargin{:});
            
            Y = classreg.learning.internal.ClassLabel(Y);

            % Get class names before any rows might be removed
            allClassNames = levels(Y);
            if isempty(allClassNames)
                error(message('stats:classreg:learning:classif:FullClassificationModel:prepareData:EmptyClassNames'));
            end

            % Require numeric X but not necessarily floating-point X.
            % The only example of non-floating-point data is
            % ClassificationKNN with user-supplied distance function.
            if ~isnumeric(X) || ~ismatrix(X)
                error(message('stats:classreg:learning:classif:FullClassificationModel:prepareData:BadXType'));
            end

            % Process Weights before arguments are passed to
            % FullClassificationRegressionModel.prepareData to ensure it
            % has the desired precision (single or double) and storage type
            % (for example, gpuArray).  It should have the same storage
            % type as X, and if weights was not specified, it should
            % default to having the same precision as X as well.
            weightsPrototype = full(internal.stats.numericPrototype(X));
            numObs = length(Y);
            if isempty(W)
                W = ones(numObs,1,"like",weightsPrototype);
            else
                % If weights is supplied ensure that the storage type of
                % the weights input (for example, gpuArray) does not
                % influence where the model is fitted.
                W = internal.stats.castToMatchingStorageTypeIfPossible(W,weightsPrototype);
            end
            
            % Pre-process
            [X,Y,W,dataSummary] = ...
                classreg.learning.FullClassificationRegressionModel.prepareDataCR(...
                X,Y,crArgs{:},'Weights',W,'VariableRange',vrange,'TableInput',wastable);
            if ~dataSummary.TableInput
                X = classreg.learning.internal.encodeCategorical(X,dataSummary.VariableRange,dataSummary.ObservationsInRows);
            end

            % Process class names
            [X,Y,W,userClassNames,nonzeroClassNames,dataSummary] = ...
                classreg.learning.classif.FullClassificationModel.processClassNames(...
                X,Y,W,userClassNames,allClassNames,dataSummary);

            % Remove missing values
            [X,Y,W,dataSummary] = ...
                classreg.learning.classif.FullClassificationModel.removeMissingVals(X,Y,W,dataSummary);
                       
            % Get matrix of class weights
            C = classreg.learning.internal.classCount(nonzeroClassNames,Y);
            WC = C.*W;
            Wj = sum(WC,1);  % This will be the sum of per class weights, used for incremental learning
                      
            % Check prior
            prior = classreg.learning.classif.FullClassificationModel.processPrior(...
                prior,Wj,userClassNames,nonzeroClassNames);

            % Get costs
            cost = classreg.learning.classif.FullClassificationModel.processCost(...
                cost,prior,userClassNames,nonzeroClassNames);
        
            % Remove observations for classes with zero prior probabilities
            [X,Y,~,WC,Wj,prior,cost,nonzeroClassNames,dataSummary] = ...
                classreg.learning.classif.FullClassificationModel.removeZeroPriorAndCost(...
                X,Y,C,WC,Wj,prior,cost,nonzeroClassNames,dataSummary);
            
            % Normalize priors in such a way that the priors in present
            % classes add up to one.  Normalize weights to add up to the
            % prior in the respective class.
            prior = prior/sum(prior);
            W = sum(WC.*prior./Wj,2);
    
            % Put processed values into summary structure
            classSummary = ...
                classreg.learning.classif.FullClassificationModel.makeClassSummary(...
                userClassNames,nonzeroClassNames,prior,cost);
                
            % Make output score transformation
            scoreTransform = ...
                classreg.learning.classif.FullClassificationModel.processScoreTransform(transformer);
        end
    end

    methods
        function cmp = compact(this)
            %COMPACT Compact neural network model.
            %   CMP = compact(MODEL) returns a
            %   CompactClassificationNeuralNetwork object holding the structure of
            %   the trained neural network classifier. The compact object does
            %   not contain the X and Y used for training.
            %
            %   See also fitcnet, ClassificationNeuralNetwork,
            %   classreg.learning.classif.CompactClassificationNeuralNetwork.

            cmp = classreg.learning.classif.CompactClassificationNeuralNetwork(...
                this.DataSummary,this.ClassSummary,this.PrivScoreTransform,[],...
                this.NeuralNetworkImpl,this.StandardizeData,this.Mu,this.Sigma,...
                this.NumPredictors,...
                this.ModelParams.LayerSizes,this.ModelParams.Activations);
        end

        function [varargout] = resubPredict(this,varargin)
            % resubPredict Predict resubstitution response.
            %     [LABEL,SCORE] = resubPredict(OBJ) returns predicted class labels
            %     LABEL and posterior probabilities SCORE for model OBJ and training data OBJ.X. Classification labels LABEL
            %     have the same type as Y used for training. Posterior probabilities
            %     SCORE are an N-by-K numeric matrix for N observations and K
            %     classes. The predicted label is assigned to the class with the maximum score.

            %   See also ClassificationNeuralNetwork/predict.

            [varargin{:}] = convertStringsToChars(varargin{:});
            [varargout{1:nargout}] = ...
                resubPredict@classreg.learning.classif.FullClassificationModel(this,varargin{:});
        end

        function [varargout] = resubLoss(this,varargin)
            % resubLoss Classification cost by resubstitution.
            %     L = resubLoss(OBJ) returns the resubstitution classification loss (or error) for model OBJ.
            %
            %     L = resubLoss(OBJ,'PARAM1',val1,'PARAM2',val2,...) specifies optional
            %     parameter name/value pairs:
            %         'LossFun'   - Function handle for loss, or character vector or
            %                       string scalar representing a built-in loss
            %                       function. Available loss functions for
            %                       classification: 'binodeviance', 'classiferror',
            %                       'classifcost', 'crossentropy', 'exponential',
            %                       'hinge', 'logit', 'mincost', and 'quadratic'. If
            %                       you pass a function handle FUN, LOSS calls it as
            %                       shown below:
            %                            FUN(C,S,W,COST)
            %                       where C is an N-by-K logical matrix for N rows in X
            %                       and K classes in the ClassNames property, S is an
            %                       N-by-K numeric matrix, W is a numeric vector with N
            %                       elements, and COST is a K-by-K numeric matrix. C has
            %                       one true per row for the true class. S is a matrix of
            %                       posterior probabilities for classes with one row per
            %                       observation, similar to POSTERIOR output from
            %                       PREDICT. W is a vector of observation weights. COST
            %                       is a matrix of misclassification costs. Default:
            %                       'mincost'
            %   See also ClassificationNeuralNetwork/loss.

            [varargin{:}] = convertStringsToChars(varargin{:});
            [varargout{1:nargout}] = ...
                resubLoss@classreg.learning.classif.FullClassificationModel(this,varargin{:});
        end

        function m = resubMargin(this,varargin)
            % resubMargin Classification margins by resubstitution.
            %     M = resubMargin(OBJ) returns resubstitution classification margins for
            %     model OBJ. The classification margin is the difference between the
            %     classification score for the true class and the maximal classification
            %     score for the false classes. The returned M is a numeric column-vector
            %     of length size(OBJ.X,1).
            %
            %     See also classreg.learning.classif.ClassificationModel/margin.

            [varargin{:}] = convertStringsToChars(varargin{:});
            classreg.learning.FullClassificationRegressionModel.catchWeights(varargin{:});
            m = margin(this,this.X,this.Y,varargin{:});
        end

        function e = resubEdge(this,varargin)
            % resubEdge Classification edge by resubstitution.
            %     E = resubEdge(OBJ) returns the classification edge obtained by model OBJ for
            %     training data OBJ.X and OBJ.Y. The classification edge is the classification
            %     margin averaged over the entire data.
            %
            %     See also ClassificationNeuralNetwork/resubMargin,
            %     classreg.learning.classif.ClassificationModel/edge.

            [varargin{:}] = convertStringsToChars(varargin{:});
            classreg.learning.FullClassificationRegressionModel.catchWeights(varargin{:});
            e = edge(this,this.X,this.Y,'Weights',this.W,varargin{:});
        end
    end

    methods(Access = private)
        function impl = createNeuralNetwork(this)
            numClasses = numel(this.ClassSummary.NonzeroProbClasses);
            impl = classreg.learning.impl.NeuralNetworkImpl.make(...
                this.ModelParameters.LayerSizes,...
                this.ModelParameters.Activations,numClasses,this.NumPredictors,...
                this.ModelParameters.LayerWeightsInitializer,...
                this.ModelParameters.LayerBiasesInitializer,...
                this.PrivX,this.LossFunction,this.LossFunctionName);
        end

        function [this,X] = standardizeTrainData(this,X,W,catPreds)
            if ~all(catPreds)
                if this.DataSummary.ObservationsInRows
                    this.Mu = zeros(1,size(X,2),'like',X);
                    this.Sigma = ones(1,size(X,2),'like',X);
                    [this.Sigma(~catPreds),this.Mu(~catPreds)] = ...
                        classreg.learning.internal.wnanvar(X(:,~catPreds),W,1,this.DataSummary.ObservationsInRows);
                else
                    this.Mu = zeros(size(X,1),1,'like',X);
                    this.Sigma = ones(size(X,1),1,'like',X);
                    [this.Sigma(~catPreds),this.Mu(~catPreds)] = ...
                        classreg.learning.internal.wnanvar(X(~catPreds,:),W,1,this.DataSummary.ObservationsInRows);
                end
                nonZeroSigma = this.Sigma>0;
                this.Sigma(~nonZeroSigma) = 0;
                this.Sigma = sqrt(this.Sigma);
                X = this.standardize(X);
            end
        end

        function activations = expandActivations(this,modelParams)
            if ~iscell(modelParams.Activations)
                activations = {modelParams.Activations};
            else
                activations = modelParams.Activations;
            end

            numActivations = length(activations);
            numHiddenLayers = length(modelParams.LayerSizes);
            if numActivations == 1 && numHiddenLayers ~= numActivations
                activations = repmat(activations,[1 numHiddenLayers]);
            end

            activations{end + 1} = this.OutputLayerActivation;

            %LBFGS Solver uses 'linear'
            for i = 1:length(activations)
                if strcmp(activations{i},'none')
                    activations{i} = 'linear';
                end
            end
        end
    end
end


function [classSummary,dataSummary,X,Y,W] = adjustPrior(classSummary,dataSummary,nonzeroClassNames,X,Y,W)

prior = classSummary.Prior;
cost  = classSummary.Cost;
if isempty(cost)
    K = numel(nonzeroClassNames);
    cost = ones(K) - eye(K);
end

% Get matrix of class weights
C = classreg.learning.internal.classCount(nonzeroClassNames,Y);
WC = C.*W;
Wj = sum(WC,1);

% Remove observations for classes with zero prior probabilities
% or costs
[X,Y,~,WC,Wj,prior,cost,nonzeroClassNames] = ...
    classreg.learning.classif.FullClassificationModel.removeZeroPriorAndCost(...
    X,Y,C,WC,Wj,prior,cost,nonzeroClassNames,dataSummary);

% Normalize priors in such a way that the priors in present
% classes add up to one.  Normalize weights to add up to the
% prior in the respective class.
prior = prior/sum(prior);
WC = WC.*(prior./Wj);
W = sum(WC,2);

% Put processed values into summary structure
classSummary = ...
    classreg.learning.classif.FullClassificationModel.makeClassSummary(...
    classSummary.ClassNames,nonzeroClassNames,prior,cost);
end



% LocalWords:  confusionchart fitnet crossentropy rowsused Wj
