function [imp, impPerPermutation, impPerClass] = permutationImportance(model,varargin)
%PERMUTATIONIMPORTANCE Permutation importance for predictor evaluation
%   imp = permutationImportance(Mdl) computes predictor importance by
%   permutation for all predictors in Mdl.PredictorNames by using
%   the model Mdl, predictor data in Mdl.X, response data in Mdl.Y and
%   observation weights in Mdl.W. This syntax is only valid when Mdl is a
%   full classification/regression model. The output IMP is a table with
%   importance mean and standard deviation computed for each predictor over
%   10 permutations. Use the NumPermutations name-value argument to change
%   the number of permutations.
%
%   imp = permutationImportance(Mdl,X,Y) computes predictor importance by
%   permutation for all predictors listed in Mdl.PredictorNames by using
%   the model Mdl, predictor data in X and response data in Y. If X is a
%   table, then Y can be a variable name in the table X, or Y can be
%   omitted if the response variable name is the same as Mdl.ResponseName.
%
%   [imp,impPerPermutation] = permutationImportance(Mdl,X,Y) also returns
%   the importance values computed for each predictor and each permutation.
%
%   [imp,impPerPermutation,impPerClass] = permutationImportance(Mdl,X,Y)
%   also returns the mean and standard deviation of importance values
%   computed for each predictor, per class in Mdl.ClassNames. The output
%   impPerClass is empty when Mdl is a regression model. impPerClass is
%   also empty when you specify the LossFun name-value argument as a
%   function handle.
%
%   [...] = permutationImportance(...,Name=Value) specifies options using
%   one or more name-value arguments in addition to any of the input argument
%   combinations in previous syntaxes.
%       'Options'   - A struct that contains options specifying whether to
%                     use parallel computation, and options specifying how
%                     to use random numbers when permuting the original data.
%                     This argument can be created by a call to STATSET.
%                     permutationImportance uses the following fields:
%                      'UseParallel'
%                      'UseSubstreams'
%                      'Streams'
%                     For information on these fields see PARALLELSTATS.
%                     NOTE: If 'UseParallel' is TRUE and 'UseSubstreams' is
%                     FALSE, then the length of 'Streams' must equal the
%                     number of workers. If a parallel pool is already open,
%                     this will be the size of the parallel pool. If a
%                     parallel pool is not already open, and Parallel
%                     Computing Toolbox is installed, depending on parallel
%                     preferences, the software opens and uses a new pool.
%       'Weights'   - Vector of observation weights. By default the weight
%                     of every observation is set to 1. The length of this
%                     vector must be equal to the number of rows in X. If X
%                     is a table, you can specify weights using the name of
%                     a variable in the table. This argument is only valid
%                     when data is provided as input.
%       'LossFun'   - Function handle for loss, or character vector or
%                     string scalar representing a built-in loss function.
%                     Available loss functions for classification:
%                     'binodeviance', 'classiferror', 'classifcost',
%                     'exponential', 'hinge', 'logit', 'mincost', and
%                     'quadratic'. If you pass a function handle FUN for
%                     classification models, permutationImportance
%                     calls it as shown below:
%                               FUN(C,S,W,COST)
%                     where C is an N-by-K logical matrix for N rows in X
%                     and K classes in the ClassNames property of Mdl, S is
%                     an N-by-K numeric matrix, W is a numeric vector with N
%                     elements, and COST is a K-by-K numeric matrix. C has
%                     one true per row for the true class. S is a matrix of
%                     predicted scores for classes with one row per
%                     observation, similar to the score output of the Mdl
%                     object function, predict. W is a vector of observation
%                     weights. COST is a matrix of misclassification costs.
%                     Available loss functions for regression:'mse'.
%                     If you pass a function handle FUN for regression
%                     models, permutationImportance calls it as shown below:
%                               FUN(Y,Yfit,W)
%                     where Y, Yfit and W are numeric vectors of length N.
%                     Y is observed response, Yfit is predicted response,
%                     and W is observation weights.
%                     Default: 'mse' for regression models; model dependent
%                     for classification models.
%       'PredictionForMissingValue' - Numeric scalar, character vector, or
%                     string scalar. If a character vector or string scalar,
%                     must be one of: 'median', 'mean', or 'omitted'. If a
%                     numeric scalar, permutationImportance uses this
%                     response for observations with missing predictor
%                     values. If 'mean' or 'median', permutationImportance
%                     uses the mean or median of observed response values
%                     in the training data. If 'omitted', permutationImportance
%                     excludes observations with missing predictor values
%                     from the computation. This argument is only valid for
%                     regression models of type SVM, Linear, Kernel, GP and
%                     Neural Network.
%                     Default: 'median'
%       'NumPermutations' - Numeric scalar value indicating the number of
%                     permutations used to compute the mean and standard
%                     deviation of predictor importance.
%                     Default: 10
%       'PredictorsToPermute' - List of predictors to permute for
%                     obtaining their importance values.
%                     Pass 'PredictorsToPermute' as one of:
%                       * A numeric vector with indices between 1 and P
%                       where P is numel(Mdl.PredictorNames).
%                       * A logical vector of length P, where a true
%                         entry means that the corresponding predictor is
%                         permuted.
%                       * 'all', meaning all predictors in Mdl.PredictorNames
%                         are permuted.
%                       * A string array or cell array of character
%                         vectors, where each element in the array is
%                         the name of a predictor variable. The names
%                         must match entries in Mdl.PredictorNames.
%                     Default: 'all'.

%   Copyright 2023 The MathWorks, Inc.

[varargin{:}] = convertStringsToChars(varargin{:});

% Validate Model
if ~isa(model,'classreg.learning.Predictor')
    error(message('stats:responsible:permutationImportance:BadModel'));
end

% Check inputs with inputParser. This step ensures that only
% allowed Name-Value pairs are passed.
dataParser = inputParser;
dataParser.KeepUnmatched = true;
addOptional(dataParser,'X',[],@(arg)validateattributes(arg,{'double','single','table'},{'2d','nonempty'}));
addOptional(dataParser,'Y',[],@(arg)true); % Dummy validation needed here so that Y can be passed as a string/char
if numel(varargin) > 1 %#ok<ISCL>
    parse(dataParser,varargin{1:2}); % Only pass the first two arguments here where X and Y may be passed
else
    parse(dataParser,varargin{:});
end
parsedResults = dataParser.Results;

X = parsedResults.X;
if isempty(X)
    % User cannot pass X as empty []; If they do not pass it, then the
    % value is empty which means Y is also empty.
    if ~isa(model,"classreg.learning.FullClassificationRegressionModel")
        error(message('stats:responsible:permutationImportance:NoPredictorData'));
    else
        [wts,~,~] = internal.stats.parseArgs({'Weights'},{[]},varargin{:});
        if ~isempty(wts)
            error(message('stats:responsible:permutationImportance:BadWeightsArg'));
        else
            % use Weights stored in the model
            varargin = [varargin {'Weights',model.W}];
        end

        % Assign to training data
        assert(isprop(model,'DataSummary'),message('stats:responsible:permutationImportance:BadModelProperty')) %internal use only
        if isfield(model.DataSummary,'ObservationsInRows') && ~model.DataSummary.ObservationsInRows
            X = model.X';
        else
            X = model.X;
        end
        Y = model.Y;
    end
else
    [Y,varargin] = classreg.learning.internal.inferResponse(model.ResponseName,X,varargin{2:end});
end

if isa(model,'classreg.learning.FullClassificationRegressionModel')
    model = model.compact;
end

p = inputParser;
addParameter(p,'Options',[]);
addParameter(p,'NumPermutations',[]);
addParameter(p,'PredictionForMissingValue',[]);
addParameter(p,'PredictorsToPermute',[]);
addParameter(p,'Weights',[]);
addParameter(p,'LossFun',[]);

parse(p,varargin{:});
impArgs = p.Results;

if nargout>2
    [imp, impPerPermutation, impPerClass] = responsible.impl.permutationImportanceImpl(model,X,Y,impArgs);
else
    [imp, impPerPermutation] = responsible.impl.permutationImportanceImpl(model,X,Y,impArgs);
end

end

% LocalWords:  IMPPERCLASS
