function T = screenpredictors(data,varargin)
% SCREENPREDICTORS Screen credit scorecard predictors for tall tables.
%
% SCREENPREDICTORS returns a table containing several metrics of
% predictive power for each predictor variable in a MATLAB tall table.
% SCREENPREDICTORS is used as a pre-processing step in the creditscorecard
% workflow in order to reduce the number of predictor variables before
% creating the scorecard.  See below for a list of the supported metrics.
%
%    metrics_table = screenpredictors(data)
%    metrics_table = screenpredictors(data, 'Param1', Value1,...)
%
% Input Arguments:
%
%   data - Tall table containing numeric and/or categorical predictors.
%       Predictors with logical or string values are also accepted.
%
% Optional Input Arguments and Name/Value Pairs:
%
%   'IDVar'        String. Name of the variable used as ID for the
%                  observations. It is provided as a convenience to remove
%                  the column corresponding to this variable from the
%                  predictor variables. By default, this argument is empty,
%                  under the assumption that there is no ID column in the
%                  data.
%
%   'ResponseVar'  String. Name of the response variable. The response data
%                  must be binary. By default, the response variable is the
%                  last column of the data input.
%
%   'PredictorVars' String array or cell of character vectors containing
%                  the names of the variables for which the metrics must be
%                  computed.
%
%   'WeightsVar'   String. Specifies which column in the data table
%                  contains the row weights. By default, this argument is
%                  empty, under the assumption that all rows have unit
%                  weights.
%
%   'NumBins'      Numeric scalar. Specifies the number of bins to use
%                  when binning the numeric data.  Numeric predictors are
%                  binned by equal frequency into NumBins bins.  Default is
%                  20 bins.
%
%   'FrequencyShift' Numeric scalar between 0 and 1. If, after binning, a
%                  predictor's frequency table contains "pure bins" (all
%                  goods or all bads), then the 'FrequencyShift' is added
%                  to all bins in the table. This small perturbation
%                  eliminates the possibility of undefined or infinite
%                  metric values. Default is 0.5.  To avoid any
%                  perturbation, set 'FrequencyShift' to 0.
%
% Output:
%
%   metrics_table - Table containing the metric values for each predictor.
%                  Each row of the table corresponds to a predictor from
%                  the data table. The columns are filled with the
%                  associated metric values. The rows are sorted in
%                  decreasing order by information value ('InfoValue'
%                  column).
%
%                  Supported metrics are (in the order they appear in the table):
%
%                  - 'InfoValue': Information value.
%                  - 'AccuracyRatio': Accuracy ratio based on the
%                        cumulative accuracy profile (CAP) curve.
%                  - 'AUROC': Area under the receiver operating
%                        characteristic (ROC) curve.
%                  - 'Entropy': Entropy measure.
%                  - 'Gini': Gini measure.
%                  - 'Chi2PValue': P-value of the Chi-squared statistic.
%                  - 'PercentMissing': Percentage of missing values in the
%                        predictor data, expressed as a decimal [0..1].
%
%                  See the SCREENPREDICTORS Reference Page in the
%                  Documentation for more details.
%
%   See also SCREENPREDICTORS.

%   Copyright 2019-2023 The MathWorks, Inc.

% Verify we have a tall table
data = tall.validateType(data,'screenpredictors',{'table'},1);

% Get table properties
dataProp = hRef(data,'Properties');

% Struct to hold params while validating
obj = struct();
obj.VarNames = dataProp.VariableNames;

if nargin > 1
    [varargin{:}] = convertStringsToChars(varargin{:});
end

parser = inputParser;
parser.addParameter('ResponseVar','',@(x)ischar(x)|iscell(x));
parser.addParameter('IDVar','',@(x)(ischar(x)||iscell(x)));
parser.addParameter('PredictorVars',{},@(x)(ischar(x)||iscell(x)));
parser.addParameter('WeightsVar','',@(x)validateattributes(x,{'char'},{'scalartext'}));
parser.addParameter('NumBins',20,@(x)validateattributes(x,{'numeric'},{'positive', 'scalar', 'integer'}));
parser.addParameter('FrequencyShift',0.5, @(x)validateattributes(x,{'numeric'},{'scalar', '>=', 0, '<=', 1}));

parser.parse(varargin{:});

% Validate response
obj.ResponseVar = parser.Results.ResponseVar;
obj.ResponseVar = internal.finance.scorecard.preprocessResponseVar(obj.ResponseVar,...
    obj.VarNames);

% Validate ID
obj.IDVar = parser.Results.IDVar;
obj.IDVar = internal.finance.scorecard.validateIDVar(obj.IDVar, obj.VarNames, obj.ResponseVar);

% Set weights var before predictor validation
obj.WeightsVar = parser.Results.WeightsVar;

% Validate data type for predictors
w = warning('off', 'MATLAB:bigdata:array:InefficientCategorical');
cleanupObj = onCleanup(@()warning(w));
data = local_convertPredictorData(data,obj.VarNames);

% Store predictors/weights vars
if isempty(parser.Results.PredictorVars)
    obj.PredictorVars = obj.VarNames;
else
    obj.PredictorVars = parser.Results.PredictorVars;
end

% Validate predictors
if any(~ismember(parser.Results.PredictorVars,obj.VarNames))
    error(message('finance:creditscorecard:creditscorecard:InvalidPredictorName'));
end

% Preprocess predictor vars
obj.PredictorVars = internal.finance.scorecard.preprocessPredictorVar(obj,...
    setdiff(obj.PredictorVars,{obj.IDVar,obj.ResponseVar,...
    obj.WeightsVar},'stable'));

% Get unique values from response
response  = hRef(data,obj.ResponseVar);
unique_responses = unique(response(:));

% Validate and set the weights
if isempty(obj.WeightsVar)
    weights = 1;
else
    weights = hRef(data,obj.WeightsVar);
end

if isa(weights,'tall')
    weights = local_validateWeights(weights);
else
    internal.finance.scorecard.validateWeights(weights);
end

% Diverges from creditscorecard here.

numbins   = parser.Results.NumBins;
freqshift = parser.Results.FrequencyShift;

statsAll    = lower({'InfoValue', 'Entropy', 'Gini', 'AUROC', 'Chi2'});
statsNoChi2 = lower({'InfoValue', 'Entropy', 'Gini', 'AUROC'});

% Find edges for frequency table bins
talledges = cell(numel(obj.PredictorVars),1);
edges     = cell(numel(obj.PredictorVars),1);
binedges  = (0:100/numbins:100)';
for i = 1:numel(obj.PredictorVars)
    
    predictori = obj.PredictorVars{i};
    datai = hRef(data,predictori);
    
    if strcmpi(datai.Adaptor.Class,'categorical')
        talledges{i} = [];
    else
        talledges{i} = prctile(datai,binedges,'method','Approximate');
    end
end
[edges{:},unique_responses] = gather(talledges{:},unique_responses);

% Validate the newly gathered response values
unique_responses(isnan(double(unique_responses))) = [];
if numel(unique_responses) ~= 2
    if ismember({'ResponseVar'}, parser.UsingDefaults)
        error(message('finance:creditscorecard:creditscorecard:DefaultResponseValues'));
    else
        error(message('finance:creditscorecard:creditscorecard:ResponseValues'));
    end
end

% Build the frequency tables
tallvars  = cell(numel(obj.PredictorVars),4);
inmemvars = cell(numel(obj.PredictorVars),4);
for i = 1:numel(obj.PredictorVars)
    
    predictori = obj.PredictorVars{i};
    datai = hRef(data,predictori);
    
    % Check for all missing
    percent_missing = sum(ismissing(datai)) / numel(datai);
    condition = percent_missing == 1;
    datai = tall.validateFalse(datai,condition,...
        message('finance:creditscorecard:creditscorecard:PredictorAllMissing',predictori));
    
    [tallvars{i,1},tallvars{i,2},tallvars{i,3}] = WeightedFrequencyTable(datai,response,unique_responses,weights,edges{i});
    tallvars{i,4} = percent_missing;
end

% Gather all of the frequency tables
[inmemvars{:}] = gather(tallvars{:});
inmemvars = reshape(inmemvars,numel(obj.PredictorVars),4);

T = table();

for i = 1:numel(obj.PredictorVars)
    
    predictori = obj.PredictorVars{i};
    datai = hRef(data,predictori);
    ft = inmemvars{i,1};
    predvals = inmemvars{i,2};
    respvals = inmemvars{i,3};
    percent_missing = inmemvars{i,4};
    
    if strcmpi(datai.Adaptor.Class,'categorical')
        c = internal.finance.binning.container.Categorical(categorical([]),[],...
            'PredictorVar',predictori,'ResponseVar',obj.ResponseVar,...
            'PredictorLabels',predvals,'ResponseLabels',respvals,...
            'FrequencyTable',ft);
        
        % Sort categories by odds
        c = c.sortPredictorCategories('Criterion', 'odds', 'Mode', 'ascending');
    else
        
        c = internal.finance.binning.container.Numeric([],[],...
            'PredictorVar',predictori,'ResponseVar',obj.ResponseVar,...
            'PredictorLabels',predvals,'ResponseLabels',respvals,...
            'FrequencyTable',ft);
    end
    
    if c.getNumBins() > 1
        
        % Container statistics (bininfo stats)
        bin_stats = c.getStatistics('StatsList', statsAll,...
            'FrequencyShift', freqshift);
        
        % Convert chi-squared into p-value (r-1) * (c-1)
        dof = size(ft,1) - 1;
        chi2pvalue = chi2cdf(bin_stats.chi2, dof, 'upper');
    else
        
        % Chi squared is not well supported for single value predictors
        bin_stats = c.getStatistics('StatsList', statsNoChi2,...
            'FrequencyShift', freqshift);
        chi2pvalue = 1;
    end
    
    % Compute accuracy ratio from AUROC
    AUROC = bin_stats.auroc;
    AR = 2 * AUROC - 1;
    
    bs = bin_stats;
    ti = table(bs.infovalue, AR, AUROC, bs.entropy, bs.gini, chi2pvalue, percent_missing,...
        'VariableNames',{'InfoValue', 'AccuracyRatio', 'AUROC', 'Entropy', 'Gini', 'Chi2PValue', 'PercentMissing'});
    ti.Properties.RowNames = {predictori};
    
    T = [T; ti]; %#ok<AGROW>
    
end

% default sorting
T = sortrows(T,'InfoValue','descend');


function out = hRef(in, arg)

out = subsref(in, substruct('.', arg));


function out = hAssign(in, arg, value)

out = subsasgn(in, substruct('.', arg), value);


function data = local_convertPredictorData(data,VarsNames)
% Helper method that converts the data type for predictors. The conversions
% are listed below:
%
% Logical is converted to Double
% Char    is converted to Categorical
% Cellstr is converted to Categorical
% String  is converted to Categorical
%
% Input parameters:
%
%   data - Table of predictors
%
% Outputs:
%
%   data - Updated table to reflect the conversion explained above.
%
% This is an internal function that is not meant to be called directly
% by the user.

NumVars = numel(VarsNames);

for i = 1:NumVars
    
    namei = VarsNames{i};
    vari  = hRef(data,namei);
    
    if ~iscategoricalT(vari) && ~isnumericT(vari)
        if islogicalT(vari)
            % Convert to double
            data = hAssign(data,namei,double(vari));
            
        elseif ischarT(vari)
            % Convert to categorical
            data = hAssign(data,namei,categorical(cellstr(vari)));
            
        elseif iscellT(vari)
            % Convert to categorical
            data = hAssign(data,namei,categorical(vari));
            
        elseif isstringT(vari)
            % Convert to categorical
            data = hAssign(data,namei,categorical(vari));
            
        else
            error(message('finance:creditscorecard:creditscorecard:DataType'));
        end
    end
end


function weights = local_validateWeights(weights)

condition = ~isnumericT(weights);
weights = tall.validateFalse(weights,condition,...
    message('finance:creditscorecard:creditscorecard:InvalidWeights0'));

condition = ~isvector(weights);
weights = tall.validateFalse(weights,condition,...
    message('finance:creditscorecard:creditscorecard:InvalidWeights1'));

condition = any(weights < 0);
weights = tall.validateFalse(weights,condition,...
    message('finance:creditscorecard:creditscorecard:InvalidWeights2'));

condition = all(weights == 0);
weights = tall.validateFalse(weights,condition,...
    message('finance:creditscorecard:creditscorecard:InvalidWeights2'));

condition = any(isnan(weights));
weights = tall.validateFalse(weights,condition,...
    message('finance:creditscorecard:creditscorecard:InvalidWeights3'));

condition = any(~isfinite(weights));
weights = tall.validateFalse(weights,condition,...
    message('finance:creditscorecard:creditscorecard:InvalidWeights3'));


function tf = isnumericT(tallVar)

tf = any(strcmpi(tallVar.Adaptor.Class,...
    {'uint8'; 'uint16'; 'uint32'; 'uint64';
    'int8'; 'int16'; 'int32'; 'int64';
    'single'; 'double'}));

function tf = iscategoricalT(tallVar)

tf = any(strcmpi(tallVar.Adaptor.Class,...
    {'categorical'}));

function tf = islogicalT(tallVar)

tf = any(strcmpi(tallVar.Adaptor.Class,...
    {'logical'}));

function tf = ischarT(tallVar)

tf = any(strcmpi(tallVar.Adaptor.Class,...
    {'char'}));

function tf = isstringT(tallVar)

tf = any(strcmpi(tallVar.Adaptor.Class,...
    {'string'}));

function tf = iscellT(tallVar)

tf = any(strcmpi(tallVar.Adaptor.Class,...
    {'cell'}));


