function [Xout,Y,vrange,wastable,args] = table2FitMatrix(X,Y,varargin)
%table2FitMatrix Convert table data to matrix data for fitting.
%
%    This utility function replicates the same behavior as
%    classreg.learning.internal.table2FitMatrix but for tall inputs.
%
%    [X,Y,VRANGE,WASTABLE,ARGS] = table2FitMatrix({X X.Adaptor},{Y Y.Adaptor/[]},...)
%    takes data X and additional arguments, and returns a data matrix X,
%    response vector Y, a cell array VRANGE of variable ranges, boolean
%    flag WASTABLE indicating table input, and additional arguments to be
%    processed by the caller.
%
%    Tall Adaptors must be passed along talls, use [] when Y is not tall.
%
%    In the first input X may be a tall table or a tall. The second input
%    may be a tall with response values, the name of the variable in table
%    X that is the response, or a formula the defines the response and
%    predictors to be used from X.
%
%    The additional arguments may be one or more of the following
%    name/value pairs:
%       'ResponseName'     Name of the response variable
%       'Weights'          Vector of weights or name of the variable to be
%                          used as weights
%       'PredictorNames'   Cell array of predictor names
%       'CategoricalPredictors' - Variables to be treated as categorical,
%                          specified either as a cell array of variable
%                          names, or an integer or logical index vector
%       'OrdinalIsCategorical' -  No tall support
%
%    The OrdinalIsCategorical parameter is for this function only and is
%    not passed into the ARGS output. Any name/value pairs with names not
%    matching the ones listed above are passed unchanged into the ARGS
%    output.
%
%   Since OrdinalIsCategorical is always true, OrdinalPredictors are always
%   returned as an array of all false values.

%   Copyright 2016-2020 The MathWorks, Inc.

args = varargin;

XAdaptor = X{2};
X = X{1};
YAdaptor = Y{2};
Y = Y{1};

% Extract the input arguments that are related to table processing
pnames = {'ResponseName' 'Weights' 'PredictorNames' 'CategoricalPredictors' 'OrdinalIsCategorical' 'acceptCategorical'};
[ResponseName,ResponseIndex,WeightsName,WeightsIndex,...
    PredictorNames,PredictorIndex,CategoricalPredictors,CategoricalIndex,...
    OrdinalIsCategorical,OrdinalIsCategoricalIndex,acceptCategoricalPredictor,...
    acceptCategoricalPredictorIndex] = processArgs(pnames,args);
args = removeArgs([OrdinalIsCategoricalIndex,acceptCategoricalPredictorIndex],args);
if ~isempty(ResponseName) && ~matlab.internal.datatypes.isCharString(ResponseName)
    error(message('stats:classreg:learning:internal:utils:BadResponseName'));
end

if isempty(acceptCategoricalPredictor)
    acceptCategoricalPredictor = false;
end

wastable = strcmp(tall.getClass(X), 'table');
if ~wastable
    vrange = {};
    Xout = X;
    return
end

VarNames = subsref(X, substruct('.', 'Properties', '.', 'VariableNames'));

% Get the weights and update args
if ~istall(WeightsName) && ~isempty(WeightsName) && matlab.internal.datatypes.isCharString(WeightsName)
    WeightsName = resolveName('WeightsName',WeightsName,'',true,VarNames);
    args{WeightsIndex} = subsref(X, substruct('.',WeightsName));
else
    WeightsName = '';  % no name, W is already in the arg list
end

% Y may be a vector of response values, the name of a variable in the table
% X, or a formula involving names of variables in X. Deal with the names
% first
[FormulaResponseName,FormulaPredictorNames] = processFormula(VarNames,Y);

% Get the Y response data
if istall(Y)
    if strcmp(tall.getClass(Y), 'table')
        % Given as table, extract the single response variable
        if width(Y)~=1
            error(message('stats:classreg:learning:internal:utils:TableResponse'));
        end
        ResponseName = subsref(Y, substruct('.', 'Properties', '.', 'VariableNames'));
        ResponseName = ResponseName{1};
        args = updateArgs('ResponseName',ResponseName,ResponseIndex,args);
        Y = Y{:,1};
    elseif ~isempty(ResponseName)
        % it is already a tall. error if the name matches a table value as this is ambiguous
        if ismember(ResponseName,VarNames)
            error(message('stats:classreg:learning:internal:utils:AmbiguousResponse',ResponseName));
        end
    end
else
    if matlab.internal.datatypes.isCharString(Y)
        % Given by name, make sure the args reflect the chosen name
        ResponseName = resolveName('ResponseName',ResponseName,FormulaResponseName,false,VarNames);
        args = updateArgs('ResponseName',ResponseName,ResponseIndex,args);
        Y = X.(ResponseName);
    elseif ~isempty(ResponseName)  % already checked this is a string above
        % Given by value, error if the name matches a table value as this is ambiguous
        if ismember(ResponseName,VarNames)
            error(message('stats:classreg:learning:internal:utils:AmbiguousResponse',ResponseName));
        end
    end
end

PredictorNames = resolveName('PredictorNames',PredictorNames,FormulaPredictorNames,true,VarNames,true);
if isempty(PredictorNames)
    PredictorNames = VarNames;
    if ~isempty(ResponseName) || ~isempty(WeightsName)
        PredictorNames = setdiff(PredictorNames,{ResponseName,WeightsName},'stable');
    end
end
args = updateArgs('PredictorNames',PredictorNames,PredictorIndex,args);

if ismember(ResponseName,PredictorNames)
    error(message('stats:classreg:learning:internal:utils:ResponseIsPredictor'))
end

if ~acceptCategoricalPredictor
    [Xout,vrange] = makeXMatrixNoCat(X,XAdaptor,PredictorNames);
else
    if isequal(CategoricalPredictors,'all')
        CategoricalPredictors = PredictorNames;
    end
    [Xout,vrange,CategoricalPredictors] = makeXMatrix(X,XAdaptor,PredictorNames,CategoricalPredictors);
    args = updateArgs('CategoricalPredictors',CategoricalPredictors,CategoricalIndex,args);
    args = updateArgs('OrdinalPredictors',false(1,numel(PredictorNames)),0,args);
end
end

function [Xout,vrange] = makeXMatrixNoCat(X,XAdaptor,PredictorNames)
% Make a design matrix
p = numel(PredictorNames);
vrange = {};
for j=1:p
    % For each predictor,check it is one column and detect invalid classes
    pname = PredictorNames{j};
    varSize = getVariableSize(XAdaptor, pname);
    if ~isempty(varSize) && varSize(2)~=1
        error(message('stats:classreg:learning:internal:utils:BadVariableSize',pname))
    end
    varClass = getVariableClass(XAdaptor, pname);
    if any(strcmp({'categorical','datetime','duration','calendarDuration','cell','char','string'},varClass))
        error(message('stats:classreg:learning:internal:utils:BadVariableType',pname))
    end
end
Xout = X{:,PredictorNames};
end

function idx = makeCategoricalIndex(CategoricalPredictors,PredictorNames)
% Make a logical index vector locating the categorical predictors
p = numel(PredictorNames);
idx = CategoricalPredictors;
if isempty(CategoricalPredictors)
    idx = false(1,p);
elseif islogical(CategoricalPredictors)
    if numel(idx)~=p || ~isvector(idx)
        error(message('stats:classreg:learning:internal:utils:CategoricalBadLogical',p))
    end
elseif internal.stats.isStrings(CategoricalPredictors)
    tf = ismember(CategoricalPredictors,PredictorNames);
    if ~all(tf)
        error(message('stats:classreg:learning:internal:utils:CategoricalNotPredictor'))
    end
    idx = ismember(PredictorNames,CategoricalPredictors);
elseif isvector(idx) && isnumeric(idx) && all(idx==round(idx)) && numel(idx)==numel(unique(idx))
    if any(idx<1) || any(idx>p)
        error(message('stats:classreg:learning:FullClassificationRegressionModel:prepareDataCR:BadCatPredIntegerIndex', p));
    else
        idx = ismember(1:p,idx);
    end
else
    error(message('stats:classreg:learning:internal:utils:BadCategorical'))
end
end

function [Xout,vrange,catidx] = makeXMatrix(X,XAdaptor,PredictorNames,CategoricalPredictors)
% Make a design matrix
catidx = makeCategoricalIndex(CategoricalPredictors,PredictorNames);
waslogical = islogical(CategoricalPredictors);

p = numel(PredictorNames);
vrange = cell(1,p);
for j=1:p
    % For each predictor,check it is one column and detect invalid classes
    pname = PredictorNames{j};
    varSize = getVariableSize(XAdaptor, pname);
    varClass = getVariableClass(XAdaptor, pname);
    if ~isempty(varSize) && varSize(2)~=1 && ~strcmp('char',varClass) % ~iscolumn
        error(message('stats:classreg:learning:internal:utils:BadVariableSize',pname))
    end
    
    if any(strcmp({'datetime','duration','calendarDuration'},varClass))
        error(message('stats:classreg:learning:internal:utils:BadVariableType',pname))
    end
    
%     if any(strcmp({'categorical','datetime','duration','calendarDuration','cell','char'},varClass))
%         error(message('stats:classreg:learning:internal:utils:BadVariableType',pname))
%     end
    
    % Variable must be categorical based on its type. Error if it is
    % explicitly specified as not categorical. internal.stats.isDiscreteVec
    % cellstr, not cell?
    if any(strcmp({'categorical','cell','char','logical'},varClass))
        if waslogical && ~catidx(j)
            error(message('stats:classreg:learning:internal:utils:CategoricalConflict',pname));
        end
    end
    
    % cellstr, not cell?
    if any(strcmp({'categorical','cell','char','logical','string'},varClass))
        % These variables must be treated as categorical
        catidx(j) = true;
    end
    
    % also consider numeric as categorical if specified by user
    if catidx(j) || any(strcmp({'categorical','cell','char','logical'},varClass))
        if any(strcmp({'cell','char','string'},varClass))
            % iscellstr
            if any(strcmp({'char','string'},varClass))
                newx = strtrim(cellstr(X.(pname)));
                X.(pname) = newx;
            elseif strcmp('cell',varClass)
                newx = strtrim(X.(pname));
                X.(pname) = newx;
            end
            
            % TODO(g1646445): The validateColumn workaround is to avoid an
            % extra pass because tall/unique has no size information about
            % the input argument. This is to be removed once enough
            % information is propagated to here.
            vrj = unique(tall.validateColumn(X.(pname), 'MATLAB:bigdata:array:UniqueRequiresVector'));
            empties = cellfun(@isempty,vrj);
            vrj(empties) = [];
            X.(pname) = double(categorical(X.(pname),vrj));
        else
            % TODO(g1646445): The validateColumn workaround is to avoid an
            % extra pass because tall/unique has no size information about
            % the input argument. This is to be removed once enough
            % information is propagated to here.
            vrj = unique(tall.validateColumn(X.(pname), 'MATLAB:bigdata:array:UniqueRequiresVector'));
            if strcmp('categorical',varClass)
                % categorical
                X.(pname) = double(removecats(X.(pname)));
                vrj(isundefined(vrj)) = [];
            else
                % numeric and logical
                vrj(isnan(vrj)) = [];
                % g1612430
                vrjName = compose("%.20g", vrj);
                X.(pname) = double(categorical(X.(pname),vrj,vrjName));
            end
        end
        vrange{j} = vrj;
    end
end
Xout = X{:,PredictorNames};
end

function ArgName = resolveName(ParameterName,ArgName,FormulaName,emptyok,VarNames,wantcell)
if nargin<6
    wantcell = false;
end
if isempty(ArgName)
    if isempty(FormulaName) && ~emptyok
        error(message('stats:classreg:learning:internal:utils:MissingArg',ParameterName))
    end
    ArgName = FormulaName;
elseif ~isempty(FormulaName)
    if ~strcmp(FormulaName,ArgName)
        error(message('stats:classreg:learning:internal:utils:ConflictingArg',ParameterName))
    end
end
if ischar(ArgName) && (wantcell || ~isrow(ArgName))
    ArgName = cellstr(ArgName);
end
if ~isempty(ArgName) && (   ~internal.stats.isStrings(ArgName) ...
        || ~all(ismember(ArgName,VarNames)) ...
        || (iscell(ArgName) && numel(ArgName)~=numel(unique(ArgName))))
    error(message('stats:classreg:learning:internal:utils:InvalidArg',ParameterName))
end
end

function args = updateArgs(ParameterName,ArgName,ArgIndex,args)
if ~isempty(ArgName)
    if ArgIndex>0
        args{ArgIndex} = ArgName;
    else
        args(end+1:end+2) = {ParameterName,ArgName};
    end
end
end
function args = removeArgs(ArgIndices,args)
if ~isempty(ArgIndices)
    ArgIndices = sort(ArgIndices,'descend');
    for j=1:length(ArgIndices)
        ArgIndex = ArgIndices(j);
        if ArgIndex>0
            args(ArgIndex-1:ArgIndex) = [];
        end
    end
end
end

% -------------
function [FormulaResponseName,FormulaPredictorNames] = processFormula(VarNames,Y)
% Get response and predictor information from the formula input
if isvarname(Y)                                   % fitxxx(tbl, 'y')
    FormulaResponseName = Y;
    FormulaPredictorNames = {};
elseif ischar(Y) && isrow(Y) ...
        && ismember({Y},VarNames)                 % fitxxx(tbl, 'y 1')
    FormulaResponseName = Y;
    FormulaPredictorNames = {};
elseif matlab.internal.datatypes.isCharString(Y)  % fitxxx(tbl, 'y~x1+x2')
    formula = classreg.regr.LinearFormula(Y,VarNames);
    termorder = sum(formula.Terms,2);
    if any(termorder>1)
        error(message('stats:classreg:learning:internal:utils:LinearOnly'))
    end
    FormulaResponseName = formula.ResponseName;
    FormulaPredictorNames = formula.PredictorNames;
else                                              % fitxxx(tbl, y)
    FormulaResponseName = '';
    FormulaPredictorNames = {};
end
end

% -----------
function [varargout] = processArgs(pnames,args)
% Find the locations and values for selected parameter names
n = numel(pnames);
varargout = cell(1,2*n);
for j=1:n
    varargout{2*j-1} = '';    % empty value for this arg
    varargout{2*j} = 0;       % no location for this arg
end

for j=1:2:length(args)-1
    pname = args{j};
    if matlab.internal.datatypes.isCharString(pname)
        argnum = find(strncmpi(pname,pnames,length(pname)));
        if isscalar(argnum)
            varargout{2*argnum-1} = args{j+1};
            varargout{2*argnum} = j+1;
        end
    end
end
end
