classdef TextPreprocessing < matlab.task.LiveTask
%TextPreprocessing  Live Task to guide through text preprocessing

    % Copyright 2022-2023 The MathWorks, Inc.

    properties(Access = private, Constant)
        fields = ["InputDataDropDown",...
            "InputDataTableVariableDropDown",...
            "DoExtractHTML",...
            "DoEraseTags",...
            "DoDecodeHTML",...
            "SelectLanguage",...
            "Split",...
            "SentenceAnalysis",...
            "DoAddPartOfSpeechDetails",...
            "DoAddEntityDetails",...
            "DoAddDependencyDetails",...
            "NormalizeWords",...
            "Case",...
            "ReplaceWords",...
            "IgnoreCase",...
            "DoRemoveStopWords",...
            "DoErasePunctuation",...
            "DoMinWordLength",...
            "MinWordLength",...
            "DoMaxWordLength",...
            "MaxWordLength",...
            "DoRemoveEmptyDocuments",...
            "RemoveWords",...
            "DoShowResults",...
            "DoShowDetails",...
            "DoWordcloud"];
    end

    properties(Access = ?tTextPreprocessing,Transient)
        Accordion                matlab.ui.container.internal.Accordion

        InputDataDropDown        matlab.ui.control.DropDown
        InputDataTableVariableDropDown matlab.ui.control.DropDown

        %% HTML Cleanup
        DoExtractHTML            matlab.ui.control.CheckBox
        DoEraseTags              matlab.ui.control.CheckBox
        DoDecodeHTML             matlab.ui.control.CheckBox
        %% Tokenization
        SelectLanguage           matlab.ui.control.DropDown
        Split                    matlab.ui.control.DropDown
        %% Analysis
        SentenceAnalysis         matlab.ui.control.CheckBox
        DoAddPartOfSpeechDetails matlab.ui.control.CheckBox
        DoAddEntityDetails       matlab.ui.control.CheckBox
        DoAddDependencyDetails   matlab.ui.control.CheckBox
        %% Change
        NormalizeWords           matlab.ui.control.DropDown
        Case                     matlab.ui.control.DropDown
        ReplaceWords             textanalytics.internal.uiTable
        IgnoreCase               matlab.ui.control.CheckBox
        %% Remove
        DoRemoveStopWords        matlab.ui.control.CheckBox
        DoErasePunctuation       matlab.ui.control.CheckBox
        DoMinWordLength          matlab.ui.control.CheckBox
        MinWordLength            matlab.ui.control.Spinner
        DoMaxWordLength          matlab.ui.control.CheckBox
        MaxWordLength            matlab.ui.control.Spinner
        DoRemoveEmptyDocuments   matlab.ui.control.CheckBox
        RemoveWords              textanalytics.internal.uiTable

        %% Visualization
        DoShowResults            matlab.ui.control.CheckBox
        DoShowDetails            matlab.ui.control.CheckBox
        DoWordcloud              matlab.ui.control.CheckBox
    end
    properties(Dependent)
        State
        Summary
    end

    methods(Access = protected)
        function setup(task)
            task.AutoRun = false;

            createComponents(task);
            setComponentsToDefault(task);
            updateComponents(task);
        end
    end
    methods(Access=private)
        function str = getCurrentInput(task)
            if task.InputDataTableVariableDropDown.Visible == "on"
                T = evalin("base",task.InputDataDropDown.Value);
                str = T.(task.InputDataTableVariableDropDown.Value);
            elseif strlength(task.InputDataDropDown.Value) > 0 && ...
                    evalin("base","exist('"+task.InputDataDropDown.Value+"','var')")
                str = evalin("base",task.InputDataDropDown.Value);
            else
                str = [];
            end
        end

        function code = generateInputCode(task,outputName)
            % Select data
            code = localizedText("CodeIntroComment");
            if task.InputDataTableVariableDropDown.Visible == "on"
                var = task.InputDataTableVariableDropDown.Value;
                if ~isvarname(var)
                    var = "(""" + var + """)";
                end
                code = code + compose("\n%s = `%s`.%s;\n", outputName, ...
                    task.InputDataDropDown.Value,var);
            else
                code = code + compose("\n%s = `%s`;\n", outputName, ...
                    task.InputDataDropDown.Value);
            end
        end

        function code = generateHTMLCleanupCode(task,outputName)
            extract = task.DoExtractHTML.Value;
            code = join([...
                conditionallyAddCall(extract, ...
                    "extractHTMLText",outputName),...
                conditionallyAddCall(~extract && task.DoEraseTags.Value, ...
                    "eraseTags",outputName),...
                conditionallyAddCall(~extract && task.DoDecodeHTML.Value, ...
                    "decodeHTMLEntities",outputName)],"");
            if strlength(code) > 0
                code = newline + "% " + localizedText("HTMLSection") + newline + code;
            end
        end

        function code = generateTokenizationCode(task,outputName)
            code = join([ ...
                conditionallyAddCall(task.SelectLanguage.Value=="auto",...
                    "tokenizedDocument",outputName),...
                conditionallyAddCall(task.SelectLanguage.Value~="auto",...
                    "tokenizedDocument",outputName,...
                    "Language="""+task.SelectLanguage.Value+""""),...
                conditionallyAddCall(task.Split.Value == "paragraphs",...
                    "splitParagraphs",outputName),...
                conditionallyAddCall(task.Split.Value == "sentences",...
                    "splitSentences",outputName)],"");
            if strlength(code) > 0
                code = newline + "% " + localizedText("TokenizeSection") + newline + code;
            end
        end

        function code = generateAnalysisCode(task,outputName)
            code = join([ ...
                conditionallyAddCall(task.SentenceAnalysis.Value,...
                    "addSentenceDetails",outputName),...
                conditionallyAddCall(task.DoAddPartOfSpeechDetails.Value,...
                    "addPartOfSpeechDetails", outputName),...
                conditionallyAddCall(task.DoAddEntityDetails.Value,...
                    "addEntityDetails", outputName),...
                conditionallyAddCall(task.DoAddDependencyDetails.Value,...
                    "addDependencyDetails",outputName)],"");
            if strlength(code) > 0
                code = newline + "% " + localizedText("TokenDetailsSection") + newline + code;
            end
        end

        function code = generateChangeCode(task,outputName)
            tblData = task.ReplaceWords.Data;
            if isempty(tblData)
                tblData = strings(0,2);
            end
            replaceFrom = splitAndPad(tblData(:,1));
            replaceTo   = splitAndPad(tblData(:,2));

            drop = all(strlength(replaceFrom)==0,2) | all(strlength(replaceTo)==0,2);
            replaceFrom(drop,:) = [];
            replaceTo(drop,:) = [];

            code = join([...
                conditionallyAddCall(task.NormalizeWords.Value ~= "none",...
                    "normalizeWords",outputName, ...
                    "Style="""+task.NormalizeWords.Value+""""),...
                conditionallyAddCall(task.Case.Value=="lower",...
                    "lower",outputName),...
                conditionallyAddCall(task.Case.Value=="upper",...
                    "upper",outputName),...
                conditionallyAddCall(~isempty(replaceFrom),...
                    "replaceNgrams",outputName,...
                    mat2str(replaceFrom),mat2str(replaceTo), ...
                    "IgnoreCase=" + task.IgnoreCase.Value)],"");
        end

        function code = generateRemoveCode(task,outputName)
            tblData = task.RemoveWords.Data;
            tblData = splitAndPad(tblData);
            drop = all(strlength(tblData)==0,2);
            tblData(drop,:) = [];

            code = join([...
                conditionallyAddCall(~isempty(tblData) && size(tblData,2) < 2,...
                    "removeWords",outputName,...
                    mat2str(tblData),"IgnoreCase="+task.IgnoreCase.Value),...
                conditionallyAddCall(~isempty(tblData) && size(tblData,2) >= 2,...
                    "replaceNgrams",outputName,...
                    mat2str(tblData),mat2str(repmat("",size(tblData,1),1)), ...
                    "IgnoreCase="+task.IgnoreCase.Value),...
                conditionallyAddCall(task.DoMinWordLength.Value &&...
                        task.MinWordLength.Value > 1,...
                    "removeShortWords",outputName,task.MinWordLength.Value-1),...
                conditionallyAddCall(task.DoMaxWordLength.Value,...
                    "removeLongWords",outputName,task.MaxWordLength.Value+1),...
                conditionallyAddCall(task.DoRemoveStopWords.Value,...
                    "removeStopWords",outputName,"IgnoreCase="+task.IgnoreCase.Value),...
                conditionallyAddCall(task.DoErasePunctuation.Value,...
                    "erasePunctuation",outputName),...
                conditionallyAddCall(task.DoRemoveEmptyDocuments.Value,...
                    "removeEmptyDocuments",outputName)],"");
        end

        function code = generateChangeRemoveCode(task,outputName)
            code = join([...
                generateChangeCode(task,outputName),...
                generateRemoveCode(task,outputName)],"");
            if strlength(code) > 0
                code = newline + "% " + localizedText("ChangeRemoveSection") + newline + code;
            end
        end

        function code = generateDisplayCode(task,outputName)
            code = string([]);
            if task.DoShowResults.Value
                code = [code; "display("+outputName+");"];
            end
            if task.DoShowDetails.Value
                code = [code; "tokenDetails("+outputName+")"]; % no semicolon
            end
            if task.DoWordcloud.Value
                code = [code; "wordcloud("+outputName+");"];
            end

            if ~isempty(code)
                code = [""; "% " + localizedText("DisplaySection"); code];
            end
            code = join(code,newline);

            if isempty(code)
                code = "";
            end
        end
    end

    methods
        function [code,outputs] = generateCode(task)
            if task.InputDataDropDown.Value == ""
                % Not enough information to generate code,
                % return empty values
                code = "";
                outputs = {};
                return
            end

            outputName = "preprocessedText";
            outputs = cellstr(outputName);
            code = "";

            code = code + task.generateInputCode(outputName);
            code = code + task.generateHTMLCleanupCode(outputName);
            code = code + task.generateTokenizationCode(outputName);
            code = code + task.generateAnalysisCode(outputName);
            code = code + task.generateChangeRemoveCode(outputName);
            code = code + task.generateDisplayCode(outputName);

            code = code + newline;

            code = replace(code,asManyOfPattern(newline,2),[newline newline]);
        end

        function summary = get.Summary(task)
            inputVar = task.InputDataDropDown.Value;
            if inputVar == ""
                summary = localizedText("SummaryNoVariableSelected");
                return
            end

            inputVar = "`" + inputVar + "`"; % to get it in bold

            % most "important" ones first
            options = [];
            if task.NormalizeWords.Value == "lemma"
                options = [options localizedText("SummaryOptionLemma")];
            end
            if task.NormalizeWords.Value == "stem"
                options = [options localizedText("SummaryOptionStem")];
            end
            if task.NormalizeWords.Value == "none" && task.Case.Value ~= "none"
                options = [options localizedText("SummaryOptionCase")];
            end
            if (task.DoMinWordLength.Value && task.MinWordLength.Value > 1) || ...
                task.DoMaxWordLength.Value || ...
                task.DoRemoveStopWords.Value || ...
                task.DoErasePunctuation.Value || ...
                any(strlength(task.RemoveWords.Data) > 0)
                options = [options localizedText("SummaryOptionWordRemoval")];
            end

            lang = task.SelectLanguage.Value;
            if lang=="auto"
                switch numel(options)
                case 0
                    summary = localizedText("SummaryDefaultLanguage", inputVar);
                case 1
                    summary = localizedText("SummaryDefaultLanguageOneOption", ...
                        inputVar, options(1));
                otherwise
                    summary = localizedText("SummaryDefaultLanguageTwoOptions", ...
                        inputVar, options(1), options(2));
                end
            else
                langString = localizedText("SummaryLanguage_"+lang);
                switch numel(options)
                case 0
                    summary = localizedText("SummaryExplicitLanguage", inputVar, langString);
                otherwise
                    summary = localizedText("SummaryExplicitLanguageOneOption", ...
                        inputVar, langString, options(1));
                end
            end
        end

        function state = get.State(task)
            state = struct;
            for field=task.fields
                if isa(task.(field),"textanalytics.internal.uiTable")
                    state.(field) = task.(field).Data;
                else
                    state.(field) = task.(field).Value;
                end
            end
        end

        function set.State(task,state)
            % if the previously selected variable is not an eligible
            % variable in the workspace, tentatively add it to the list. It
            % is not unlikely that evaluating the whole Live Script is
            % going to recreate this particular variable.
            if isfield(state,'InputDataDropDown')
                var = state.InputDataDropDown;
                if ~any(task.InputDataDropDown.ItemsData == string(var))
                    task.InputDataDropDown.ItemsData = ...
                        [task.InputDataDropDown.ItemsData, string(var)];
                    task.InputDataDropDown.Items = ...
                        [task.InputDataDropDown.Items, cellstr(string(var))];
                end
            end

            if isfield(state,'InputDataTableVariableDropDown')
                var = state.InputDataTableVariableDropDown;
                if ~any(string(task.InputDataTableVariableDropDown.ItemsData) == string(var))
                    task.InputDataTableVariableDropDown.ItemsData = ...
                        [task.InputDataTableVariableDropDown.ItemsData; string(var)];
                    task.InputDataTableVariableDropDown.Items = ...
                        [task.InputDataTableVariableDropDown.Items; string(var)];
                end
            end
            
            for field=task.fields
                if isfield(state,field)
                    if isa(task.(field),"textanalytics.internal.uiTable")
                        task.(field).Data = state.(field);
                    else
                        task.(field).Value = state.(field);
                    end
                end
            end
            updateComponents(task);
        end

        function reset(task)
            setComponentsToDefault(task);
            updateComponents(task);
        end
    end
    methods(Access = private)
        function createComponents(task)
            task.Accordion = matlab.ui.container.internal.Accordion(...
                "Parent",task.LayoutManager);

            checkbox = @(g,label,tooltip) uicheckbox(Parent=g, ...
                ValueChangedFcn=@task.updateComponents, ...
                Text=label, Tooltip=tooltip);
            function h = dropdown(g,label,tooltip,values,internalValues)
                if strlength(label) > 0
                    uilabel(g,Text=label,Tooltip=tooltip);
                end
                h = uidropdown(g, ...
                    Items=values, ...
                    Value=values(1), ...
                    ValueChangedFcn=@task.updateComponents,...
                    Tooltip=tooltip);
                if nargin > 4
                    h.ItemsData = internalValues;
                end
            end

            g = task.newSection(localizedText("SelectDataSection"),[1,3]);
            task.InputDataDropDown = dropdown(g,...
                localizedText("VariableLabel"),...
                localizedText("VariableTooltip"),...
                localizedText("SelectPlaceholder"));
            task.InputDataDropDown.DropDownOpeningFcn=@task.populateWSDropdownItems;
            task.populateWSDropdownItems(task.InputDataDropDown);
            task.InputDataTableVariableDropDown = dropdown(g,...
                "",...
                localizedText("TableVariableTooltip"),...
                localizedText("SelectPlaceholder"));
            task.InputDataTableVariableDropDown.Visible = "off";

            [g,s] = task.newSection(localizedText("HTMLSection"),[1,3]);
            s.Collapsed = true;
            task.DoExtractHTML = checkbox(g,localizedText("ExtractHTML"),localizedText("ExtractHTMLTooltip"));
            task.DoEraseTags = checkbox(g,localizedText("DropTags"),localizedText("DropTagsTooltip"));
            task.DoDecodeHTML = checkbox(g,localizedText("DecodeEntities"),localizedText("DecodeEntitiesTooltip"));

            [g,s] = task.newSection(localizedText("TokenizeSection"),[1,4]);
            s.Collapsed = true;
            task.SelectLanguage = dropdown(g,...
                localizedText("LanguageSelection"),...
                localizedText("LanguageTooltip"),...
                [localizedText("LanguageAuto");...
                 localizedText("Language_en");...
                 localizedText("Language_de");...
                 localizedText("Language_ja");...
                 localizedText("Language_ko")],...
                ["auto";"en";"de";"ja";"ko"]);
            task.SelectLanguage.DropDownOpeningFcn=@task.populateLanguageSelector;
            task.populateLanguageSelector(task.SelectLanguage);

            task.Split = dropdown(g,...
                localizedText("SplitParaSentence"),...
                localizedText("SplitParaSentenceTooltip"),...
                [localizedText("None"),...
                 localizedText("SplitParagraphs"),...
                 localizedText("SplitSentences")],...
                ["none" "paragraphs" "sentences"]);

            [g,s] = task.newSection(localizedText("TokenDetailsSection"),[1,4]);
            s.Collapsed = true;
            task.SentenceAnalysis         = checkbox(g,...
                localizedText("AddSentenceNumbers"),...
                localizedText("AddSentenceNumbersTooltip"));
            task.DoAddPartOfSpeechDetails = checkbox(g,...
                localizedText("AddPOSTags"),...
                localizedText("AddPOSTagsTooltip"));
            task.DoAddEntityDetails       = checkbox(g,...
                localizedText("AddNER"),...
                localizedText("AddNERTooltip"));
            task.DoAddDependencyDetails   = checkbox(g,...
                localizedText("AddDependencyDetails"),...
                localizedText("AddDependencyDetailsTooltip"));

            g = task.newSection(localizedText("ChangeRemoveSection"),[5,2]);

            g2 = uigridlayout("Parent",g, ...
                "ColumnWidth",["fit","fit"],"RowHeight","fit", ...
                "Padding",[0 0 0 0]);
            task.NormalizeWords           = dropdown(g2,...
                localizedText("NormalizeWords"), ...
                localizedText("NormalizeWordsTooltip"), ...
                [localizedText("None") localizedText("Lemma") localizedText("Stem")],...
                ["none" "lemma" "stem"]);
            g2 = uigridlayout("Parent",g, ...
                "ColumnWidth",["fit","fit"],"RowHeight","fit", ...
                "Padding",[0 0 0 0]);
            task.Case                     = dropdown(g2,...
                localizedText("NormalizeCase"), ...
                localizedText("NormalizeCaseTooltip"), ...
                [localizedText("None") localizedText("Lowercase") localizedText("Uppercase")],...
                ["none" "lower" "upper"]);

            g2 = uigridlayout("Parent",g, ...
                "ColumnWidth",["fit","1x"],"RowHeight","fit", ...
                "Padding",[0 0 0 0]);
            g2.ColumnSpacing = 10;

            task.DoMinWordLength          = checkbox(g2,...
                localizedText("MinWordLength"),...
                localizedText("MinWordLengthTooltip"));
            task.MinWordLength            = uispinner(g2,Value=1,Limits=[1 inf]);

            g2 = uigridlayout("Parent",g, ...
                "ColumnWidth",["fit","1x"],"RowHeight","fit", ...
                "Padding",[0 0 0 0]);
            g2.ColumnSpacing = 10;

            task.DoMaxWordLength          = checkbox(g2,...
                localizedText("MaxWordLength"),...
                localizedText("MaxWordLengthTooltip"));
            task.MaxWordLength            = uispinner(g2,Value=42,Limits=[1 inf]);

            task.DoRemoveStopWords        = checkbox(g,...
                localizedText("RemoveStopWords"),...
                localizedText("RemoveStopWordsTooltip"));

            task.DoErasePunctuation       = checkbox(g,...
                localizedText("ErasePunctuation"),...
                localizedText("ErasePunctuationTooltip"));

            g2 = uigridlayout("Parent",g, ...
                "ColumnWidth","fit","RowHeight",["fit","fit"], ...
                "Padding",[0 0 0 0]);
            g2.Layout.Column = [1,2];
            g2.RowSpacing=0;
            uilabel(g2,Text=localizedText("ReplaceWords"),FontWeight="bold");
            task.ReplaceWords             = textanalytics.internal.uiTable( ...
                g2,PlaceHolders=[localizedText("Source") localizedText("Target")], ...
                EditCallback=@task.updateComponents);
            
            g2 = uigridlayout("Parent",g, ...
                "ColumnWidth","fit","RowHeight",["fit","fit"], ...
                "Padding",[0 0 0 0]);
            g2.RowSpacing=0;
            uilabel(g2,Text=localizedText("RemoveWords"),FontWeight="bold");
            task.RemoveWords              = textanalytics.internal.uiTable( ...
                g2,PlaceHolders=localizedText("Word"), ...
                EditCallback=@task.updateComponents);

            uilabel(g,Text="");

            task.DoRemoveEmptyDocuments   = checkbox(g,...
                localizedText("RemoveEmptyDocuments"),...
                localizedText("RemoveEmptyDocumentsTooltip"));

            task.IgnoreCase               = checkbox(g,...
                localizedText("IgnoreCase"),...
                localizedText("IgnoreCaseTooltip"));

            g = task.newSection(localizedText("DisplaySection"),[1,3]);
            g.ColumnWidth = ["fit","fit","fit"];
            task.DoShowResults            = checkbox(g,...
                localizedText("ShowResults"),...
                localizedText("ShowResultsTooltip"));
            task.DoShowDetails            = checkbox(g,...
                localizedText("TokenDetails"),...
                localizedText("TokenDetailsTooltip"));
            task.DoWordcloud              = checkbox(g,...
                localizedText("Wordcloud"),...
                localizedText("WordcloudTooltip"));
        end

        function updateComponents(task,~,~)
            task.populateTableVariable;

            task.populateLanguageSelector(task.SelectLanguage);

            task.DoEraseTags.Enable = ~task.DoExtractHTML.Value;
            task.DoDecodeHTML.Enable = ~task.DoExtractHTML.Value;

            % some options are not available for all languages
            lang = task.SelectLanguage.Value;
            task.populateLemmaStem(lang);

            % splitting is only available for scalar input strings
            task.populateSplitOptions;

            if tokenizedDocument.canAddDependencyDetails
                task.DoAddDependencyDetails.Enable = true;
            else
                task.DoAddDependencyDetails.Enable = false;

                name     = 'Text Analytics Toolbox Model for UDify Data';
                basecode = 'UDIFY';

                msg = message('textanalytics:supportpackages:NotInstalled',...
                    "addDependencyDetails", name, basecode);

                task.DoAddDependencyDetails.Tooltip = eraseTags(getString(msg));
            end

            task.MinWordLength.Enable = task.DoMinWordLength.Value;
            task.MaxWordLength.Enable = task.DoMaxWordLength.Value;

            % Trigger the Live Editor to update the generated script
            notify(task,"StateChanged");
        end

        function [G,S] = newSection(task,label,size)
            S = matlab.ui.container.internal.AccordionPanel(...
                "Parent",task.Accordion);
            S.Title = label;
            G = uigridlayout(S,"ColumnWidth",repmat("fit",1,size(2)),...
                "RowHeight",repmat("fit",1,size(1)),...
                Padding=[10 10 10 0]);
        end

        function populateWSDropdownItems(~,src,~)
            workspaceVariables = evalin("base","whos");
            types = {workspaceVariables.class};
            tableWithString = false(size(types));
            tableWithString(types == "table") = ...
                cellfun(@(T) any(varfun(@(s) isstring(s) | ischar(s) | iscellstr(s),...
                        evalin("base",T),OutputFormat="uniform")),...
                    {workspaceVariables(types=="table").name});
            % cellstr?
            use = (types == "string") | (types == "char") | tableWithString;
            names = string({workspaceVariables.name});
            src.Items = [localizedText("SelectPlaceholder"); names(use).'];
            src.ItemsData = [""; names(use).'];
        end

        function populateTableVariable(task)
            varname = task.InputDataDropDown.Value;
            if varname == "" || ~evalin("base","exist('"+varname+"','var')")
                task.InputDataTableVariableDropDown.Visible = "off";
            else
                varvalue = evalin("base",varname);
                if isa(varvalue,"table")
                    task.InputDataTableVariableDropDown.Visible = "on";
                    stringVariables = varvalue.Properties.VariableNames;
                    isStringy = cellfun(...
                        @(name) isstring(varvalue.(name)) || iscellstr(varvalue.(name)), ...
                        stringVariables);
                    task.InputDataTableVariableDropDown.Items = stringVariables(isStringy);
                else
                    task.InputDataTableVariableDropDown.Visible = "off";
                end
            end
        end

        function populateLanguageSelector(task,src,~)
            task.InputDataDropDown.UserData = string(missing);
            selected = src.Value;

            data = task.getCurrentInput;

            if isequal(data, [])
                src.Items(1)=cellstr(localizedText("LanguageAuto"));
            else
                autoLang=corpusLanguage(data);
                src.Items(1)=cellstr(localizedText("LanguageAutoValue",...
                    localizedText("Language_"+autoLang)));
                task.InputDataDropDown.UserData = autoLang;
            end
            src.Value = selected;
        end

        function populateLemmaStem(task,lang)
            src = task.NormalizeWords;
            selected = string(src.Value);
            if lang == "auto"
                lang = task.InputDataDropDown.UserData;
            end

            switch lang
            case "de"
                src.Items = [localizedText("None");localizedText("Stem")];
                src.ItemsData = ["none";"stem"];
            case {"ja", "ko"}
                src.Items = [localizedText("None");localizedText("Lemma")];
                src.ItemsData = ["none";"lemma"];
            otherwise % en or undecided auto
                src.Items = [localizedText("None");localizedText("Lemma");localizedText("Stem")];
                src.ItemsData = ["none";"lemma";"stem"];
            end

            if any(selected == src.Items)
                src.Value = selected;
            end
        end

        function populateSplitOptions(task)
            data = task.getCurrentInput;
            if isempty(data) || isscalar(string(data))
                task.Split.Enable = "on";
            else
                task.Split.Enable = "off";
                task.Split.Value = "none";
            end
        end
    end

    methods(Access=?tTextPreprocessing)
        function setComponentsToDefault(task)
            % HTML
            task.DoExtractHTML.Value = false;
            task.DoEraseTags.Value   = false;
            task.DoDecodeHTML.Value  = false;

            % Tokenization
            task.SelectLanguage.Value = "auto";
            task.Split.Value          = "none";
            task.SelectLanguage.ValueChangedFcn(); % update lemma/stem restrictions

            % Analysis
            task.SentenceAnalysis.Value         = false;
            task.DoAddPartOfSpeechDetails.Value = true;
            task.DoAddEntityDetails.Value       = false;
            task.DoAddDependencyDetails.Value   = false;

            % change
            try
                task.NormalizeWords.Value  = "lemma";
            catch
            end
            task.Case.Value                = "none";
            task.ReplaceWords.Data         = [];
            task.IgnoreCase.Value          = false;

            % Remove
            task.DoRemoveStopWords.Value  = true;
            task.DoErasePunctuation.Value = true;
            task.DoMinWordLength.Value    = true;
            task.MinWordLength.Value      = 3;
            task.DoMaxWordLength.Value    = true;
            task.MaxWordLength.Value      = 14;
            task.DoRemoveEmptyDocuments.Value = false;
            task.RemoveWords.Data         = [];

            % Visualization
            task.DoShowResults.Value      = false;
            task.DoShowDetails.Value      = false;
            task.DoWordcloud.Value        = true;
        end
    end
end

function code = conditionallyAddCall(condition,funcName,outputName,varargin)
    if condition
        args = join([outputName,varargin{:}],",");
        code = compose("%s = %s(%s);\n",outputName,funcName,args);
    else
        code = "";
    end
end

function s = splitAndPad(s)
    s = num2cell(s);
    s = cellfun(@split,s,'UniformOutput',false);
    nTokens = max(cellfun(@numel,s));
    s = cellfun(@(str) [str;repelem("",nTokens-numel(str),1)].', ...
        s,"UniformOutput",false);
    s = string(vertcat(s{:}));
end

function s = localizedText(key,varargin)
    s = string(getString(message("textanalytics:preprocessLivetask:"+key,varargin{:})));
end
