classdef BayesOptPrintFitauto < bayesoptim.BayesOptPrint
    %BayesOptPrintFitcauto    Customized verbose display for fitcauto/fitrauto.
    % Each method corresponds to a part of the verbose display.
    % The display is broken down into the following parts
    %
    % Header Bar - A contiguous line of "=" that span the width of the
    % display
    %
    % Generic header line 1 - displays the first line of generic
    % bayesopt info like Iter, Eval etc
    %
    % Generic header line 2 - displays the second line of generic
    % bayesopt info like Iter, Eval etc. This line is usually empty
    % and is used when the field names are too long to fit in a single line
    %
    % Optional header line 1- displays the first line of optional header
    % info. Unused by default. Override if you need additioanl values that
    % are not present in bayesopt.
    %
    % Optional header line 2- displays the second line of optional header
    % info. Unused by default. Override if you need additional values that
    % are not present in bayesopt.
    %
    % hyperparameters header line 1- displays the first line of hyperparameter names
    % Override if you need to use different hyperparameter names.
    %
    % hyperparameters header line 2- displays the second line of hyperparameter names
    % Override if you need to use different hyperparameter names.
    %
    % generic optim values - displays the generic iteration result values
    % Override if you need to use different names instead of the generic names.
    %
    % additional optim values - displays the additional iteration result values
    % Override if you need display iteration result values for any additional columns.
    %
    % hyperparameter optim values - displays the hyperparmeter values
    % for the current iteration.

    %   Copyright 2021-2023 The MathWorks, Inc.
    properties(Access = private)
        IntialScalarPadding = '  ';
        % value changed to accomodate for the removal of the estimated loss
        % column and addition of training set size
        ASHAbarWidth = -3;
        LastColumnWidth = 73;

        FitcautoSingleLearner logical
        FitcautoMultipleLearners logical
    end

    % Constructor
    methods(Access = public)
        function this = BayesOptPrintFitauto(fitAutoSingleLearner, fitautoMultipleLearners)
            this.FitcautoSingleLearner = fitAutoSingleLearner;
            this.FitcautoMultipleLearners = fitautoMultipleLearners;
        end
    end

    %% Overrides of PROTECTED base class methods:
    methods(Access = protected)
        function [objectiveFieldLine1, objectiveFieldLine2] = updateObjectiveFieldLines(this)
            if this.IsRegression
                objectiveFieldLine1 = ' log(1+valLoss)';
                objectiveFieldLine2 = '               ';
            else
                objectiveFieldLine1 = ' Validation ';
                objectiveFieldLine2 = ' loss       ';
            end
        end

        function this = generateHeader(this,bayesianOptimization,XTable,isASHA)

            [this.ObjectiveFieldLine1, this.ObjectiveFieldLine2] = this.updateObjectiveFieldLines();
            this.NumVars = length(XTable.Properties.VariableNames);
            this.VarNames = XTable.Properties.VariableNames;
            this.HeaderBar = this.generateHeaderBar(bayesianOptimization,isASHA);

            %generate header line 1
            this.HeaderLine1 = [this.genericOptimHeaderLine1(isASHA),...
                this.additionalOptimHeaderLine1(isASHA),...
                this.hyperparametersHeaderLine1()];

            %generate header line 2
            this.HeaderLine2 = [this.genericOptimHeaderLine2(isASHA),...
                this.additionalOptimHeaderLine2(isASHA),...
                this.hyperparametersHeaderLine2(XTable)];

            this.IsHeaderConstructed = true;
        end

        function out = genericOptimHeaderLine1(this,isASHA)
            % generates part of the first header line used by bayesopt.
            % For a classification function with 'UseParallel' enabled, is
            % shown below.
            % | Iter | Active  | Eval   | Objective   | Objective   | BestSoFar   | BestSoFar   |
            out = [];
            if this.UseParallel
                out = [out,sprintf('| Iter | Active  | Eval   |%s| Time for training | Observed min    |', this.ObjectiveFieldLine1)];
            else
                out = [out,sprintf('| Iter | Eval   |%s| Time for training | Observed min    |', this.ObjectiveFieldLine1)];
            end

            if ~isASHA
                out = [out,sprintf(' Estimated min   |')];
            end

            for c = 1:this.NumCoupledConstraints
                out = [out,sprintf(' Constraint%d  |', c)];
            end
        end

        function out = genericOptimHeaderLine2(this,isASHA)
            % generates part of the second header line used by bayesopt.
            % For a classification function with 'UseParallel' enabled, is
            % shown below.
            % |      | workers | result |             | runtime     | (observed)  | (estim.)    |
            % Can be orverridden to customize it.
            out = [];
            if this.UseParallel
                out = [out,sprintf('|      | workers | result |%s| & validation (sec)| validation loss |', this.ObjectiveFieldLine2)];
            else
                out = [out,sprintf('|      | result |%s| & validation (sec)| validation loss |', this.ObjectiveFieldLine2)];
            end

            if ~isASHA
                out = [out,sprintf(' validation loss |')];
            end

            for c = 1:this.NumCoupledConstraints
                out = [out,sprintf('              |')];
            end
        end

        function bar = generateHeaderBar(this,bayesopt,isASHA)
            NumEquals = this.IterWidth + ...
                this.EvalWidth + ...
                this.MandatoryFields*(this.MandatoryFieldWidth) - 1 + ...
                this.LastColumnWidth ; % width of the last column
            if this.UseParallel
                NumEquals = NumEquals + this.ParallelWidth;
            end
            if isASHA
                NumEquals =  NumEquals+this.ASHAbarWidth;
            end
            % the decrease in width is due to using "validation" instead of
            % "log(1+val)"
            if ~bayesopt.PrivOptions.IsClassregRegressionFunction
                NumEquals = NumEquals-3;
            end


            bar = [sprintf('|'), sprintf(repmat('=', 1, NumEquals)), sprintf('|')];

        end

        function out = genericOptimValues(this,bayesianOptimization, iteration,numFinishedIterations,action)
            % generates the iteration result values that are generic for
            % bayesian optimization/ASHA.
            isASHA = isequal('asha', bayesianOptimization.PrivOptions.AcquisitionFunctionName);
            if this.UseParallel && bayesianOptimization.PrivOptions.IsClassregRegressionFunction
                % Number of jobs that will be running after we launch a new one:
                assert(numFinishedIterations > 0);  % Because printProgress should only be called when jobs finish.
                out = sprintf('| %4d | %7d | %s | %13.5g | %17.5g | %15.5g |', ...
                    iteration, 1+numPendingX(bayesianOptimization.Parallel), action, bayesianOptimization.FTrain(iteration), bayesianOptimization.ObjectiveEvaluationTimeTrain(iteration), ...
                    bayesianOptimization.MinObjective);
            elseif ~this.UseParallel && bayesianOptimization.PrivOptions.IsClassregRegressionFunction
                out = sprintf('| %4d | %s | %13.5g | %17.5g | %15.5g |', ...
                    iteration, action, bayesianOptimization.FTrain(iteration), bayesianOptimization.ObjectiveEvaluationTimeTrain(iteration), ...
                    bayesianOptimization.MinObjective);
            elseif this.UseParallel && ~bayesianOptimization.PrivOptions.IsClassregRegressionFunction
                assert(numFinishedIterations > 0);  % Because printProgress should only be called when jobs finish.
                out = sprintf('| %4d | %7d | %s | %10.5g | %17.5g | %15.5g |', ...
                    iteration, 1+numPendingX(bayesianOptimization.Parallel), action, bayesianOptimization.FTrain(iteration), bayesianOptimization.ObjectiveEvaluationTimeTrain(iteration), ...
                    bayesianOptimization.MinObjective);
            else
                out = sprintf('| %4d | %s | %10.5g | %17.5g | %15.5g |', ...
                    iteration, action, bayesianOptimization.FTrain(iteration), bayesianOptimization.ObjectiveEvaluationTimeTrain(iteration), ...
                    bayesianOptimization.MinObjective);
            end

            if ~isASHA
                out = [out,sprintf(' %15.5g |', bayesianOptimization.MinEstimatedObjective)];
            end

            for c = 1:this.NumCoupledConstraints
                out = [out,sprintf(' %12.3g |', bayesianOptimization.ConstraintTrain(iteration,c))];
            end
        end

        function printIterResults(this,bayesianOptimization,XTable,iteration,numFinishedIterations,action,isASHA)
            fprintf(this.genericOptimValues(bayesianOptimization,iteration,numFinishedIterations,action))
            fprintf(this.additionalOptimValues(bayesianOptimization,iteration))
            fprintf(this.hyperparameterValues(bayesianOptimization,XTable))
            fprintf('\n');
        end

        function out = additionalOptimHeaderLine1(~,isASHA)
            % generates part of the first header line used by fitcauto/fitrauto.
            % This method add ASHA specific changes        .
            if isASHA
                out = ' Training set |';
            else
                out = '';
            end
        end

        function out = additionalOptimHeaderLine2(~,isASHA)
            % generates part of the second header line used by fitcauto/fitrauto.
            % This method add ASHA specific changes
            if isASHA
                out = ' size         |';
            else
                out = '';
            end
        end

        function out = additionalOptimValues(~,bayesianOptimization,iteration)
            % generates the iteration result values in addition to the generic
            % fitcauto/fitrauto values.
            % This method add the training set size for ASHA specific changes
            isASHA = isequal('asha', bayesianOptimization.PrivOptions.AcquisitionFunctionName);
            if isASHA
                out = sprintf(' %12d |', bayesianOptimization.XTrace.TrainingSetSize(iteration));
            else
                out = '';
            end
        end

        function out = hyperparametersHeaderLine1(~)

            out = ' Learner      | Hyperparameter:                 Value   |';

        end

        function out = hyperparametersHeaderLine2(~, XTable)
            % generates the part of the hyperparameter names left over from the
            % first line.
            nvars = 2;
            Names = XTable.Properties.VariableNames;
            VarNameFieldWidth=30;
            for v = 1:nvars
                Name = Names{v};
                if numel(Name) > VarNameFieldWidth+1
                    str = Name(VarNameFieldWidth+1 : min(numel(Name), 2*VarNameFieldWidth));
                else
                    str = '';
                end
                width = VarNameFieldWidth;
                ControlString = sprintf(' %%-%ds |', width);
                if ~(v == nvars)
                    out=sprintf(ControlString, str);
                else
                    out = sprintf('%15s','|');
                    out=[out,sprintf('%42s','|')];
                end
            end
        end

        function Line = hyperparameterValues(~,bayesianOptimization,XTable)
            % generates the hyperparameter values generated for the current iteration
            % of bayesian optimization.
            isASHA = isequal('asha', bayesianOptimization.PrivOptions.AcquisitionFunctionName);
            nvars = size(XTable,2);
            Line = '';
            for v = 1:nvars
                val = XTable.(v);
                if iscell(val) && isscalar(val) && istable(val{1})
                    % For AutoML, read this learner's parameter names and
                    % values out of a table. Display them all in the final
                    % cell of the displayed table, across multiple lines.
                    tbl = val{1};
                    nparams = size(tbl,1);
                    block = repmat(' ',nparams,39);
                    for j=1:nparams
                        % Left-justify parameter name
                        pname = [tbl{j,1}{1},':'];
                        plength = min(size(block,2),numel(pname));
                        block(j,1:plength) = pname(1:plength);

                        % Right-justify parameter value
                        pval = tbl{j,2}{1};
                        if isnumeric(pval)
                            if size(pval,2)>1 % if the hyperparamter value is a row vector
                                pval = char("[ "+num2str(pval)+" ]");
                            else
                                pval = strip(sprintf('%11.5g',pval));
                            end
                        else
                            if ismissing(pval)
                                pval = '-';
                            else
                                pval = strip(char(pval));                            
                            end
                        end
                        vlength = min(size(block,2),numel(pval));
                        block(j,end-vlength+1:end) = pval(1:vlength);
                    end
                    Line = [Line, sprintf(' %12s |', block(1,:))];
                    if size(block,1)>1
                        % All lines after the header is indented to the right
                        if ~isASHA
                            if bayesianOptimization.PrivOptions.UseParallel && ~bayesianOptimization.PrivOptions.IsClassregRegressionFunction
                                fitcautoPadding = sprintf('| %4s | %7s | %6s | %10s | %17.5s | %15.5s |%16s |%14s','','','','','','','','');
                            elseif bayesianOptimization.PrivOptions.UseParallel && bayesianOptimization.PrivOptions.IsClassregRegressionFunction
                                fitcautoPadding = sprintf('| %4s | %7s | %6s | %13s | %17.5s | %15.5s |%16s |%14s','','','','','','','','');
                            elseif ~bayesianOptimization.PrivOptions.IsClassregRegressionFunction
                                fitcautoPadding = sprintf('| %4s | %6s | %10s | %17.5s | %15.5s |%16s |%14s','','','','','','','');
                            else
                                fitcautoPadding = sprintf('| %4s | %6s | %13s | %17.5s | %15.5s |%16s |%14s','','','','','','','');
                            end
                        else
                            if bayesianOptimization.PrivOptions.UseParallel && ~bayesianOptimization.PrivOptions.IsClassregRegressionFunction
                                fitcautoPadding = sprintf('| %4s | %7s | %6s | %10s | %17.5s | %16.5s|%13s | %12s ','','','','','','','','');
                            elseif bayesianOptimization.PrivOptions.UseParallel && bayesianOptimization.PrivOptions.IsClassregRegressionFunction
                                fitcautoPadding = sprintf('| %4s | %7s | %6s | %13s | %17.5s | %16.5s|%13s | %11s  ','','','','','','','','');
                            elseif ~bayesianOptimization.PrivOptions.IsClassregRegressionFunction
                                fitcautoPadding = sprintf('| %4s | %6s | %10s | %17.5s | %16.5s|%13s | %12s ','','','','','','','');
                            else
                                fitcautoPadding = sprintf('| %4s | %6s | %13s | %17.5s | %16.5s|%13s | %12s ','','','','','','','');

                            end
                        end

                        for idx = 2:size(block,1)
                            Line = [Line, newline, fitcautoPadding, sprintf('| %8s |', block(idx,:))];
                        end
                    end
                elseif iscategorical(val)
                    if ismissing(val)
                        Name = '-';
                    else
                        Name = char(val);
                    end
                    Line = [Line, sprintf(' %12s |', Name(1:min(numel(Name),12)))];
                else
                    if ismissing(val)
                        Name = '-';
                        Line = [Line, sprintf(' %12s |', Name(1:min(numel(Name),12)))];
                    else
                        Line = [Line, sprintf(' %12.5g |', val)];
                    end
                end
            end
        end
    end

    %% Overrides of PUBLIC base class methods:
    methods(Access = public)
        function XTable = customizeHyperparameters(this, bayesianOptimization,XTable)
            % customizeHyperparameters is used to rearrange the
            % model-hyperparameter information in a format suitable for
            % verbose display
            tblNames=XTable.Properties.VariableNames;
            if this.FitcautoMultipleLearners
                learnerName = char(XTable{end,1});
                ind = find(startsWith(tblNames(2:end),learnerName))+1;
            elseif this.FitcautoSingleLearner
                splits = split(XTable.Properties.VariableNames{1},'_');
                if size(splits,1) > 1
                    learnerName = splits{1};
                    ind = find(startsWith(tblNames(1:end),learnerName));
                end
            end
            fitcnames = cell(numel(ind),1);
            fitcvals  = cell(numel(ind),1);
            for idx = 1:numel(ind)
                i = ind(idx);
                c = XTable{end,i};
                cname = char(erase(tblNames(i),[learnerName,'_']));
                fitcnames{idx} = cname;
                fitcvals{idx} = c;
            end
            if strcmp(learnerName,'net')
                % converting LayerSizes into one array instead of
                % individual params for each layer
                [fitcnames,fitcvals] = simplifyNetResults(fitcnames,fitcvals);
            end
            smalltable = table(fitcnames,fitcvals);
            XTable = XTable(:,1);
            XTable.Parameters = {smalltable};
            XTable.Properties.VariableNames = {'Learner','Hyperparameter:           Value'};
            if this.FitcautoSingleLearner
                XTable.Learner = categorical({learnerName});
            end
        end

        function line = hyperparametersDisplayLine(this,bayesianOptimization,XTable)
            line = hyperparameterValues(this,bayesianOptimization,XTable);
        end

        function printProgress(this, bayesianOptimization,iteration, NumFinishedIterations,isResuming)

            if this.Verbose < 1
                return;
            end

            isASHA = isequal('asha', bayesianOptimization.PrivOptions.AcquisitionFunctionName);
            XTable = bayesianOptimization.XTrace(iteration,:);
            XTable = customizeHyperparameters(this, bayesianOptimization,XTable);

            % generate header if required
            if ~this.IsHeaderConstructed
                this = this.generateHeader(bayesianOptimization,XTable,isASHA);
            end

            % Display header if required
            if rem(iteration, this.DisplayHeaderInterval) == 1 || this.Verbose == 2 || ...
                    isResuming
                this.printHeader();
            end

            % Display results of the current iteration
            action = iDetermineAction(bayesianOptimization,iteration);
            this.printIterResults(bayesianOptimization,XTable,iteration,...
                NumFinishedIterations,action);
        end

        function verboseXNextDisplay(this, bayesianOptimization, XTable)
            % This gets called when Verbose >= 2
            XTable = customizeHyperparameters(this, bayesianOptimization, XTable);
            [learnerName,tbl] = simplifyAutoResults(this, bayesianOptimization, XTable);
            displayFitcautoResults(this, tbl,learnerName, 'IterationMessageFitcauto');
            fprintf('\n');
        end

        function printVerboseSummary(this, bo)
            % Gets run if iteration == 1 && PrivOptions.Verbose >= 1
            fprintf([bayesoptim.infoString('LearnerTypesFitcauto'),' ']);
            if this.FitcautoSingleLearner
                learnerNameSplit = split(bo.VariableDescriptions(1).Name,'_');
                fprintf('%s ',learnerNameSplit{1});
            elseif this.FitcautoMultipleLearners
                for i = 1:length(bo.Options.VariableDescriptions(1).Range)
                    fprintf('%s',bo.Options.VariableDescriptions(1).Range{i});
                    if i ~= length(bo.Options.VariableDescriptions(1).Range)
                        fprintf(', ');
                    end
                end
            end
            fprintf('\n');
            fprintf([bayesoptim.infoString('TotalIterationsFitcauto'),' (MaxObjectiveEvaluations): %d\n'],bo.Options.MaxObjectiveEvaluations);
            fprintf([bayesoptim.infoString('TotalTimeFitcauto'),' (MaxTime): %d\n'],bo.Options.MaxTime);
            fprintf('\n');
        end

        function showStoppingReason(this, bo)
            bayesoptim.printInfo('TotalEvaluationsFitcauto', bo.NumObjectiveEvaluations);
            bayesoptim.printInfo('TotalTime', num2str(bo.TotalElapsedTime));
            bayesoptim.printInfo('TotalEvalTimeFitcauto', num2str(nansum(bo.ObjectiveEvaluationTimeTrain)));
        end

        function showBestPoints(this, bo)
            if bo.PrivOptions.Verbose >= 1
                [MinObsXTable, MinObsObjective, MinObsLoc] = bestPoint(bo, 'Criterion', 'min-observed');
                if ~isfinite(MinObsLoc)
                    bayesoptim.printInfo('NoFeasibleResult');
                else
                    % Show min observed feasible point
                    if isequal('asha', bo.PrivOptions.AcquisitionFunctionName)
                        % Remove asha dummy hyperparams from X
                        MinObsXTable.TrainingSetSize = [];
                        MinObsXTable.ASHARung = [];
                    end
                    fprintf('\n');
                    % changing the best model-hyperparamter value to a
                    % format accepted by simplifyAutoResults
                    MinObsXTableCustom = customizeHyperparameters(this, bo,MinObsXTable);
                    [learnerName,fitcautoResults] = simplifyAutoResults(this,bo,MinObsXTableCustom);
                    displayFitcautoResults(this, fitcautoResults,learnerName,'BestObservedFitcauto')
                    if bo.PrivOptions.IsClassregRegressionFunction
                        bayesoptim.printInfo('ObservedObjectiveValueFitrauto', num2str(MinObsObjective));
                    else
                        bayesoptim.printInfo('ObservedObjectiveValueFitcauto', num2str(MinObsObjective));
                    end
                    bayesoptim.printInfo('ObservedEvaluationTimeFitcauto', num2str(bo.ObjectiveEvaluationTimeTrace(MinObsLoc)));
                    if ~isempty(bo.ConstraintsTrace)
                        bayesoptim.printInfoNoReturn('ObservedConstraintViolations');
                        fprintf('[ ');
                        fprintf('%f ', bo.ConstraintsTrace(MinObsLoc,:));
                        fprintf(']\n');
                    end
                    % Unless using grid, random, or asha, show best point
                    % according to the default criterion
                    if ~acqFcnIsNonBayes(bo)
                        BestPoint = bestPoint(bo,'Criterion','min-visited-mean');
                        if ~isempty(BestPoint)
                            fprintf('\n');
                            % changing the best model-hyperparamter value to a
                            % format accepted by simplifyAutoResults
                            BestPointCustom = customizeHyperparameters(this, bo,BestPoint);
                            [learnerName,fitcautoResults] = simplifyAutoResults(this,bo,BestPointCustom);
                            displayFitcautoResults(this, fitcautoResults,learnerName,'BestEstimatedFitcauto')
                            if bo.PrivOptions.IsClassregRegressionFunction
                                bayesoptim.printInfo('EstimatedObjectiveValueFitrauto', num2str(predictObjective(bo, BestPoint)));
                            else
                                bayesoptim.printInfo('EstimatedObjectiveValueFitcauto', num2str(predictObjective(bo, BestPoint)));
                            end
                            bayesoptim.printInfo('EstimatedEvaluationTimeFitcauto', num2str(predictObjectiveEvaluationTime(bo, BestPoint)));
                            if bo.PrivOptions.NumCoupledConstraints > 0
                                ConstraintPred = cellfun(@(GP)predict(GP, transformPoints(bo, BestPoint)), ...
                                    bo.ConstraintModels);
                                bayesoptim.printInfoNoReturn('EstimatedConstraintViolations');
                                fprintf('[ ');
                                fprintf('%f ', ConstraintPred);
                                fprintf(']\n');
                            end
                        end
                    end
                    fprintf('\n');
                end
            end
        end

        function showVerboseDocHelp(this, bo)
            if bo.PrivOptions.Verbose >= 1
                if bo.PrivOptions.IsClassregRegressionFunction
                    veboseLinkText = bayesoptim.infoString('veboseLinkTextFitrauto');
                    fprintf(['<a href="matlab:helpview(fullfile(docroot,''stats'',''stats.map''), ''fitrautoVerboseDoc'')">',veboseLinkText,'</a>\n']);
                else
                    veboseLinkText = bayesoptim.infoString('veboseLinkTextFitcauto');
                    fprintf(['<a href="matlab:helpview(fullfile(docroot,''stats'',''stats.map''), ''fitcautoVerboseDoc'')">',veboseLinkText,'</a>\n']);
                end
            end
        end

        % Text and formats in plots
        function [Title1, XLabel1, YLabel] = textInMinObjectivePlot(this, bo)
            Title1 = bayesoptim.infoString('MinObjTitleFitcauto');
            XLabel1 = bayesoptim.infoString('FEvalsFitcautoPlot');
            if bo.PrivOptions.IsClassregRegressionFunction
                YLabel = bayesoptim.infoString('MinObjFitrauto');
            else
                YLabel = bayesoptim.infoString('MinObjFitcauto');
            end
        end

        function Curves = plotMinObjective(this, bo, Axes, Curves)
            % Called each iteration. Plots on Axes, adds to Curves
            if isvalid(Axes)
                Curves{end+1} = plotCurveFitcauto(1:numel(bo.ObjectiveMinimumTrace), ...
                    bo.ObjectiveMinimumTrace, Axes, '#0072bd',[0.00,0.45,0.74],[0.00,0.45,0.74]);
                if all(isnan(bo.EstimatedObjectiveMinimumTrace))
                    legend(Axes, {bayesoptim.infoString('MinObsObjFitcauto')}, 'Location', 'best');
                else
                    Curves{end+1} = plotCurveFitcauto(1:numel(bo.ObjectiveMinimumTrace), ...
                        bo.EstimatedObjectiveMinimumTrace, Axes, '#a6dfff',[0.65,0.87,1.00],[0.65,0.87,1.00]);
                    legend(Axes, {bayesoptim.infoString('MinObsObjFitcauto'), ...
                        bayesoptim.infoString('EstMinObjFitcauto')}, 'Location', 'best');
                end
            end
        end

        % Format of the optimizationResults struct produced when using ASHA
        function optimizationResults = buildAshaOptimizationResults(this, bo)
            % asha in fit*auto. TrainingSetSize appears in third
            % position
            optimizationResults = table;
            optimizationResults.Objective = bo.ObjectiveTrace;
            optimizationResults.ObjectiveEvaluationTime = bo.ObjectiveEvaluationTimeTrace;
            optimizationResults.TrainingSetSize = bo.XTrace.TrainingSetSize;              	% Add TrainingSetSize in third position.
            XTrace = bo.XTrace;
            XTrace.TrainingSetSize = [];
            XTrace.ASHARung = [];
            optimizationResults = [optimizationResults, XTrace];                        	% Add hypers without ASHA vars
        end

    end

    %% Completely local methods
    methods(Access = private)
        function displayFitcautoResults(this,tbl,learnerName,displayMessage)
            % used to display model information when verbose is set to 2

            % remove learner from tbl to prevent duplicate printing
            learnerIdx = find(strcmp('learner',tbl.Properties.VariableNames));

            if ~isempty(learnerIdx)
                tbl(:,learnerIdx)= [];
            end

            if strcmp(learnerName,'nb')
                learnerName = 'naive Bayes';
            end

            codingIdx = find(strcmp('Coding',tbl.Properties.VariableNames));
            if ~isempty(codingIdx)
                learnerName = ['multiclass ' learnerName];
            end

            if any(strcmp(learnerName,{'svm','ensemble'}))
                displayMessage = [displayMessage,'EnsembleSVM'];
            end

            if startsWith(displayMessage,'Iteration')
                fprintf('\n');
            end

            bayesoptim.printInfo(displayMessage,learnerName);

            for i=1:size(tbl,2)
                block = repmat(' ',1,30);

                % Left-justify parameter name
                pname = [tbl.Properties.VariableNames{i},':'];

                if strcmp(pname,'Coding:')
                    pname = 'Coding (ECOC):';
                end

                plength = min(size(block,2),numel(pname));
                block(1:plength) = pname(1:plength);

                % Right-justify parameter value
                pval = tbl.(tbl.Properties.VariableNames{i});
                if strcmp(tbl.Properties.VariableNames{i},'LayerSizes')
                    if numel(pval)~=1
                        pval = strcat('[',num2str(pval),']');
                    end
                end
                if isnumeric(pval)
                    pval = strip(sprintf('%11.5g',pval));
                else
                    pval = strip(char(pval));
                end

                vlength = min(size(block,2),numel(pval));
                block(end-vlength+1:end) = pval(1:vlength);

                if i == size(tbl,2) && startsWith(displayMessage,'Iteration')
                    fprintf('\t%s',block);
                else
                    fprintf('\t%s\n',block);
                end
            end
        end

        function [learnerName,tblNew] = simplifyAutoResults(this, bayesianOptimization, tbl)
            % utility to convert model information suitable to be passed
            % to displayAutoResults
            tblNew = table;
            if this.FitcautoMultipleLearners
                tblNew.Learner = tbl.Learner;
                learnerName = char(tbl.Learner);
            else
                learnerName = char(tbl.Learner);
            end
            tbl = tbl.("Hyperparameter:           Value"){1};
            fitcnames = tbl.fitcnames; fitcvals = tbl.fitcvals;
            for i=1:size(fitcnames,1)
                tblNew.(fitcnames{i}) = fitcvals{i};
            end
        end

    end
end

function Curve = plotCurveFitcauto(X, TraceToPlot, Axes, color,markerFaceColor,markerEdgeColor)
HoldOff = bayesoptim.holdOn(Axes);
Curve = plot(Axes, X, TraceToPlot, ['-' 'o'], 'MarkerSize', 2,...
    'color',color,'MarkerFaceColor',markerFaceColor,'MarkerEdgeColor',markerEdgeColor);
end

function action = iDetermineAction(bayesianOptimization,iteration)
[~,~,MinObsLoc] = bestPoint(bayesianOptimization, 'Criterion', 'min-observed');
if isnan(bayesianOptimization.FTrain(iteration))
    action = 'Error ';
elseif ~isempty(bayesianOptimization.ConstraintsTrace) &&...
        any(bayesianOptimization.ConstraintsTrace(iteration,:) > 0)
    action = 'Infeas';
elseif MinObsLoc == iteration
    action = 'Best  ';
else
    action = 'Accept';
end
end

function [fitcnames,fitcvals] = simplifyNetResults(fitcnames,fitcvals)
% utility to change individual layer params into an array of layerSizes
Numlayers= fitcvals{find(strcmp(fitcnames,'NumLayers'))};
Startidx = find(strcmp(fitcnames,'NumLayers'))+1;
Stopidx = find(strcmp(fitcnames,'Layer_1_Size'))-1;
layerSizes=[];
for i=1:Numlayers
    layerSizes = [layerSizes fitcvals{Stopidx+i}];
end
for idx=Startidx:Stopidx
    fitcnames1{idx-1} = fitcnames{idx};
    fitcvals1{idx-1} = fitcvals{idx};
end
fitcnames1{Stopidx} = 'LayerSizes';
fitcvals1{Stopidx} = layerSizes;
fitcnames = fitcnames1';
fitcvals = fitcvals1';
end
