function sOut = mpiprofile(varargin)
%MPIPROFILE Profile parallel communication and execution times
%   MPIPROFILE enables or disables the parallel profiler data collection on a
%   MATLAB worker running in a parallel pool. You can use MPIPROFILE either
%   from the MATLAB client or directly from the worker from within an SPMD
%   block. When you run MPIPROFILE from the MATLAB client, MPIPROFILE
%   carries out the action on the MATLAB workers.
%
%   MPIPROFILE aggregates statistics on execution time and communication
%   times. MPIPROFILE collects statistics in a manner similar to running the
%   PROFILE command on each MATLAB worker. By default, the parallel profiling
%   extensions include array fields that collect information on communication
%   with each of the other workers.
%
%   MPIPROFILE ON <options> starts the parallel profiler and clears previously
%   recorded profile statistics.
%
%   MPIPROFILE takes the following options:
%
%    -MESSAGEDETAIL LEVEL
%         LEVEL can be 'default', 'simplified' or 'off'. This option specifies
%         the detail at which communication information is stored. By default,
%         information is collected per worker instance.  If you have a very
%         large cluster, you may want to change the LEVEL to
%         'simplified'. This will reduce the profiling overhead.  However, you
%         will not get all the detailed inter-worker communication plots in the
%         viewer. Setting LEVEL to 'simplified' will turn off collection for
%         PerLab data fields. Note that changing the messagedetail will clear
%         any previously stored data.
%
%   MPIPROFILE also accepts the following options which have the same meaning
%   as for the PROFILE command:
%
%    -HISTORY
%    -NOHISTORY
%    -HISTORYSIZE SIZE
%
%   No other PROFILE options are supported by MPIPROFILE.
%
%   MPIPROFILE OFF stops the parallel profiler. To reset the state of the
%   profiler and disable collecting communication information, you should also
%   call MPIPROFILE RESET.
%
%   MPIPROFILE RESET turns off the parallel profiler and resets the data
%   collection back to the standard profiler. If you do not call reset,
%   subsequent PROFILE commands will collect MPI information.
%
%   MPIPROFILE VIEWER stops the parallel profiler and opens the graphical profile
%   browser with parallel options. The output is an HTML report displayed in
%   the Profiler window. The file listing at the bottom of the function
%   profile page shows several columns to the left of each line of code. In the
%   summary page:
%     * Column 1 indicates the number of calls to that line.
%     * Column 2 indicates total time spent on the line in seconds.
%     * Columns 3-6 contain the communication information specific to the
%       parallel profiler
%
%   MPIPROFILE RESUME restarts the profiler without clearing previously
%   recorded function statistics.
%
%   MPIPROFILE CLEAR clears the profile information.
%
%   S = MPIPROFILE('STATUS') returns a structure containing information about
%   the current parallel profiler state. S contains these fields:
%
%       ProfilerStatus   -- 'on' or 'off'
%       DetailLevel      -- 'mmex'
%       Timer            -- 'performance'
%       HistoryTracking  -- 'on' or 'off'
%       HistorySize      -- 1000000 (default)
%       MpiMessageDetail -- 'default', 'simplified', or 'off'
%
%   STATS = MPIPROFILE('INFO') stops the parallel profiler and returns a
%   structure containing the profiler statistics. STATS contains the same
%   fields as returned by PROFILE('INFO'), with the following additional
%   fields in the FunctionTable:
%
%     BytesSent           -- Records the quantity of data sent
%     BytesReceived       -- Records the quantity of data received
%     TimeWasted          -- Records communication waiting time
%     CommTime            -- Records the communication time
%     CommTimePerLab      -- Vector of communication receive time for each worker
%     TimeWastedPerLab    -- Vector of communication waiting time for each worker
%     BytesReceivedPerLab -- Vector of data received from each worker
%
%   The three PerLab fields are collected only on a per-function basis, and
%   can be turned off by using the following command:
%   MPIPROFILE on -messagedetail simplified
%
%   When you run it from the MATLAB client, STATS = MPIPROFILE('INFO')
%   returns information from all workers. When you run it on a worker,
%   MPIPROFILE('INFO') returns the profile information specific to that worker.
%
%   MPIPROFILE('VIEWER', STATS) opens the graphical profile browser
%   showing the profiling information contained in STATS. You can use
%   STATS = MPIPROFILE('INFO') on the client to create the structure
%   array.
%
%   MPIPROFILE will not accept -TIMER CLOCK options. The timer will always be
%   set to real and not cpu time.
%
% Example:
%   % Turn on the profiler. With default preferences, turning on the profiler
%   % will create a parallel pool automatically if there is not one already
%   % created.
%   mpiprofile on
%   % Run some distributed array code
%   A = rand(1000, 'distributed');
%   b = sum(A, 2);
%   x = A\b;
%   % Show the collected profile information.
%   mpiprofile viewer
%
% See also profile, mpiprofview, mpiSettings, spmd, distributed.

%   Copyright 2007-2020 The MathWorks, Inc.

import matlab.internal.lang.capability.Capability

try
    % Convert any string inputs to character vectors
    [varargin{:}] = convertStringsToChars( varargin{:} );

    [action, argsForProfile, comDetailLevel, errmsg, option] = iParseInputs( varargin{:});
catch err
    throw(err);
end

if ~isempty(errmsg)
    error(message('parallel:lang:mpi:ProfInvalidInputArgument', errmsg));
end

% If we've got this far, always ensure that our filters are in place.
iAddProfilerFileFilters();

% ensure output is initialized to empty if user request output argument
if nargout > 1
    sOut = [];
end
action = lower(action);
% send the arguments that are set by profile to profile command
if ~isempty(argsForProfile)
    iRunOnWorkers(@profile, argsForProfile{:});
end

switch action
  case 'on'
    iRunOnWorkers(@iOnAction, comDetailLevel);
  case 'off'
    iRunOnWorkers(@iOffAction);
  case 'resume'
    iRunOnWorkers(@iResumeAction, comDetailLevel);
  case 'clear'
    % clear profiler info
    iRunOnWorkers(@profile, 'clear');
  case 'reset'
    iRunOnWorkers(@iResetAction);
  case 'status'
    if iIsOnClient() && isempty(gcp('nocreate'))
        % On client and no pool - return fabricated status (nothing should really need this).
        sOut = struct('ProfilerStatus', 'off', ...
                      'Timer', iDefaultTimer());
    elseif iIsOnClient()
        % Return status from lab 1.
        statusComposite = iRunOnWorkers(@iStatus);
        sOut = statusComposite{1};
    else
        % On a worker - return the real thing.
        sOut = iStatus();
    end
  case 'undoc:timer'
    % Undocumented option: get the timer from the workers. This is used only by
    % mpiprofview. It has the same flaw that PROFILE has - it might be
    % completely wrong. Unfortunately, the return of "profile('info')" doesn't
    % specify the timer that was used, so we are obliged to check the current
    % value, which might be wrong. We also wish to avoid creating a parallel
    % pool simply to return this value.
    if iIsOnClient() && isempty(gcp('nocreate'))
        % return whatever the default would be
        sOut = iDefaultTimer();
    else
        statusComposite = iRunOnWorkers(@iStatus);
        status1 = statusComposite{1};
        sOut = status1.Timer;
    end
  case 'info'
    sOut = iRunOnWorkers(@profile, 'info');
    if iIsOnClient()
        % Concatenate all the info structs and gather.
        spmd
            sOut = gcat(sOut, 1, 1);
        end
        sOut = sOut{1};
    end
  case 'viewer'
    % mpiprofile viewer and mpiprofview are not supported in MATLAB clients which are not local (see g2175913)
    Capability.require([Capability.LocalClient]);
    % gather the data and display in Profiler window
    if iIsOnClient()
        if ~isempty(option) && isstruct(option)
            profileData = option;
        else
            profileDataComposite = iRunOnWorkers(@iTurnOffLoggingAndReturnData);
            profileData = profileDataComposite{1};
        end
        mpiprofview(0, profileData);
    else
        % generate profile info vector
        % gather to a single lab as vector as gcat is more efficient than
        % socket based lab2client
        if ~isempty(option)
            % Workers cannot specify the profile info to "mpiprofile('viewer')".
            warning(message('parallel:lang:mpi:ProfInvalidViewerArgOnWorker'));
        end
        MPI_PROF_VECTOR = iTurnOffLoggingAndReturnData();
        mpiVarString = 'MPI_PROF_VECTOR';
        assignin('base', mpiVarString, MPI_PROF_VECTOR);
        pmode('lab2client', mpiVarString, '1', mpiVarString);
        % then copy to client and run mpiprofview once
        if labindex == 1
            try
                runCmdOnClient(sprintf('mpiprofview(0, %s);clear %s;', mpiVarString, mpiVarString));
            catch err
                throw(err);
            end
        end
    end

  case ''
    % no caught action input in command line so we
    % do nothing and pass the arguments to profile at end of switch
    % function

  otherwise
    % error unknown action
    error(message('parallel:lang:mpi:ProfInvalidArguments'));
end
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function sOut = iStatus()
sOut = profile('status');
% calls callstats('message') with no args to get current state of
% parallel options
switch iGetParallelMsgDetail()
  case 0
    detailString= 'off';
  case 1
    detailString = 'simplified';
  case 2
    detailString = 'default';
  otherwise
    detailString = 'unknown';
end
sOut.MpiMessageDetail = detailString;
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function iResetAction()
% resets everything back to normal profiler
profile ('off');
profile('-timer', 'cpu'); % reset timer back to cpu
iClearParallelMessageLogging();
iTurnMpiLogOff();
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function iResumeAction(comDetailLevel)
% we need to check to make sure mpiprofile is on before we can resume
if iIsProfilerOn()
    profile('resume');
else
    %turn on the parallel profiler
    iTurnMpiLogOn(comDetailLevel);
    profile('resume');
end
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function iOffAction()
iTurnMpiLogOff();
profile('off');
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function out = iTurnOffLoggingAndReturnData()
out = profile('info');
iTurnMpiLogOff();
profile('off');
out = gcat(out, 1, 1);
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function iOnAction(comDetailLevel)
% Does not execute on the client
% Only set timer and message details option if profiler is
% already off

% iAddProfilerFileFilters ensures profiler commands are not profiled.
iAddProfilerFileFilters();

if ~iIsProfilerOn()
    iTurnMpiLogOn(comDetailLevel);
end
profile('on');
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function varargout = iRunOnWorkers(fcn, varargin)
if iIsOnClient()
    pool = parallel.internal.pool.getOrCreatePool( ...
        parallel.internal.pool.PoolPurpose.Spmd );
    if isempty(pool)
        % We're on the client, but user's preference disallows automatic creation of a
        % parallel pool. Disallow this case.
        throwAsCaller(MException(message('parallel:lang:mpi:ProfRequiresRunningPool')));
    end
    [varargout{1:nargout}] = spmd_feval_fcn(fcn, varargin);
else
    [varargout{1:nargout}] = fcn(varargin{:});
end
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function str = iDefaultTimer()
str = 'real';
end
%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function iTurnMpiLogOn(comDetailLevel)
% no need to throw warning as action commands ensure this can only be
% executed on client

callstats('message', comDetailLevel);
profile('-timer', iDefaultTimer());

% It's possible to trigger the MPI profiler on the client but not from within
% an SPMD block, but there will never be any MPI information to record, so
% don't bother enabling the collection. So, if MPI itself has not been
% initialized, don't attempt to enable the MPI-specific profiling collection.
if mpiInitialized
    % set callback from gateway to profiler
    mpigateway( 'logprof' );
    % turn on logging of mpi primitives
    mpigateway( 'logon' );
end
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function iTurnMpiLogOff
if mpiInitialized
    mpigateway( 'logoff' );
end
end

%--------------------------------------------------------------------------
% parses the inputs and gets the action and arguments to pass to profile
%--------------------------------------------------------------------------
function [action, argsForProfile, detailLevel, msg, vieweroptions] = iParseInputs(varargin)
% default outputs
argsForProfile = {};
% get the user settable default message detail from private function
detailLevel = userMessageDetail();
vieweroptions = '';
action = '';
msg = '';
try
    narginchk(1, Inf);
catch err
    msg = err.message;
    return;
end
isPairedArg = false;

% The last action will take precedence
% no error is thrown just like profile on off;
for k = 1:length(varargin)
    if isPairedArg
        % second argument has already been consumed
        isPairedArg = false;
        % hence skip this argument
        continue;
    end
    arg = varargin{k};
    if ( (~ischar(arg) && ~isPairedArg) || isempty(arg))
        msg = getString(message('parallel:lang:mpi:MpiprofileInvalidInput'));
        return;
    end
    switch lower(arg)

        case 'viewer'
            % This is the only action we deal with that is also dealt with
            % by parser since it can be paired.
            % Here we look for a potential followup profile info argument
            action = arg;
            if numel(varargin)> k
                vieweroptions =  varargin{k+1};
            end
            % if its a profiler structure
            if isstruct(vieweroptions)
                % we have a profile info vector input
                % so consume the following argument
                isPairedArg = true;
            end

        case '-messagedetail'
            detailLevel = iVerifyNumAndTypeOfArgs(varargin, k, k + 1, 'detaillevel');
            isPairedArg = true;
            msg = iSetParallelMsgDetailAndEnsureOnWorker( detailLevel );


            % *******arguments to pass to standard profile command*******
        case {'-historysize', '-detail'}
            % lets profile ensure the type of the second argument
            iVerifyNumAndTypeOfArgs(varargin, k, k+1,'');
            % type empty means the type of the second(k+1) argument is not checked
            argsForProfile{end+1} = arg; %#ok
            argsForProfile{end+1} =  varargin{k+1}; %#ok
            isPairedArg = true;

        case {'-history', '-nohistory'}
            argsForProfile{end+1} = arg; %#ok
            % *******END arguments to pass to standard profile command*******

        case '-timer'
            iVerifyNumAndTypeOfArgs(varargin, k, k + 1,'string');
            isPairedArg = true;
            iDealWithUnSupportedOption(arg);

        case '-reset'
            % removes parallel profiler message collection and resets
            % profile info
            profile('off');
            iClearParallelMessageLogging();

        otherwise
            isOption = arg(1)=='-';
            if isOption
                % don't store the argument as action and error out
                msg = getString(message('parallel:lang:mpi:MpiprofileUnsupportedOrInvalidOption'));
            else
                if ~isempty(action)
                    warning(message('parallel:lang:mpi:ProfUnsupportedNumberOfActions', upper( action ), upper( arg )));
                end
                action = arg;
            end
    end
end
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function msg = iSetParallelMsgDetailAndEnsureOnWorker(comDetailLvl)
% returns an error message if it fails to set the comDetailLevel because
% the profile is on
if ~iIsProfilerOn()
    callstats( 'message', comDetailLvl );
    userMessageDetail(comDetailLvl);
    msg = '';
else
    % This message will eventually be thrown at the top level
    msg = getString(message('parallel:lang:mpi:MpiprofileAlreadyStartedCannotChangeMessageDetail'));
end
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function detailLvl = iGetParallelMsgDetail()
% get the message detail level from the profiler
detailLvl = callstats('message');
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function iClearParallelMessageLogging()
% stops profiler from collecting communication information

try
    callstats( 'message', 0 );
    err = [];
catch exception
    err = exception;
end

% reset the user persistent messagedetail setting back to default which currently is 2.
userMessageDetail(-1);
if ~isempty(err)
    rethrow(err);
end
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function isOn = iIsProfilerOn()
% Error if we are not on the worker. The action is only used to generate a more
% customized error.
pStatus = profile('status');
isOn = strcmpi(pStatus.ProfilerStatus,'on');
% throw warning if its on but no communication fields are being recorded
% because profile on was used.
if isOn && iGetParallelMsgDetail()==0 && userMessageDetail ~= 0
    warning(message('parallel:lang:mpi:ProfNotInParallelMode'));
end
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function verifiedVal = iVerifyNumAndTypeOfArgs(argsToParse, argNum, valNum, valType)
% iVerifyNumAndTypeOfArgs(argsToParse, inK, Type)
% verifies the type and number of options to output a relevant message.
ERROR = -1;
if numel(argsToParse) < valNum
    error(message('parallel:lang:mpi:ProfInvalidNumArgs', ...
                  upper(argsToParse{argNum}), valType));
else
    switch(valType)
            % case deal with message detail enumeration
        case 'detaillevel'
            msgDetail = argsToParse{valNum};
            if any(strcmpi(msgDetail, {'simplified', 'novectorfields'}))
                verifiedVal = 1;
            elseif any(strcmpi(msgDetail, {'default', 'withvectorfields'}))
                verifiedVal = 2;
            elseif strcmpi(msgDetail, 'off')
                verifiedVal = 0;
            else
                % indicate error
                verifiedVal = ERROR;
                valType = 'default | simplified | off';
            end

        case 'string'
            if ischar( argsToParse{argNum})
                verifiedVal = 1;
            else
                verifiedVal = ERROR;
            end

        otherwise
            verifiedVal = true;
    end

    if verifiedVal == ERROR
        error(message('parallel:lang:mpi:ProfInvalidInputType', upper( argsToParse{ argNum } ), upper( valType )));
    end
end
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function onclient = iIsOnClient()
if system_dependent('isdmlworker')
    % On a worker, we consider the worker with a pool available to be the
    % 'client'.
    onclient = ~isempty(gcp('nocreate'));
else
    % Not a worker process, so always true.
    onclient = true;
end
end

%--------------------------------------------------------------------------
%--------------------------------------------------------------------------
function iDealWithUnSupportedOption(arg)
warning(message('parallel:lang:mpi:ProfUnsupportedOptions', arg));
end

%--------------------------------------------------------------------------
% This function ensures the mpiprofile commands are not profiled, also various
% boring bits of infrastructure are filtered.
%--------------------------------------------------------------------------
function iAddProfilerFileFilters()
% make sure the profiler filters out functions and files used M profiler
% interface. This implementation follows the pattern of
% profile>filterFilesInProfile.
if ~any(contains(callstats('ffilter'), 'mpiprofile.m'))
    files = {'mpiprofile', 'mpiprofview', 'private/userMessageDetail'};
    cellfun(@(f) callstats('ffilter', 'add', which(f)), files);
end
end
