% PbsHelper - a helper object for use with PBS submissions.
% Also a collection of useful static methods.

% Copyright 2011-2021 The MathWorks, Inc.

classdef ( Hidden, Sealed ) PbsHelper < handle

    properties ( GetAccess = private, SetAccess = immutable )
        ClusterOs % OsType instance
        ResourceTemplate
        UseJobArray
        UseAttach
        RcpCommand
        DataLocStruct
        NumThreads
        JobId
        TaskIds
        TaskSkipString = ''
        ScriptExt
    end
    properties ( SetAccess = immutable )
        SkippedTaskIDs = ''
    end

    properties ( Constant, GetAccess = private )
        % A series of known outputs from various PBS utilities.

        ACCEPTABLE_QDEL_ERRORS = {'qdel: Unknown Job Id', ...
                                  'qdel: nonexistent job id', ...
                                  'qdel: Request invalid for state of job'};
        QSTAT_UNKNOWN_JOB    = 'qstat: Unknown Job Id';
        GENERAL_UNKNOWN_JOB  = 'Unknown Job Id';
        GENERAL_JOB_FINISHED = 'Job has finished';
    end

    methods
        function obj = PbsHelper( clusterOs, resourceTemplate, useJA, ...
                                  useAttach, rcpCommand, dataLocStruct, ...
                                  numThreads, jobID, taskIDs )
            import parallel.internal.apishared.OsType
            import parallel.internal.apishared.FilenameUtils
            import parallel.internal.apishared.PbsHelper

            obj.ClusterOs        = OsType.fromName( clusterOs );
            obj.ResourceTemplate = resourceTemplate;
            obj.UseJobArray      = useJA;
            obj.UseAttach        = useAttach;
            obj.RcpCommand       = rcpCommand;
            obj.DataLocStruct    = dataLocStruct;
            obj.NumThreads       = numThreads;
            obj.JobId            = jobID;
            obj.TaskIds          = taskIDs;

            if ~isempty( taskIDs )
                [obj.TaskSkipString, obj.SkippedTaskIDs] = ...
                    PbsHelper.calcSkipString( taskIDs );
            end

            % Set up script ending
            switch obj.ClusterOs
              case OsType.PC
                obj.ScriptExt = '.bat';
              case OsType.UNIX
                obj.ScriptExt = '.sh';
              otherwise
                error(message('parallel:cluster:PbsUnsupportedCluster'));
            end
        end

        function headerEls = calcHeaderElements( obj )
        % calcHeaderElements - calculate the header elements needed in the
        % submission script, for independent jobs.
            headerEls = { '-h', '-j oe' };

            if ~isempty( strtrim( obj.ResourceTemplate ) )
                resourceString = obj.generateResourceArgument( obj.ResourceTemplate, 1, obj.NumThreads, 1 );
                headerEls{end+1} = resourceString;
            end
        end

        function [logArgs, relLoc, absLoc] = chooseIndTaskLog( obj, jobSId )
            trailPart = 'Task^array_index^.log';
            [logArgs, relLoc, absLoc] = obj.chooseLog( jobSId, trailPart );
        end
        function [logArgs, relLoc, absLoc] = chooseCommJobLog( obj, jobSId )
            trailPart = sprintf( 'Job%d.log', obj.JobId );
            [logArgs, relLoc, absLoc] = obj.chooseLog( jobSId, trailPart );
        end
        function [cmdlineDirectiveArg, fileDirective] = getDirective( obj )
        % handleDirective - calculate the 'directive' to use in a script, and
        % any command-line argument required.
            import parallel.internal.apishared.OsType

            switch obj.ClusterOs
              case OsType.PC
                cmdlineDirectiveArg = '-C "REM PBS"';
                fileDirective       = 'REM PBS ';
              case OsType.UNIX
                cmdlineDirectiveArg = ''; % none required
                fileDirective       = '#PBS ';
            end
        end

        function scriptName = chooseSubmitScriptName( obj, jobSId )
            import parallel.internal.apishared.FilenameUtils

            trailPart  = sprintf( 'Job%d_script%s', obj.JobId, obj.ScriptExt );
            scriptName = FilenameUtils.jobSpecificFilename( obj.ClusterOs.Api1Name, ...
                                                            obj.DataLocStruct, ...
                                                            jobSId, trailPart, true );
        end

        function addHeaderToScript( obj, fh, headerEls, directive )
        % addHeaderToScript - add the header elements to the script
        % Stuff like:
        % "#PBS -h" or "REM PBS -j oe"
            tmpl = iReadTemplate( 'header', obj.ScriptExt );
            fmt  = [directive, '%s\n'];
            replacement = sprintf( fmt, headerEls{:} );

            fprintf( fh, '%s\n', strrep( tmpl, '<HEADERS>', replacement ) );

            if obj.UseJobArray
                obj.addJobArrayPrologueToScript( fh );
            end
        end

        function addCopyInToScript( obj, fh )
        % addCopyInToScript - copy files into cluster
            tmpl = iReadTemplate( 'nonshared-copyin', obj.ScriptExt );

            [rcp, ~, taskIdEnv] = iCalculateRcpPieces( obj.DataLocStruct, obj.ClusterOs.Api1Name );

            % NB: here we assume we know the results of
            % parallel.internal.cluster.FileStorage.serializeForSubmission
            jobCopy  = iBuildRcpCommand( obj.RcpCommand, sprintf( '%sJob%d.zip', rcp, obj.JobId ), ...
                                         'Job.zip', obj.ScriptExt );
            taskCopy = iBuildRcpCommand( obj.RcpCommand, sprintf( '%sJob%d%sTask%s.zip', rcp, ...
                                                              obj.JobId, filesep, taskIdEnv ), ...
                                         sprintf( 'Task.%s.zip', taskIdEnv ), obj.ScriptExt );

            replacement = sprintf( '%s\n%s\n', jobCopy, taskCopy );

            fprintf( fh, '%s\n', strrep( tmpl, '<COPY_FILES>', replacement ) );
        end

        function addExecutionToScript( obj, fh )
            execute = iReadTemplate( 'independentJobWrapper', obj.ScriptExt );
            if obj.UseAttach
                replacement = 'pbs_attach -j ${PBS_JOBID} ';
            else
                replacement = '';
            end
            execute = strrep( execute, '<PBS_ATTACH>', replacement );

            fprintf( fh, '%s\n', execute  );
        end

        function addCopyOutToScript( obj, fh )
            tmpl = iReadTemplate( 'nonshared-copyout', obj.ScriptExt );

            [rcp, clusterSlash, taskIdEnv] = iCalculateRcpPieces( ...
                obj.DataLocStruct, obj.ClusterOs.Api1Name );

            % TODO:later - one day, this could possibly re-use JobFileInfo.
            jobPieces  = { '.common.mat', '.state.mat' };
            taskPieces = { '.common.mat', '.out.mat', '.state.mat', '.diary.txt' };

            strs = {};

            for ii = jobPieces
                fname = sprintf( 'Job%d%s', obj.JobId, ii{1} );
                strs{end+1} = iBuildRcpCommand( obj.RcpCommand, fname, [rcp, fname], ...
                                                obj.ScriptExt ); %#ok<AGROW>
            end

            for ii = taskPieces
                fname = sprintf( 'Job%d%sTask%s%s', ...
                                 obj.JobId, clusterSlash, taskIdEnv, ii{1} );
                strs{end+1} = iBuildRcpCommand( obj.RcpCommand, fname, [rcp, fname], ...
                                                obj.ScriptExt ); %#ok<AGROW>
            end

            copy_files = sprintf( '%s\n', strs{:} );

            tmpl = strrep( tmpl, '<COPY_FILES>', copy_files );

            fprintf( fh, '%s', tmpl );
        end

        function [clientWrapper, doAppend] = copyParallelWrapper( obj, jobSId, parallelWrapper )

            import parallel.internal.apishared.FilenameUtils

            if exist( parallelWrapper, 'file' ) == 2

                [~, shortPart, ext] = fileparts( parallelWrapper );

                clientWrapper = FilenameUtils.jobSpecificFilename( ...
                    obj.ClusterOs.Api1Name, ...
                    obj.DataLocStruct, ...
                    jobSId, [shortPart, ext], true );

                [success, msg] = copyfile( parallelWrapper, clientWrapper );
                if ~success
                    error(message('parallel:cluster:JobWrapperCopyFailed', parallelWrapper, clientWrapper, msg));
                end

                % We want to append to the file, so ensure it's writable
                [success, msg] = fileattrib( clientWrapper, '+w' );
                doAppend = success;
                if ~success
                    parallel.internal.warningNoBackTrace(message('parallel:cluster:PbsCannotAppendToWrapper', msg));
                end

            else
                error(message('parallel:cluster:JobWrapperMissing', parallelWrapper));
            end
        end

        function [FAILED, out, jobIDs] = submitCommunicatingJob( obj, scriptName, cmdLineArgs, doAppend, envMap )

            import parallel.internal.apishared.PbsHelper
            import parallel.internal.apishared.ThirdPartySchedulerUtils

            envSubmitString = PbsHelper.buildEnvString( envMap );

            cmdDirective = obj.getDirective();
            pbsJobName = PbsHelper.chooseJobName( sprintf( 'Job%d', obj.JobId ) );
            cmdLine = sprintf( 'qsub -h -j oe %s %s -v "%s" -N %s "%s"', ...
                               cmdDirective, cmdLineArgs, envSubmitString, pbsJobName, scriptName );

            if doAppend
                obj.appendSubmitString( scriptName, cmdLine );
            end

            [FAILED, out] = ThirdPartySchedulerUtils.system( cmdLine );
            if FAILED
                jobIDs = {};
            else
                jobIDs = { PbsHelper.extractJobId( out ) };
            end
        end

        function releaseHold( ~, jobIDs )
        % releaseHold - do a "qalter -h n <job>" on each of the jobs.
            import parallel.internal.apishared.ThirdPartySchedulerUtils

            for ii = 1:length( jobIDs )
                [FAILED, out] = ThirdPartySchedulerUtils.system( ...
                    sprintf( 'qalter -h n "%s"', jobIDs{ii} ) );
            end
            if FAILED
                error(message('parallel:cluster:PbsUnableToCallQalter', out));
            end
        end
        
        function [FAILED, out, jobIDs] = submitJobUsingArray( obj, scriptName, cmdLineArgs, envArg, arrayLimits )
        %submitJobUsingArray Submit tasks from an independent job using a
        %   PBS job array. The range of tasks to submit is defined by the
        %   arrayLimits.

            import parallel.internal.apishared.PbsHelper
            import parallel.internal.apishared.ThirdPartySchedulerUtils

            pbsJobName = PbsHelper.chooseJobName( sprintf( 'Job%d', obj.JobId ) );
            submitString = sprintf( 'qsub %s -v "%s" -J %s -N %s "%s"', ...
                                    cmdLineArgs, envArg, arrayLimits, pbsJobName, scriptName );

            obj.appendSubmitString( scriptName, submitString );

            [FAILED, out] = ThirdPartySchedulerUtils.system( submitString );

            if ~FAILED
                jobIDs = { PbsHelper.extractJobId( out ) };
            else
                jobIDs = [];
            end
        end

        function [FAILED, out, jobIDs] = submitJobWithoutArray( obj, scriptName, cmdLineArgs, envArg )
        %submitJobWithoutArray Submit all the tasks of an independent job
        %   without using a PBS job array.

            import parallel.internal.apishared.PbsHelper
            import parallel.internal.apishared.ThirdPartySchedulerUtils
            
            jobIDs = cell( 1, numel( obj.TaskIds ) );

            for ii = 1:numel( obj.TaskIds )

                taskId = obj.TaskIds(ii);

                % Append the task ID to the general environment list string.
                thisEnvArg = PbsHelper.buildEnvString( { 'MDCE_TASK_ID', num2str(taskId) }, ...
                                                       envArg );

                % Log location will contain ^array_index^ which we must manually change
                thisCmdLineArgs = strrep( cmdLineArgs, '^array_index^', num2str( ii ) );

                pbsJobName = PbsHelper.chooseJobName( sprintf( 'Job%dTask%d', obj.JobId, taskId ) );
                submitString = sprintf( 'qsub %s -v "%s" -N %s "%s"', ...
                                        thisCmdLineArgs, thisEnvArg, pbsJobName, scriptName );
                if ii == 1
                    obj.appendSubmitString( scriptName, submitString );
                end

                [FAILED, out] = ThirdPartySchedulerUtils.system( submitString );

                if ~FAILED
                    jobIDs{ii} = PbsHelper.extractJobId( out );
                else
                    jobIDs = [];
                    % Return the fail status, the outer layer will deal with this
                    return
                end
            end
        end

    end

    methods ( Access = private )

        function addJobArrayPrologueToScript( obj, fh )
        % addJobArrayPrologueToScript - add the loop which remaps task ID from PBS_ARRAY_INDEX
            prologue = iReadTemplate( 'job-array-prologue', obj.ScriptExt );
            prologue = strrep( prologue, '<SKIP_LIST>', obj.TaskSkipString );
            fprintf( fh, '%s\n', prologue );
        end

        function appendSubmitString( obj, scriptName, submitString )
        % appendSubmitString - stuff the submit string used into the end of the
        % script.
            import parallel.internal.apishared.OsType

            fh = fopen( scriptName, 'at' );
            if fh == -1
                error(message('parallel:cluster:PbsCannotWriteScript', scriptName));
            end
            closer = onCleanup( @() fclose( fh ) );

            switch obj.ClusterOs
              case OsType.PC
                commentStart = 'REM ';
              case OsType.UNIX
                commentStart = '# ';
            end
            fprintf( fh, '\n%sThis script was submitted with the following command line:\n', ...
                     commentStart );
            fprintf( fh, '%s%s\n', commentStart, submitString );
        end

        function [logArgs, relLoc, absLoc] = chooseLog( obj, jobSId, trailPart )
            import parallel.internal.apishared.FilenameUtils

            absLoc = FilenameUtils.jobSpecificFilename( obj.ClusterOs.Api1Name, ...
                                                        obj.DataLocStruct, ...
                                                        jobSId, trailPart, true );
            relLoc = [jobSId, '/', trailPart];

            % Things are somewhat complicated on PC *clients*. If the storage location
            % is within a UNC share, then the output file delivery simply doesn't
            % work correctly. Therefore, we must use a local directory to store the
            % job output.
            if ispc
                % Check to see if the absLoc returned is a UNC path
                if ~isempty( regexp( absLoc, '^\\\\', 'once' ) )
                    % Ok, remap the absLoc, no relLoc
                    relLoc = '';
                    absLoc = sprintf( '%s.%s', ...
                                      tempname, trailPart );
                end
            end
            logArgs = ['-o "' absLoc '"'];
        end

    end

    methods ( Static, Access = private )
        function [skipString, skipIDs] = calcSkipString( taskIds )
        % Calculate the "skip string". To encode the missing tasks, we build a
        % string which will be inserted into the resulting script to map
        % PBS_ARRAY_INDEX back to a real task index.

            % Given the taskIds, calculate the missing ones
            allTasks = 1:max( taskIds );
            skipIDs = setdiff( allTasks, taskIds );

            % Calculate the array needed by the wrapper scripts - skipByArray - which
            % encodes how many missing tasks there are after each given array index
            diffTasks = diff( [0; taskIds] ) - 1;
            skipByArray = [];
            for ii=1:length( diffTasks )
                % Grow the skipByArray array dynamically - we don't expect this to be very
                % large.
                if diffTasks(ii) > 0
                    skipByArray = [ skipByArray, repmat( ii, 1, diffTasks(ii) ) ]; %#ok<AGROW>
                end
            end

            skipString = sprintf( '%d ', skipByArray );
            skipString = skipString(1:end-1);

            % Assert that we got this (somewhat) tricky stuff right. If we got it wrong,
            % then the shell scripts will break. Note that the calculation below is
            % mirrored in the job-array-prologue.* scripts.
            for ii=1:length( taskIds )
                thisId = ii;
                for jj = skipByArray
                    if ii >= jj
                        thisId = thisId + 1;
                    end
                end
                if thisId ~= taskIds(ii)
                    error(message('parallel:cluster:PBSSkipCalculation'));
                end
            end
        end
    end

    methods ( Static )

        function envSubmitString = buildEnvString( envMap, optExistingEnvString )
        % Build an environment string in the form NAME1=value1,NAME2=value2.
        % 2nd optional input argument is an existing environment string to which the
        % entries from envMap will be appended.

            envValues  = envMap(:,2);
            gotValue   = cellfun( @(x) ~isempty(strtrim(x)), envValues );

            % Get the non-empty values in transposed-form
            nonEmptyValues = transpose(envMap(gotValue, :));

            % Build the NAME=value,NAME2=value2 string.
            nameEqualsValue = sprintf( '%s=%s,', nonEmptyValues{:} );

            % Trim the final trailing comma.
            envSubmitString = nameEqualsValue(1:end-1);

            % If we have the optional second argument, prepend that.
            if nargin > 1
                envSubmitString = sprintf( '%s,%s', optExistingEnvString, envSubmitString );
            end
        end
        
        function resourceArgument = generateResourceArgument(template, numWorkers, numThreads, procsPerNode)
            % Generate the resource argument for job submission based on an
            % input template.
            %
            % All occurrences of the following tokens will be replaced by
            % variables according to the following schema:
            %
            % Token     Replaced with
            % -----     -------------
            % ^N^       numWorkers
            % ^T^       numThreads
            % ^PPN^     procsPerNode
            % ^NODES^   ceil(numWorkers / procsPerNode)
            %
            % If only one node is required, then ^PPN^ will be set to
            % numWorkers * numThreads to avoid using the entire node.
            %
            % For communicating jobs, numWorkers should be the number of
            % workers in the job.  For independent jobs/tasks, numWorkers
            % should always be 1.
            
            % Trim the template of any leading or trailing white space.
            template = strtrim(template);
            if isempty(template)
                error(message('parallel:cluster:ThirdPartySchedulerNoResourceTemplate'));
            end
            
            workersPerNode = max(floor(procsPerNode/numThreads), 1);
            nodesRequired = ceil(numWorkers / workersPerNode);
            ppnRequired = min(workersPerNode, numWorkers) * numThreads;
            ppnRequired = min(ppnRequired, procsPerNode);
            
            tokens = {'^N^', '^T^', '^PPN^', '^NODES^'};
            values = arrayfun(@num2str, ...
                [numWorkers, numThreads, ppnRequired, nodesRequired], ...
                'UniformOutput', false);
            resourceArgument = replace(template, tokens, values);
                
        end

        function txt = readQstatInfo( pbsId )
            import parallel.internal.apishared.PbsHelper
            import parallel.internal.apishared.ThirdPartySchedulerUtils

            cmd = sprintf( 'qstat -f "%s"', pbsId );
            [FAILED, txt] = ThirdPartySchedulerUtils.system( cmd );
            if FAILED
                % Check that the output isn't something like
                % qstat: Unknown Job Id 5.uk-eellis0l.dhcp.mathworks.com
                jobNotFound = contains( txt, PbsHelper.QSTAT_UNKNOWN_JOB );
                txt = '';
                if jobNotFound
                    % ok, don't warn
                else
                    parallel.internal.warningNoBackTrace(message('parallel:cluster:PbsQstatCallFailed', txt));
                end
            end
        end

        function OK = isAcceptableQdelError( out )
        % Check whether a given error message from qdel simply indicates
        % that the job isn't there. This often happens if you try to cancel
        % a job that has already finished and is no longer tracked by the
        % scheduler.

            import parallel.internal.apishared.PbsHelper

            % Some versions of TORQUE return an empty message if the job
            % could not be found.
            OK = isempty( out );

            % Check if the error is in our list of acceptable responses
            OK = OK || startsWith( out, PbsHelper.ACCEPTABLE_QDEL_ERRORS );
            
            % Job already finished
            OK = OK || contains( out, PbsHelper.GENERAL_JOB_FINISHED );
        end

        function arrayID = calcTaskIdentifier( taskID, job )
        % Used to work out what the log file for the task is called, as the
        % log for task X is not necessarily TaskX.log on PBS
            
            data = job.JobSchedulerData;

            % Calculate the list of submitted tasks with IDs less than the current one
            allTasks = 1:taskID;
            submittedTasks = setdiff( allTasks, data.skippedTaskIDs );

            % Which element in the array of tasks is this one?
            arrayID = find( submittedTasks == taskID, 1, 'first' );
        end

        function str = chooseJobName( str )
        % PBS has a 15-character restriction on the name of a job. Pass in the
        % name that you'd like, and this function will truncate it if necessary.

            PBS_MAX_JOB_NAME_LENGTH = 15;

            if length( str ) > PBS_MAX_JOB_NAME_LENGTH
                str = str( 1:PBS_MAX_JOB_NAME_LENGTH );
            end
        end

        function jobId = extractJobId( cmdOut )
        % jobId could be in one of two expected forms:
        % 123[].server-name
        % 123.server-name

        % The second piece of the regexp must pick out valid (fully-qualified) server names
            jobId = regexp( cmdOut, '[0-9\[\]]+\.[a-zA-Z0-9-\.]*', 'match', 'once' );

            if isempty( jobId )
                parallel.internal.warningNoBackTrace(message('parallel:cluster:PbsCouldNotParseQsubOutput', cmdOut));
            end
        end

        function [anyRun, anyPend, FAILED] = getRunningPending( stateIndicators, jobIDs )
        % getRunningPending - return if any of jobs/tasks are running or
        % pending. Uses "qstat -f" on each job ID in turn. The cunning use of
        % qselect doesn't work because you need multiple calls, and the state of
        % a job can change in between those calls, so this method is more
        % conservative, and ensures that it retrieves the state of each PBS job
        % precisely once.

            import parallel.internal.apishared.PbsHelper
            import parallel.internal.apishared.ThirdPartySchedulerUtils

            anyRun  = false;
            anyPend = false;
            FAILED  = 0;

            for ii=1:length( jobIDs )
                cmdLine = sprintf( 'qstat -f "%s"', jobIDs{ii} );
                [FAILED, out] = ThirdPartySchedulerUtils.system( cmdLine );

                if ~isempty( strfind( out, PbsHelper.GENERAL_UNKNOWN_JOB ) ) || ...
                     ~isempty( strfind( out, PbsHelper.GENERAL_JOB_FINISHED ) )
                    % Get here for a job that PBS has no knowledge of or that has already finished.
                    FAILED = 0;
                    continue;
                end

                if FAILED
                    % Some other problem with qstat -f
                    parallel.internal.warningNoBackTrace(message('parallel:cluster:PbsUnableToQueryState', cmdLine, out));
                    return
                end
                stateLine = regexp( out, 'job_state = [A-Z]', 'match', 'once' );
                if isempty( stateLine )
                    parallel.internal.warningNoBackTrace(message('parallel:cluster:PbsFailedToParseQstat', out));
                else
                    stateLetter = stateLine(end);

                    % Use the scheduler-specific state indicators to see if this
                    % particular letter indicates running or pending.
                    if ~isempty( strfind( stateIndicators{1}, stateLetter ) )
                        anyRun  = true;
                    elseif ~isempty( strfind( stateIndicators{2}, stateLetter ) )
                        anyPend = true;
                    end
                end

                if anyRun && anyPend
                    % If we get here, we'll never get any further - so break out.
                    break;
                end
            end
        end
    end
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% iCalculateRcpPieces - return the slashes to use locally and on the
% cluster, and the rcp prefix
function [rcpPrefix, clusterSlash, taskIdEnv] = iCalculateRcpPieces( dlStruct, clusterOs )

    import parallel.internal.apishared.OsType
    import parallel.internal.apishared.FilenameUtils

    cfg = pctconfig;
    localhost = cfg.hostname;

    os = OsType.fromName( clusterOs );

    switch os
      case OsType.PC
        clusterSlash = '\';
        taskIdEnv = '%MDCE_TASK_ID%';
      case OsType.UNIX
        clusterSlash = '/';
        taskIdEnv = '${MDCE_TASK_ID}';
    end

    clientStorage = FilenameUtils.getClientRootFromStruct( dlStruct );
    rcpPrefix = sprintf( '%s:%s%s', localhost, clientStorage, clusterSlash );
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% iBuildRcp - build an RCP command
function str = iBuildRcpCommand( rcpCommand, from, to, clusterExt )

    tmpl = iReadTemplate( 'rcp', clusterExt );

    tmpl = strrep( tmpl, '<IN>', from );
    tmpl = strrep( tmpl, '<OUT>', to );
    str  = strrep( tmpl, '<RCP>', rcpCommand );
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% iReadTemplate - read a given template
function str = iReadTemplate( shortName, clusterScriptExt )
    fname = fullfile( toolboxdir('parallel'), 'bin', 'util', 'pbs', ...
                      [shortName, clusterScriptExt] );

    fh = fopen( fname, 'rt' );

    if fh == -1
        error(message('parallel:cluster:PbsCannotReadTemplate', fname));
    end

    try
        % Read whole file in one huge slurp
        str = fread( fh, Inf, 'char' );
        % Strip ^M
        str( str == sprintf( '\r' ) ) = [];
        % Convert to a normal string
        str = char( str(:).' );
        err = [];
    catch exception
        err = exception;
    end

    fclose( fh );
    if ~isempty( err )
        rethrow( err );
    end
end
