classdef CCSSchedulerConnection < distcomp.AbstractMicrosoftSchedulerConnection
    %CCSSchedulerConnection Class for connections to Microsoft CCS scheduler APIs
    %   Class for interfacing to ccpapi.dll using MATLAB COM Client Support.

    %  Copyright 2009-2022 The MathWorks, Inc.


    properties (Constant, GetAccess = protected)
        FoundComputeClusterLibraries = distcomp.CCSSchedulerConnection.addAndCreateMicrosoftComputeCluster;
        
        MicrosoftJobStates = distcomp.CCSSchedulerConnection.getMicrosoftJobStates();
        MatlabJobStates = distcomp.CCSSchedulerConnection.getMatlabJobStates();
    end
    
    properties (Access = private)
        % The actual connection to the CCS API.  This is a .NET object of type
        % Microsoft.ComputeCluster.Cluster
        ComputeClusterConnection;
    end
    

    methods (Static)% AbstractMicrosoftSchedulerConnection Abstract, static method implementation
        %---------------------------------------------------------------
        % getAPIVersion
        %---------------------------------------------------------------
        function version = getAPIVersion
            % NB This needs to be a character vector that matches the value
            % of the distcomp.microsoftclusterversion and 
            % parallel.internal.types.HPCServerClusterVersion enums
            version = 'CCS';
        end

        %---------------------------------------------------------------
        % testClientCompatibilityWithMicrosoftAPI
        %---------------------------------------------------------------
        % Test whether or not Microsoft.ComputeCluster.Cluster can be 
        % constructed.
        function isCompatible = testClientCompatibilityWithMicrosoftAPI
            isCompatible = distcomp.CCSSchedulerConnection.FoundComputeClusterLibraries;
        end
    end
    
    methods (Static, Access = private) % Methods to support Constant, protected properties.
        % Methods in this section should NEVER be called directly - use the constant properties instead.

        %---------------------------------------------------------------
        % addAndCreateMicrosoftComputeCluster
        %---------------------------------------------------------------
        % NB This will only get called once by the constant FoundComputeClusterLibraries
        % property.  distcomp.FoundComputeClusterLibraries will "persist" until
        % "clear classes" is called.
        function canCreate = addAndCreateMicrosoftComputeCluster
            % Return early if we aren't on Windows
            if ~ispc
                canCreate = false;
                return;
            end
            try
                NET.addAssembly('ccpapi');
                % See if we can create a .NET connection to the scheduler
                testObj = Microsoft.ComputeCluster.Cluster;
                % Make sure we dispose of it as well
                testObj.Dispose();
                canCreate = true;
            catch err %#ok<NASGU>
                canCreate = false;
            end
        end

        %---------------------------------------------------------------
        % getMicrosoftJobStates
        %---------------------------------------------------------------
        % NB This will only get called once by the constant MicrosoftJobStates
        % property.  distcomp.MicrosoftJobStates will "persist" until
        % "clear classes" is called.
        function jobStates = getMicrosoftJobStates
            import Microsoft.ComputeCluster.*
            
            % Return empty if Microsoft.Hpc.Scheduler.dll hasn't been loaded.
            if ~distcomp.CCSSchedulerConnection.FoundComputeClusterLibraries
                jobStates = [];
                return;
            end
            
            try
                % Map the job states to values in distcomp.jobexecutionstate.
                % NB Ensure that all the job states listed in
                % http://msdn.microsoft.com/en-us/library/microsoft.computecluster.jobstatus(v=VS.85).aspx
                % appear here.
                jobStates = {
                    JobStatus.Cancelled; ...              %'finished'; ...
                    JobStatus.Failed; ...                 %'failed'; ...
                    JobStatus.Finished; ...               %'finished'; ...
                    JobStatus.NotSubmitted; ...           %'pending'; ...
                    JobStatus.Queued; ...                 %'queued'; ...
                    JobStatus.Running; ...                %'running'; ...
                    };
            catch err
                % Really shouldn't ever get in here because we've already checked
                % if Microsoft.Hpc.Scheduler.dll is loaded
                dctSchedulerMessage(5, 'Failed to get job states from Microsoft.ComputeCluster.\nReason: %s', ...
                    err.getReport());
            end
        end
        
        %---------------------------------------------------------------
        % getDistcompJobStates
        %---------------------------------------------------------------
        % NB This will only get called once by the constant MatlabJobStates
        % property.  distcomp.MatlabJobStates will "persist" until
        % "clear classes" is called.
        function jobStates = getMatlabJobStates
            % NB The states listed here MUST match the order of the states
            % listed in the getMicrosoftJobStates function!
            jobStates = {
                    'finished'; ...     %JobStatus.Cancelled; ...
                    'failed'; ...       %JobStatus.Failed; ...
                    'finished'; ...     %JobStatus.Finished; ...
                    'pending'; ...      %JobStatus.NotSubmitted; ...
                    'queued'; ...       %JobStatus.Queued; ...                 
                    'running'; ...      %JobStatus.Running; ...                
                };
        end
    end

    methods (Static, Access = private) % Other static, private methods
        %---------------------------------------------------------------
        % isTaskStateOKToCancel
        %---------------------------------------------------------------
        function isOK = isTaskStateOKToCancel(microsoftTaskState)
            import Microsoft.ComputeCluster.*
            % From http://msdn.microsoft.com/en-us/library/microsoft.computecluster.icluster.canceltask(v=VS.85).aspx
            % To cancel a task, the task's status must be: not submitted, queued, or running. 
            taskStateOKToCancel = {
                TaskStatus.NotSubmitted; ...
                TaskStatus.Queued; ...
                TaskStatus.Running};
        
            % We really want to do "ismember" here, but ismember isn't supported
            % for .NET types.
            compareFcn = @(x) isequal(x, microsoftTaskState);
            isOK = any(cellfun(compareFcn, taskStateOKToCancel));
        end
        
        %---------------------------------------------------------------
        % isJobStateOKToCancel
        %---------------------------------------------------------------
        function isOK = isJobStateOKToCancel(microsoftJobState)
            import Microsoft.ComputeCluster.*
            % Only cancel the job if the scheduler thinks it's not finished
            % From http://msdn.microsoft.com/en-us/library/microsoft.computecluster.icluster.canceljob(v=VS.85).aspx
            % To cancel a job, the job's status must be: not submitted, queued, or running. 
            jobStateOKToCancel = {
                JobStatus.NotSubmitted; ...
                JobStatus.Queued; ...
                JobStatus.Running};

            % We really want to do "ismember" here, but ismember isn't supported
            % for .NET types
            compareFcn = @(x) isequal(x, microsoftJobState);
            isOK = any(cellfun(compareFcn, jobStateOKToCancel));
        end
        
        %---------------------------------------------------------------
        % getMatlabJobStateFromMicrosoftState
        %---------------------------------------------------------------
        function matlabJobState = getMatlabJobStateFromMicrosoftState(microsoftJobState)
            compareFcn = @(x) isequal(x, microsoftJobState);
            stateIndex = cellfun(compareFcn, distcomp.CCSSchedulerConnection.MicrosoftJobStates);
            if ~any(stateIndex)
                % Just being paranoid.  Shouldn't get here.
                error(message('parallel:cluster:HPCServerUnknownJobStatus', microsoftJobState));
            end
            matlabJobState = distcomp.CCSSchedulerConnection.MatlabJobStates{stateIndex};
        end
    end

    methods
        %---------------------------------------------------------------
        % CCSSchedulerConnection constructor
        %---------------------------------------------------------------
        function obj = CCSSchedulerConnection
            if ~obj.FoundComputeClusterLibraries
                error(message('parallel:cluster:HPCServerCannotFindCCSLibraries'));
            end
            
            try
                % Create the .NET connection to the scheduler
                obj.ComputeClusterConnection = Microsoft.ComputeCluster.Cluster;
            catch err
                ex = MException(message('parallel:cluster:HPCServerCannotCreateCCS'));
                ex = ex.addCause(err);
                throw(ex);
            end
        end
        
        %---------------------------------------------------------------
        % Delete
        %---------------------------------------------------------------
        function delete(obj)
            % Dispose of the .NET scheduler connection
            % Note that .NET Dispose functions should never throw exceptions
            % Note: obj.ComputeClusterConnection will never be empty
            if ~isempty(obj.ComputeClusterConnection) && isa(obj.ComputeClusterConnection, 'System.IDisposable')
                obj.ComputeClusterConnection.Dispose();
            end
        end

    end
    
    methods % AbstractMicrosoftSchedulerConnection Abstract public method implementation
        
        %---------------------------------------------------------------
        % connect
        %---------------------------------------------------------------
        % Connect to the specified scheduler hostname.  Note that v1 server connections
        % can connect to both v1 (CCS) and v2 (HPCServer) schedulers.
        function connect(obj, schedulerHostname)
            if obj.isConnectedToScheduler(schedulerHostname)
                % Already connected to that scheduler
                return;
            end
            
            try
                obj.ComputeClusterConnection.Connect(schedulerHostname);
                % NB must set the hostname and version when we successfully connect
                obj.SchedulerVersion = obj.getVersionFromScheduler;
                obj.SchedulerHostname = schedulerHostname;
                % Indicate that we've connected at least once
                obj.HaveSuccessfullyConnectedOnce = true;
            catch err
                % Throw an error
                ex = MException(message('parallel:cluster:HPCServerCannotConnectCCS', schedulerHostname));
                ex = ex.addCause(err);
                throw(ex)
            end
            
            % Always set the maximum number of workers to a sensible value when 
            % connecting to a new cluster
            obj.MaximumNumberOfWorkersPerJob = obj.TotalNumberOfCores;
        end
        
        %---------------------------------------------------------------
        % submitJob 
        %---------------------------------------------------------------
        % NB Assumes that the job has already been prepared for submission
        %
        % matlabExe                             the matlab executable that should be run
        % matlabArgs                            the command line arguments for the matlabExe
        % jobEnvironmentVariables               all the environment variables that are common for the distributed job
        % taskLocationEnvironmentVariableName   the name of the environment variable to use for the task location
        % jobID                                 the ID for the job
        % jobLocation                           the full path to the job location
        % taskIDs                               the IDs for all the tasks
        % taskLocations                         task location relative to the datalocation
        % taskLogLocations                      an array of log locations (full path) for each task
        % username                              the username which will be used to launch the matlab processes
        % password                              the password associated with the username (an empty string will
        %                                       cause the user to be prompted to enter a password).
        function [schedulerJobIDs, schedulerTaskIDs, schedulerJobName] = submitJob(obj, matlabExe, matlabArgs, ...
                jobEnvironmentVariables, taskLocationEnvironmentVariableName, ...
                ~, ~, ~, taskLocations, taskLogLocations, ...
                username, password, ~)
            obj.errorIfNotConnected;
            
            % Job Name is relevant only for SOA v2 jobs
            schedulerJobName = '';
            % We're sure the cluster connection is actually connected to a scheduler
            numTasks = numel(taskLocations);
            matlabCommand = sprintf('"%s" %s', matlabExe,  matlabArgs);
            
            % Determine whether or not we should set min/max and isExclusive properties on ccsJob 
            % or if we should defer to the values already loaded from the XML
            try
                [minWorkersDefined, maxWorkersDefined, isExclusiveDefined] = obj.parseDescriptionFile;
            catch err
                % Something went wrong when parsing the XML file. (Unlikely to get in here
                % since if the XML file is incorrect, createJobOnScheduler would already have
                % errored.)
                % Assume that we need to override the properties.
                warning(message('parallel:cluster:HPCServerFailedToParseJobDescriptionFile', err.message));
                minWorkersDefined = false;
                maxWorkersDefined = false;
                isExclusiveDefined = false;
            end

            % Create a new job
            ccsJob = obj.createJobOnScheduler();
            % Set minimum and maximum resource usage and IsExclusive as appropriate
            if ~maxWorkersDefined
                ccsJob.MaximumNumberOfProcessors = min(numTasks, obj.MaximumNumberOfWorkersPerJob);
            end
            if ~minWorkersDefined
                ccsJob.MinimumNumberOfProcessors = 1;
            end
            if ~isExclusiveDefined
                ccsJob.IsExclusive = obj.DoNotUseResourcesExclusively;
            end
            
            % For each task add the task to the job and set the environment variables
            % Need to store the ID of the task to reindex into the CCS tasks
            schedulerTaskIDs = zeros(numTasks, 1);
            for i = 1:numTasks
                % Create the task
                t = obj.ComputeClusterConnection.CreateTask();
                
                % Now set the common environment variables
                for k = 1:size(jobEnvironmentVariables, 1)
                    t.SetEnvironmentVariable(jobEnvironmentVariables{k, 1}, jobEnvironmentVariables{k, 2});
                end
                % Additional task-specific environment variables
                t.SetEnvironmentVariable(taskLocationEnvironmentVariableName, taskLocations{i});
                
                % Redirect stdout and stderr to the log
                t.Stdout = taskLogLocations{i};
                t.Stderr = taskLogLocations{i};
                
                % Setting task to not exclusive ensures that multiple tasks from the same job 
                % can run on the same node.
                t.IsExclusive = obj.DoNotUseResourcesExclusively;
                t.CommandLine = matlabCommand;
                
                % Now add the task to the job
                ccsJob.AddTask(t);
                % NOTE - CCS doesn't actually know the ID of it's task at this point so
                % we will assume it grows monotonically from 1
                schedulerTaskIDs(i) = i;
            end
            
            % Now queue the job on the current controller to get a job id
            ccsJobID = obj.ComputeClusterConnection.AddJob(ccsJob);
            
            % Set the credentials for the job and submit the job
            obj.ComputeClusterConnection.SubmitJob(ccsJobID, username, password, ...
                obj.IsConsoleApplication, obj.ParentWindow);
            
            % Only one ccs job ID for v1 jobs.
            schedulerJobIDs = ccsJobID;
        end
        
        %---------------------------------------------------------------
        % submitParallelJob
        %---------------------------------------------------------------
        % NB Assumes that the job's tasks have already been duplicated and that the job has been
        % prepared for submission
        %
        % taskCommandToRun          the command to run by the task (i.e. mpiexec .......)
        % workersRange              a vector of 2 numbers indicating min/max workers
        % numTasks                  the number of tasks in this job
        % jobEnvironmentVariables   all the environment variables that are common for the distributed job
        % logLocation               the (full path to the) location to use for the task's log
        % username                  the username which will be used to launch the matlab processes
        % password                  the password associated with the username (an empty string will
        %                           cause the user to be prompted to enter a password).
        function [schedulerJobIDs, schedulerTaskIDs] = submitParallelJob(obj, taskCommandToRun, ...
                workersRange, numTasks, ...
                jobEnvironmentVariables, logLocation, ...
                username, password)
            obj.errorIfNotConnected;
            
            % Ensure we have enough cores on the cluster
            minProcessors = workersRange(1);
            maxProcessors = workersRange(2);
            totalCores = obj.TotalNumberOfCores;
            assert(totalCores >= minProcessors, 'parallel:cluster:HPCServerResourceLimit', ...
                    'The job has requested %d workers. The scheduler only has %d cores in the cluster.', ...
                    minProcessors, totalCores);
            
            % Determine whether or not we should set isExclusive or 
            % if we should defer to the XML file values.  We aren't interested
            % in whether or not the XML file defines min/max processors, as we
            % will be using the values defined in the parallel job.
            try
                [~, ~, isExclusiveDefined] = obj.parseDescriptionFile;
            catch err
                % Something went wrong when parsing the XML file. (Unlikely to get in here
                % since if the XML file is incorrect, createJobOnScheduler would already have
                % errored.)
                % Assume that we need to override the properties.
                warning(message('parallel:cluster:HPCServerFailedToParseJobDescriptionFile', err.message));
                isExclusiveDefined = false;
            end

            % Create a new job
            ccsJob = obj.createJobOnScheduler;
            % Always set the requested number of processors for parallel jobs, regardless
            % of whether these values may/may not be defined in the XML.
            ccsJob.MaximumNumberOfProcessors = maxProcessors;
            ccsJob.MinimumNumberOfProcessors = minProcessors;
            if ~isExclusiveDefined
                ccsJob.IsExclusive = obj.DoNotUseResourcesExclusively;
            end
            
            % All tasks will attach to task 1 on the scheduler
            schedulerTaskIDs = ones(numTasks, 1);
            
            % Create the one parallel task
            t = obj.ComputeClusterConnection.CreateTask();
            % Set the requested number of processors
            t.MaximumNumberOfProcessors = maxProcessors;
            t.MinimumNumberOfProcessors = minProcessors;
            % Setting task to not exclusive ensures that multiple tasks from the same job 
            % can run on the same node.
            t.IsExclusive = obj.DoNotUseResourcesExclusively;
            
            % Now set the environment variables
            for k = 1:size(jobEnvironmentVariables, 1)
                t.SetEnvironmentVariable(jobEnvironmentVariables{k, 1}, jobEnvironmentVariables{k, 2});
            end
            
            if ~isempty(logLocation)
                % Redirect stdout and stderr to the log
                t.Stdout = logLocation;
                t.Stderr = logLocation;
            end
            
            t.CommandLine = taskCommandToRun;
            
            % Now add the task to the job
            ccsJob.AddTask(t);
            
            % Now queue the job on the current controller to get a job id
            ccsJobID = obj.ComputeClusterConnection.AddJob(ccsJob);
            
            % Set the credentials for the job and submit the job
            obj.ComputeClusterConnection.SubmitJob(ccsJobID, username, password, ...
                obj.IsConsoleApplication, obj.ParentWindow);
            
            % Only one ccs job ID for v1 jobs.
            schedulerJobIDs = ccsJobID;
        end
        
        %---------------------------------------------------------------
        % cancelTaskByID
        %---------------------------------------------------------------
        % Cancel the specified task in the specified job on CCS.  The jobID and taskID
        % are CCS's IDs.
        function cancelTaskByID(obj, jobID, taskID)
            obj.errorIfNotConnected;
            % Only cancel the task if the scheduler thinks it's not finished.
            % ccpapi.dll will return an empty task if it doesn't actually exist.
            t = obj.ComputeClusterConnection.GetTask(jobID, taskID);
            if isempty(t)
                warning(message('parallel:cluster:HPCServerUnableToCancelTask', taskID, jobID));
                return;
            end
            
            if obj.isTaskStateOKToCancel(t.Status)
                % And cancel the task
                cancellationMessage = '';
                obj.ComputeClusterConnection.CancelTask(jobID, taskID, cancellationMessage);
            end
        end
        
        %---------------------------------------------------------------
        % getJobStateByID
        %---------------------------------------------------------------
        % Get the state of the specified job.  The jobID is the CCS job ID.
        function state = getJobStateByID(obj, jobID)
            obj.errorIfNotConnected;

            % Ask CCS about this job
            j = obj.getJobByID(jobID);
            state = obj.getMatlabJobStateFromMicrosoftState(j.Status);
        end
        
        %---------------------------------------------------------------
        % getSchedulerDetailsForFailedJob
        %---------------------------------------------------------------
        % Get job details from CCS for the specified job and task IDs.
        % Returns a string for use in debug logs.  The jobID and taskID are
        % CCS's IDs. TaskID is optional
        function jobDetails = getSchedulerDetailsForFailedJob(obj, jobID, taskID)
            obj.errorIfNotConnected;
            
            if nargin < 2
                taskID = [];
            end

            % Start with the error message for the job
            try
                ccsJob = obj.getJobByID(jobID);
            catch err %#ok<NASGU>
                jobDetails = getString(message('parallel:cluster:HPCServerSchedDetailsUnableToRetrieveJob', jobID));
                return;
            end

            % We have a valid job, so find the job details
            cellout{1} = getString(message('parallel:cluster:CCSSchedDetailsGettingDataForJob', jobID));
            cellout{2} = getString(message('parallel:cluster:HPCServerSchedDetailsErrorMessage', char(ccsJob.ErrorMessage)));
            
            % Then the details for the task, if specified
            if ~isempty(taskID)
                ccsTask = obj.ComputeClusterConnection.GetTask(jobID, taskID);
                if isempty(ccsTask)
                    cellout{3} = getString(message('parallel:cluster:HPCServerSchedDetailsUnableToRetrieveTask', taskID, jobID));
                else
                    cellout{3} = getString(message('parallel:cluster:CCSSchedDetailsGettingDataForTask', jobID, taskID));
                    cellout{4} = getString(message('parallel:cluster:HPCServerSchedDetailsCommandLine', char(ccsTask.CommandLine)));
                    cellout{5} = getString(message('parallel:cluster:HPCServerSchedDetailsStdOut', char(ccsTask.Stdout)));
                    cellout{6} = getString(message('parallel:cluster:HPCServerSchedDetailsErrorMessage', char(ccsTask.ErrorMessage)));
                end
            end
            jobDetails = sprintf('%s', cellout{:});
        end
        
        
        %---------------------------------------------------------------
        % getAllClusterEnvironmentVariables
        %---------------------------------------------------------------
        function [names, values] = getAllClusterEnvironmentVariables(obj)
            obj.errorIfNotConnected;
            allVariables = obj.ComputeClusterConnection.EnvironmentVariables;
            names = cell(allVariables.Count, 1);
            values = cell(allVariables.Count, 1);

            try
                enumerator = allVariables.GetEnumerator();
                % The documentation indicates that the enumerator is always 
                % positioned before the first element of the collection:
                % http://msdn.microsoft.com/en-us/library/system.collections.ienumerator(v=VS.85).aspx
                % but there appears to be no Reset method that we can use from 
                % MATLAB, so just double check that the current is empty
                if ~isempty(enumerator.Current)
                    error(message('parallel:cluster:HPCServerUnexpectedEnumerator'));
                end

                ii = 1;
                while enumerator.MoveNext()
                    currentVariable = enumerator.Current();
                    names{ii} = char(currentVariable.Name);
                    values{ii} = char(currentVariable.Value);
                    ii = ii + 1;
                end
            catch err
                ex = MException(message('parallel:cluster:HPCServerFailedToGetEnvironmentVariables', variableName, obj.SchedulerHostname));
                ex = ex.addCause(err);
                throw(ex);
            end
        end
    end

    methods (Access = protected) % AbstractMicrosoftSchedulerConnection Abstract protected method implementation
        %---------------------------------------------------------------
        % getHostnameFromScheduler
        %---------------------------------------------------------------
        % Get the scheduler hostname from the server connection.
        function hostname = getHostnameFromScheduler(obj)
            try
                hostname = obj.ComputeClusterConnection.Name;
            catch err
                ex = MException(message('parallel:cluster:HPCServerFailedToRetrieveNameFromCCS'));
                ex = ex.addCause(err);
                throw(ex);
            end
            
            % NB Microsoft.ComputeCluster.Cluster sets name to an empty string
            % when it is constructed.  Furthermore, attempting to connect to a
            % hostname = '' will generate an error.
            % Therefore, if the hostname is an empty string, then we know that
            % we're not actually connected to a scheduler
            if strcmp(hostname, '')
                error(message('parallel:cluster:HPCServerNotConnectedToCCS'))
            end
        end
        
        %---------------------------------------------------------------
        % getVersionFromScheduler
        %---------------------------------------------------------------
        % Get the scheduler version from the server connection.
        % This is a bit of a hack for CCS because there is no way of 
        % querying the scheduler to find out which version it is, so
        % we just have to assume that if we are using the CCS api to 
        % connect to the scheduler, then it must be version 1.
        function version = getVersionFromScheduler(~)
            version = 1;
        end

        
        %---------------------------------------------------------------
        % getTotalCores
        %---------------------------------------------------------------
        % Gets the total number of cores in the cluster. 
        % Note that this value represents the actual number of cores in the cluster
        % and does not provide any information about how many cores actually belong
        % to compute nodes.
        function totalCores = getTotalCores(obj)
            totalCores = obj.ComputeClusterConnection.ClusterCounter.TotalNumberOfProcessors;
        end
        
        %---------------------------------------------------------------
        % cancelJobsByID
        %---------------------------------------------------------------
        % Cancel the specified job on CCS.  jobID is a vector of CCS job IDs.
        function cancelJobsByID(obj, jobID)
            if isempty(jobID)
                return;
            end

            numJobsToCancel = length(jobID);
            allErrors = {};
            allErroredJobIds = -ones(numJobsToCancel, 1);
            for i = 1:numJobsToCancel
                currJobID = jobID(i);
                try
                    ccsJob = obj.getJobByID(currJobID);
                    
                    if obj.isJobStateOKToCancel(ccsJob.Status)
                        % And cancel the job
                        cancellationMessage = '';
                        obj.ComputeClusterConnection.CancelJob(currJobID, cancellationMessage);
                    end
                catch err
                    % Save up the errors for a MultipleException later
                    allErroredJobIds(i) = currJobID;
                    allErrors = [allErrors; err]; %#ok<AGROW>
                end
            end

            % Now throw the errors that were caught during the cancellation
            if ~isempty(allErrors)
                actualErroredJobIds = allErroredJobIds(allErroredJobIds ~= -1);
                err = MException(message('parallel:cluster:HPCServerFailedToCancelOnScheduler', sprintf( ' %d', actualErroredJobIds )));
                for ii = 1:numel(allErrors)
                    err = err.addCause(allErrors(ii));
                end
                throw(err);
            end
        end
    end

    methods (Access = protected)  
        %---------------------------------------------------------------
        % getJobByID
        %---------------------------------------------------------------
        function job = getJobByID(obj, jobID)
            % NB For V1, the returned job is empty if the job doesn't actually exist on the scheduler.
            % No error is thrown, so we need to throw our own error if the job couldn't be found (to 
            % ensure that the behaviour here is consistent with HPCServerSchedulerConnection).
            job = obj.ComputeClusterConnection.GetJob(jobID);
            if isempty(job)
                error(message('parallel:cluster:HPCServerUnableToRetrieveJobNotExist', jobID));
            end
        end

        %---------------------------------------------------------------
        % createJob
        %---------------------------------------------------------------
        % Creates a ccs Job, applying the job XML Description file, if appropriate.
        function schedulerJob = createJobOnScheduler(obj)
            % If an XML file is defined, then create the job from the XML.
            
            if isempty(obj.JobDescriptionFile)
                % Create a new job
                schedulerJob = obj.ComputeClusterConnection.CreateJob();
                return;
            end
            
            try
                schedulerJob = obj.ComputeClusterConnection.CreateJobFromXmlFile(obj.JobDescriptionFile);
            catch err
                ex = MException(message('parallel:cluster:HPCServerFailedToCreateJobFromXML', obj.JobDescriptionFile));
                ex = ex.addCause(err);
                throw(ex);
            end
        end

        %---------------------------------------------------------------
        % parseDescriptionFile
        %---------------------------------------------------------------
        % Parses the JobDescriptionFile and determines if the attributes for
        % min/max processors and IsExclusive are defined in the file.
        function [minWorkersDefined, maxWorkersDefined, isExclusiveDefined] = parseDescriptionFile(obj)
            if isempty(obj.JobDescriptionFile)
                minWorkersDefined = false;
                maxWorkersDefined = false;
                isExclusiveDefined = false;
                return;
            end
            
            % Read JobDescriptionFile into struct
            xmlFile = readstruct(obj.JobDescriptionFile);
            
            % Ensure there is exactly 1 Job entry
            if ~isfield(xmlFile, 'Job')
                error(message('parallel:cluster:HPCServerDescriptionFileNoJob', obj.JobDescriptionFile));
            end
            jobNode = xmlFile.Job;
            if numel(jobNode) > 1
                error(message('parallel:cluster:HPCServerDescriptionFileTooManyJobs', obj.JobDescriptionFile, numel(jobNode)));
            end
            
            % If jobNode is empty, isfield will return false as desired
            minWorkersDefined = isfield(jobNode, 'MinimumNumberOfProcessorsAttribute');
            maxWorkersDefined = isfield(jobNode, 'MaximumNumberOfProcessorsAttribute');
            isExclusiveDefined = isfield(jobNode, 'IsExclusiveAttribute');
        end
    end

end

