classdef HPCServerSchedulerConnection < distcomp.AbstractMicrosoftSchedulerConnection
    %HPCServerSchedulerConnection Class for connections to Microsoft HPC Server scheduler APIs
    %   Class for interfacing to Microsoft.Hpc.Scheduler.Scheduler.dll using
    %   MATLAB Interface to .NET Framework
    
    %  Copyright 2009-2022 The MathWorks, Inc.
    
    
    properties (Dependent)
        % NB AbortSet MUST be false for UseSOAJobSubmission and JobTemplate.
        % AbortSet is false by default.
        UseSOAJobSubmission;
        JobTemplate;
    end
    
    properties (Constant, GetAccess = protected)
        FoundSchedulerLibraries = distcomp.HPCServerSchedulerConnection.addAndCreateMicrosoftHpcScheduler();
        
        MicrosoftJobStates = distcomp.HPCServerSchedulerConnection.getMicrosoftJobStates();
        MatlabJobStates = distcomp.HPCServerSchedulerConnection.getMatlabJobStates();
        
        DefaultJobTemplate = distcomp.HPCServerSchedulerConnection.getDefaultJobTemplate();
    end
    
    properties (Access = private)
        % The actual connection to the HPC Server API.  This is an .NET object
        % of type Microsoft.Hpc.Scheduler.Scheduler
        HPCSchedulerConnection;
        % The connection to the SOA client for HPC Server.  This is an object
        % of type AbstractHPCServerSOAConnection
        SOAConnection;
        % NB AbortSet MUST be false for PrivateUseSOAJobSubmission and PrivateJobTemplate.
        % AbortSet is false by default.
        % Private UseSOA to support the dependent UseSOAJobSubmission property
        PrivateUseSOAJobSubmission = distcomp.HPCServerSchedulerConnection.DoNotUseSOAJobSubmission;
        % Private JobTemplate to support the dependent JobTemplate property
        % Default to empty to start with.
        PrivateJobTemplate = '';
    end
    
    properties (Constant, GetAccess = private)
        DoNotUseSOAJobSubmission = false;
    end
    
    
    methods (Static)% AbstractMicrosoftSchedulerConnection Abstract, static method implementation
        %---------------------------------------------------------------
        % getAPIVersion
        %---------------------------------------------------------------
        function version = getAPIVersion
            version = parallel.internal.types.HPCServerClusterVersion.HPCServer.Name;
        end

        %---------------------------------------------------------------
        % testClientCompatibilityWithMicrosoftAPI
        %---------------------------------------------------------------
        % Test whether or not Microsoft.Hpc.Scheduler.Scheduler can be 
        % constructed.  
        function isCompatible = testClientCompatibilityWithMicrosoftAPI
            isCompatible = distcomp.HPCServerSchedulerConnection.FoundSchedulerLibraries;
        end
    end
    
    methods (Static, Access = private) % Methods to support Constant, protected properties.
        % Methods in this section should NEVER be called directly - use the constant properties instead.
        
        %---------------------------------------------------------------
        % addAndCreateMicrosoftHpcScheduler
        %---------------------------------------------------------------
        % NB This will only get called once by the constant FoundSchedulerLibraries
        % property.  FoundSchedulerLibraries will "persist" until
        % "clear classes" is called.
        function success = addAndCreateMicrosoftHpcScheduler
            % Return early if we aren't on the correct platform
            if ~ispc
                success = false;
                return;
            end
            
            try
                parallel.internal.apishared.HPCServerUtils.addHPCServerAssembly('Microsoft.Hpc.Scheduler');
                parallel.internal.apishared.HPCServerUtils.addHPCServerAssembly('Microsoft.Hpc.Scheduler.Properties');
                % See if we can create a .NET connection to the scheduler
                testObj = Microsoft.Hpc.Scheduler.Scheduler;
                % Make sure we dispose of it as well
                testObj.Dispose();
                success = true;
            catch err %#ok<NASGU>
                success = false;
            end
        end
        
        %---------------------------------------------------------------
        % getMicrosoftJobStates
        %---------------------------------------------------------------
        % NB This will only get called once by the constant MicrosoftJobStates
        % property.  distcomp.MicrosoftJobStates will "persist" until
        % "clear classes" is called.
        function jobStates = getMicrosoftJobStates
            import Microsoft.Hpc.Scheduler.Properties.*
            
            % Return empty if Microsoft.Hpc.Scheduler.dll hasn't been loaded.
            if ~distcomp.HPCServerSchedulerConnection.FoundSchedulerLibraries
                jobStates = [];
                return;
            end
            
            try
                % Map the job states to values in distcomp.jobexecutionstate.
                % NB Ensure that all the job states listed in
                % http://msdn.microsoft.com/en-us/library/microsoft.hpc.scheduler.properties.jobstate(VS.85).aspx
                % appear here.
                jobStates = {
                    JobState.Canceled; ...              %'finished'; ...
                    JobState.Canceling; ...             %'finished'; ...
                    JobState.Configuring; ...           %'queued'; ...
                    JobState.ExternalValidation; ...    %'queued'; ...
                    JobState.Failed; ...                %'failed'; ...
                    JobState.Finished; ...              %'finished'; ...
                    JobState.Finishing; ...             %'finished'; ...
                    JobState.Queued; ...                %'queued'; ...
                    JobState.Running; ...               %'running'; ...
                    JobState.Submitted; ...             %'queued'; ...
                    JobState.Validating; ...            %'queued'; ...
                    };
            catch err
                % Really shouldn't ever get in here because we've already checked
                % if Microsoft.Hpc.Scheduler.dll is loaded
                dctSchedulerMessage(5, 'Failed to get job states from Microsoft.Hpc.Scheduler.Properties.\nReason: %s', ...
                    err.getReport());
            end
        end
        
        %---------------------------------------------------------------
        % getDistcompJobStates
        %---------------------------------------------------------------
        % NB This will only get called once by the constant MatlabJobStates
        % property.  distcomp.MatlabJobStates will "persist" until
        % "clear classes" is called.
        function jobStates = getMatlabJobStates
            % NB The states listed here MUST match the order of the states
            % listed in the getMicrosoftJobStatesAsDouble function!
            jobStates = {
                'finished'; ...     %JobState.Canceled,
                'finished'; ...     %JobState.Canceling,
                'queued'; ...       %JobState.Configuring,
                'queued'; ...       %JobState.ExternalValidation,
                'failed'; ...       %JobState.Failed,
                'finished'; ...     %JobState.Finished,
                'finished'; ...     %JobState.Finishing,
                'queued'; ...       %JobState.Queued,
                'running'; ...      %JobState.Running,
                'queued'; ...       %JobState.Submitted,
                'queued'; ...       %JobState.Validating,
                };
        end
        
        %---------------------------------------------------------------
        % getDefaultJobTemplate
        %---------------------------------------------------------------
        function defaultJobTemplate = getDefaultJobTemplate
            defaultJobTemplate = getenv('CCP_JOBTEMPLATE');
        end
    end
    
    methods (Static, Access = private) % Other static, private methods
        %---------------------------------------------------------------
        % isTaskStateOKToCancel
        %---------------------------------------------------------------
        function isOK = isTaskStateOKToCancel(microsoftTaskState)
            import Microsoft.Hpc.Scheduler.Properties.*
            % From http://msdn.microsoft.com/en-us/library/microsoft.hpc.scheduler.ischedulerjob.canceltask(VS.85).aspx
            % To cancel a task, the state of the task must be: Configuring, Submitted, Queued, or Running
            taskStateOKToCancel = {
                TaskState.Configuring; ...
                TaskState.Submitted; ...
                TaskState.Queued; ...
                TaskState.Running};

            % We really want to do "ismember" here, but ismember isn't supported
            % for .NET types.
            compareFcn = @(x) isequal(x, microsoftTaskState);
            isOK = any(cellfun(compareFcn, taskStateOKToCancel));
        end
        
        %---------------------------------------------------------------
        % isJobStateOKToCancel
        %---------------------------------------------------------------
        function isOK = isJobStateOKToCancel(microsoftJobState)
            import Microsoft.Hpc.Scheduler.Properties.*
            % Only cancel the job if the scheduler thinks it's not finished
            % From http://msdn.microsoft.com/en-us/library/microsoft.hpc.scheduler.ischeduler.canceljob(VS.85).aspx
            % To cancel a job, the state of the job must be
            % configuring, submitted, validating, queued, or running.
            jobStateOKToCancel = {
                JobState.Configuring; ...
                JobState.Submitted; ...
                JobState.Validating; ...
                JobState.Queued; ...
                JobState.Running};

            % We really want to do "ismember" here, but ismember isn't supported
            % for .NET types
            compareFcn = @(x) isequal(x, microsoftJobState);
            isOK = any(cellfun(compareFcn, jobStateOKToCancel));
        end
        
        %---------------------------------------------------------------
        % getMatlabJobStateFromMicrosoftState
        %---------------------------------------------------------------
        function matlabJobState = getMatlabJobStateFromMicrosoftState(microsoftJobState)
            compareFcn = @(x) isequal(x, microsoftJobState);
            stateIndex = cellfun(compareFcn, distcomp.HPCServerSchedulerConnection.MicrosoftJobStates);
            if ~any(stateIndex)
                % Just being paranoid.  Shouldn't get here.
                error(message('parallel:cluster:HPCServerUnknownJobStatus', microsoftJobState));
            end
            matlabJobState = distcomp.HPCServerSchedulerConnection.MatlabJobStates{stateIndex};
        end
    end
    
    methods
        %---------------------------------------------------------------
        % Constructor
        %---------------------------------------------------------------
        function obj = HPCServerSchedulerConnection
            if ~obj.FoundSchedulerLibraries
                error(message('parallel:cluster:HPCServerCannotFindLibraries'));
            end
            
            % See if we can create a .NET connection to the scheduler
            try
                obj.HPCSchedulerConnection = Microsoft.Hpc.Scheduler.Scheduler;
                % Tell the scheduler that MATLAB is not a console application - this ensures
                % that the password prompt dialog is displayed (if the user's credentials
                % aren't cached).  The default interface mode is console, which causes MATLAB
                % to appear as if it has hung because the prompt has disappeared to the "console"
                % which doesn't really exist.
                obj.HPCSchedulerConnection.SetInterfaceMode(obj.IsConsoleApplication, ...
                    System.IntPtr(obj.ParentWindow));
            catch err
                ex = MException(message('parallel:cluster:HPCServerCannotCreateHPCScheduler'));
                ex = ex.addCause(err);
                throw(ex);
            end
        end
        
        %---------------------------------------------------------------
        % Delete
        %---------------------------------------------------------------
        function delete(obj)
            % Dispose of the .NET scheduler connection
            % Note that .NET Dispose functions should never throw exceptions
            % Note: obj.HPCSchedulerConnection will never be empty, but
            % obj.SOAConnection may be empty if we couldn't create an SOA client.
            if ~isempty(obj.HPCSchedulerConnection) && isa(obj.HPCSchedulerConnection, 'System.IDisposable')
                obj.HPCSchedulerConnection.Dispose();
            end
        end
        
        %---------------------------------------------------------------
        % get.UseSOAJobSubmission
        %---------------------------------------------------------------
        function useSOA = get.UseSOAJobSubmission(obj)
            % just defer to the Private property
            useSOA = obj.PrivateUseSOAJobSubmission;
        end
        
        %---------------------------------------------------------------
        % set.UseSOAJobSubmission
        %---------------------------------------------------------------
        function set.UseSOAJobSubmission(obj, useSOA)
            % If useSOA is false, then we can always set it.
            % If useSOA is true AND we are not connected, then error
            % If useSOA is true AND there is no SOA connection, then warn that
            % SOA is not supported and default it back to false.
            % In all other circumstances, just set the useSOA value as requested.
            if useSOA 
                obj.errorIfNotConnected();
                if isempty(obj.SOAConnection)
                    warning(message('parallel:cluster:HPCServerDefaultToNonSOAJobs'));
                    useSOA = false;
                end
            end
            obj.PrivateUseSOAJobSubmission = useSOA;
        end
        
        %---------------------------------------------------------------
        % set.JobTemplate
        %---------------------------------------------------------------
        function set.JobTemplate(obj, jobTemplate)
            % Cannot set jobTemplate to a non-empty value if we are not connected.
            if ~isempty(jobTemplate)
                obj.errorIfNotConnected();
                % Check if the specified job template is a valid one
                allJobTemplates = obj.HPCSchedulerConnection.GetJobTemplateList;
                % Convert StringCollection to a cell array of strings
                allJobTemplateNames = cell(allJobTemplates.Count, 1);
                for i = 1:allJobTemplates.Count
                    allJobTemplateNames{i} = char(allJobTemplates.Item(i-1));
                end
                
                if ~any(strcmpi(allJobTemplateNames, jobTemplate))
                    error(message('parallel:cluster:HPCServerInvalidJobTemplate', jobTemplate));
                end
            end
            
            % Set the private job template property
            obj.PrivateJobTemplate = jobTemplate;
        end
        
        %---------------------------------------------------------------
        % get.JobTemplate
        %---------------------------------------------------------------
        function jobTemplate = get.JobTemplate(obj)
            % defer to the private property
            jobTemplate = obj.PrivateJobTemplate;
        end
    end
    
    methods % AbstractMicrosoftSchedulerConnection Abstract, public method implementation
        %---------------------------------------------------------------
        % connect
        %---------------------------------------------------------------
        % NB Limit the number of times Connect is called since this
        % leaks a thread every time a connection is made (even if you
        % connect to the same scheduler again).
        %
        % Note that once we have connected successfully, we will stay
        % connected to that scheduler.
        function connect(obj, schedulerHostname)
            if obj.isConnectedToScheduler(schedulerHostname)
                % Already connected to that scheduler
                return;
            end
            
            % We already know we are not connected to the requested cluster,
            % so if we are already connected, we must be connected to some other
            % cluster.  We only support one connection, so assert that we are
            % not connected.
            assert(~obj.IsConnected, 'parallel:cluster:HPCServerAlreadyConnected', ...
                'HPCSchedulerConnection is already connected to a scheduler.');
            
            try
                obj.HPCSchedulerConnection.Connect(schedulerHostname);
                % NB must set the hostname and version when we successfully connect
                obj.SchedulerVersion = obj.getVersionFromScheduler;
                obj.SchedulerHostname = schedulerHostname;
                % Indicate that we've connected at least once
                obj.HaveSuccessfullyConnectedOnce = true;
            catch err
                % Throw an error
                ex = MException(message('parallel:cluster:HPCServerCannotConnect', schedulerHostname));
                ex = ex.addCause(err);
                throw(ex)
            end
            
            % Try to create the SOA connection with the highest version first.  If that 
            % errors, we'll progressively downgrade.
            import parallel.internal.cluster.*
            try
                if HPCServerFireAndForgetSOAConnection.isClientCompatible(obj.SchedulerVersion)
                    obj.SOAConnection = HPCServerFireAndForgetSOAConnection(obj.HPCSchedulerConnection, obj.SchedulerHostname);
                elseif HPCServer2008SOAConnection.isClientCompatible(obj.SchedulerVersion)
                    obj.SOAConnection = HPCServer2008SOAConnection(obj.HPCSchedulerConnection, obj.SchedulerHostname);
                else
                    % Shouldn't get in here, but error so that we trigger
                    % the warning below
                    error(message('parallel:cluster:HPCServerUnsupportedSchedulerVersion', obj.SchedulerVersion));
                end
            catch err
                if obj.UseSOAJobSubmission
                    % Just warn here rather than throwing an error as we may not want to submit SOA jobs anyway.
                    warning(message('parallel:cluster:HPCServerUnableToConnectSOAClient', obj.SchedulerVersion, obj.SchedulerHostname, err.getReport()));
                end	
                obj.UseSOAJobSubmission = obj.DoNotUseSOAJobSubmission;
            end
            
            % If the service is installed centrally, then set UseSOAJobSubmission to true
            % because we know it exists.  If it hasn't been installed centrally, it may be
            % installed locally, or it may not be installed at all - we can't tell, so leave 
            % UseSOAJobSubmission as false.
            if ~isempty(obj.SOAConnection)
                obj.UseSOAJobSubmission = obj.SOAConnection.isServiceLikelyInstalledCentrally();
            end
            
            % Always set the maximum number of workers to a sensible value when
            % connecting to a new cluster
            obj.MaximumNumberOfWorkersPerJob = obj.TotalNumberOfCores;
            
            % Ensure we set the JobTemplate to the default value.  Note that 
            % JobTemplate is guaranteed to be empty at this point.  
            try
                obj.JobTemplate = obj.DefaultJobTemplate;
            catch err %#ok<NASGU>
                warning(message('parallel:cluster:HPCServerInvalidJobTemplateResetToDefault', obj.DefaultJobTemplate, obj.SchedulerHostname));
                % NB Empty string maps to the job template with name 'Default'.
                obj.JobTemplate = '';
            end
        end
        
        %---------------------------------------------------------------
        % submitJob
        %---------------------------------------------------------------
        % NB Assumes that the job has already been prepared for submission
        %
        % matlabExe                             the matlab executable that should be run
        % matlabArgs                            the command line arguments for the matlabExe
        % jobEnvironmentVariables               all the environment variables that are common for the distributed job
        % taskLocationEnvironmentVariableName   the name of the environment variable to use for the task location
        % jobID                                 the ID for the job
        % jobLocation                           the full path to the job location
        % taskIDs                               the IDs for all the tasks
        % taskLocations                         task location relative to the datalocation
        % taskLogLocations                      an array of log locations (full path) for each task
        % username                              the username which will be used to launch the matlab processes
        % password                              the password associated with the username (an empty string will
        %                                       cause the user to be prompted to enter a password)
        % numThreads                            number of cores to request per worker.
        function [schedulerJobIDs, schedulerTaskIDs, schedulerJobName] = submitJob(obj, matlabExe, matlabArgs, ...
                jobEnvironmentVariables, taskLocationEnvironmentVariableName, ...
                jobID, jobLocation, taskIDs, taskLocations, taskLogLocations, ...
                username, password, numThreads)
            obj.errorIfNotConnected;
            
            if obj.UseSOAJobSubmission
                [schedulerJobIDs, schedulerTaskIDs, schedulerJobName] = submitSOAJob(obj, ...
                    matlabExe, matlabArgs, ...
                    jobEnvironmentVariables, ...
                    jobID, jobLocation, taskIDs, taskLocations, taskLogLocations, ...
                    username, password);
            else
                [schedulerJobIDs, schedulerTaskIDs, schedulerJobName] = submitNonSOAJob(obj, ...
                    matlabExe, matlabArgs, ...
                    jobEnvironmentVariables, taskLocationEnvironmentVariableName, ...
                    taskLocations, taskLogLocations, ...
                    username, password, numThreads);
            end
        end
        
        %---------------------------------------------------------------
        % submitParallelJob
        %---------------------------------------------------------------
        % NB Assumes that the job's tasks have already been duplicated and that the job has been
        % prepared for submission
        %
        % taskCommandToRun          the command to run by the task (i.e. mpiexec .......)
        % workersRange              a vector of 2 numbers indicating min/max workers
        % numTasks                  the number of tasks in this job
        % jobEnvironmentVariables   all the environment variables that are common for the distributed job
        % logLocation               the (full path to the) location to use for the task's log
        % username                  the username which will be used to launch the matlab processes
        % password                  the password associated with the username (an empty string will
        %                           cause the user to be prompted to enter a password).
        function [schedulerJobIDs, schedulerTaskIDs] = submitParallelJob(obj, taskCommandToRun, ...
                workersRange, numTasks, ...
                jobEnvironmentVariables, logLocation, ...
                username, password)
            obj.errorIfNotConnected;
            
            % Ensure we have enough processors on the cluster.
            % NB we can only find out the total number of cores that are physically
            % available on the cluster - they may not all be configured as "compute
            % cores".  Furthermore, the job template may limit the maximum number of
            % cores that can get allocated.
            minProcessors = workersRange(1);
            maxProcessors = workersRange(2);
            totalCores = obj.TotalNumberOfCores;
            assert(totalCores >= minProcessors, 'parallel:cluster:HPCServerResourceLimit', ...
                'The job has requested %d workers. The scheduler only has %d cores in the cluster.', ...
                minProcessors, totalCores);
            
            % Create a new job
            hpcsJob = obj.createJobOnScheduler;
            % Always set the requested number of processors for parallel jobs
            hpcsJob.MaximumNumberOfCores = maxProcessors;
            hpcsJob.MinimumNumberOfCores = minProcessors;
            % Leave IsExclusive unset on the job - let the XML or Job Template take care of this
            
            % All tasks will attach to task 1 on the scheduler
            schedulerTaskIDs = ones(numTasks, 1);
            
            % Create the one parallel task
            t = hpcsJob.CreateTask();
            % Set the requested number of processors
            t.MaximumNumberOfCores = maxProcessors;
            t.MinimumNumberOfCores = minProcessors;
            % Neither the job XML nor job template defines IsExclusive on a task, so set it here
            % This ensures that multiple tasks from the same job can run on the same node.
            t.IsExclusive = obj.DoNotUseResourcesExclusively;
            
            % Now set the environment variables
            for k = 1:size(jobEnvironmentVariables, 1)
                t.SetEnvironmentVariable(jobEnvironmentVariables{k, 1}, jobEnvironmentVariables{k, 2});
            end
            
            if ~isempty(logLocation)
                % Redirect stdout and stderr to the log
                t.StdOutFilePath = logLocation;
                t.StdErrFilePath = logLocation;
            end
            
            t.CommandLine = taskCommandToRun;
            
            % Now add the task to the job
            hpcsJob.AddTask(t);
            
            % The V2 API has a bug such that if the password is an zero length string,
            % the credentials will not be retrieved from the cache, and the user is
            % prompted to enter their password every single time.  Setting the password
            % to [] (i.e. a null string) means that the cached credentials are correctly
            % used.
            if strcmpi(password, '')
                password = [];
            end
            % Set the credentials for the job and submit the job
            obj.HPCSchedulerConnection.SubmitJob(hpcsJob, username, password);
            
            schedulerJobIDs = double(hpcsJob.Id);
        end
        
        %---------------------------------------------------------------
        % cancelTaskByID
        %---------------------------------------------------------------
        % Cancel the specified task in the specified job on HPC Server.  The jobID and taskID
        % are HPC Server's IDs.
        function cancelTaskByID(obj, jobID, taskID)
            obj.errorIfNotConnected;
            
            job = obj.getJobByID(jobID);
            dotnetTaskID = Microsoft.Hpc.Scheduler.Properties.TaskId(taskID);
            try
                t = job.OpenTask(dotnetTaskID);
            catch err
                ex = MException(message('parallel:cluster:HPCServerUnableToFindTask', taskID, jobID));
                ex = ex.addCause(err);
                throw(ex);
            end
            
            if obj.isTaskStateOKToCancel(t.State)
                % Now cancel the task
                job.CancelTask(dotnetTaskID);
            end
        end
        
        %---------------------------------------------------------------
        % getJobStateByID
        %---------------------------------------------------------------
        % Get the state of the specified job.  The jobID is the HPC Server job ID.
        function state = getJobStateByID(obj, jobID)
            obj.errorIfNotConnected;
            j = obj.getJobByID(jobID);
            
            state = obj.getMatlabJobStateFromMicrosoftState(j.State);
        end
        
        %---------------------------------------------------------------
        % getSchedulerDetailsForFailedJob
        %---------------------------------------------------------------
        % Get job details from HPC Server for the specified job and task IDs.
        % Returns a string for use in debug logs.  The jobID and taskID are
        % HPC Server's IDs.  TaskID is an optional input argument.
        function jobDetails = getSchedulerDetailsForFailedJob(obj, jobID, taskID)
            obj.errorIfNotConnected;
            if nargin < 2
                taskID = [];
            end
            
            % Start with the error message for the job
            try
                job = obj.getJobByID(jobID);
            catch err %#ok<NASGU>
                jobDetails = getString(message('parallel:cluster:HPCServerSchedDetailsUnableToRetrieveJob', jobID));
                return;
            end
            
            cellout{1} = getString(message('parallel:cluster:HPCServerSchedDetailsGettingDataForJob', jobID));
            cellout{2} = getString(message('parallel:cluster:HPCServerSchedDetailsErrorMessage', char(job.ErrorMessage)));
            
            if ~isempty(taskID)
                % Need to use the full namespace name for TaskId,
                % otherwise it doesn't seem to work.
                hpcsTaskID = Microsoft.Hpc.Scheduler.Properties.TaskId(taskID);
                try
                    % job.OpenTask will throw an error if the task doesn't exist
                    % for this job
                    hpcsTask = job.OpenTask(hpcsTaskID);
                    cellout{3} = getString(message('parallel:cluster:HPCServerSchedDetailsGettingDataForTask', jobID, taskID));
                    % NB need to convert all System.Strings into char arrays.
                    cellout{4} = getString(message('parallel:cluster:HPCServerSchedDetailsCommandLine', char(hpcsTask.CommandLine)));
                    cellout{5} = getString(message('parallel:cluster:HPCServerSchedDetailsStdOut', char(hpcsTask.StdOutFilePath)));
                    cellout{6} = getString(message('parallel:cluster:HPCServerSchedDetailsErrorMessage', char(hpcsTask.ErrorMessage)));
                catch err %#ok<NASGU>
                    cellout{end+1} = getString(message('parallel:cluster:HPCServerSchedDetailsUnableToRetrieveTask', taskID, jobID));
                end
            end
            jobDetails = sprintf('%s', cellout{:});
        end

        
        %---------------------------------------------------------------
        % getAllClusterEnvironmentVariables
        %---------------------------------------------------------------
        function [names, values] = getAllClusterEnvironmentVariables(obj)
            obj.errorIfNotConnected;
            allVariables = obj.HPCSchedulerConnection.EnvironmentVariables;
            numVars = allVariables.Count;
            names = cell(numVars, 1);
            values = cell(numVars, 1);
            
            try
                % convert the variables from a Microsoft.Hpc.Scheduler.INameValueCollection to a generic list 
                % because INameValueCollection.GetEnumerator() doesn't seem to work.
                envsList = NET.createGeneric('System.Collections.Generic.List', ...
                    {'Microsoft.Hpc.Scheduler.NameValue'}, allVariables);

                for ii = 1:numVars
                    currEnv = envsList.Item(ii-1); % zero based indexing
                    names{ii} = char(currEnv.Name);
                    values{ii} = char(currEnv.Value);
                end
            catch err
                ex = MException(message('parallel:cluster:HPCServerFailedToGetEnvironmentVariables', variableName, obj.SchedulerHostname));
                ex = ex.addCause(err);
                throw(ex);
            end
        end
    end
    
    methods % AbstractMicrosoftSchedulerConnection overrides
        %---------------------------------------------------------------
        % cancelJob
        %---------------------------------------------------------------
        function cancelJob(obj, jobSchedulerData)
            obj.errorIfNotConnected;
            
            % If the job was an SOA job, then ensure that we close the SOA client
            if jobSchedulerData.IsSOAJob && ~isempty(obj.SOAConnection)
                try
                    obj.SOAConnection.closeJob(jobSchedulerData.SchedulerJobID, ...
                        jobSchedulerData.SchedulerJobName);
                catch err
                    warning(message('parallel:cluster:HPCServerCannotCloseSOAClient', err.message));
                end
            end
            
            % Call the AbstractMicrosoftSchedulerConnection cancel Job method to cancel the
            % jobs on the scheduler. (Note that closing the SOA client may already have
            % cancelled the jobs if the SOA had already started to run.  If the job
            % was still in the queued state, then closing the job on the SOA client will not cancel
            % the actual jobs.)
            cancelJob@distcomp.AbstractMicrosoftSchedulerConnection(obj, jobSchedulerData);
        end
    end
    
    methods (Access = protected) % AbstractMicrosoftSchedulerConnection Abstract, protected method implementation
        %---------------------------------------------------------------
        % getHostnameFromScheduler.  Returns the hostname as a MATLAB char array
        %---------------------------------------------------------------
        function hostname = getHostnameFromScheduler(obj)
            % BEWARE: A scheduler that has not been connected will throw an error when
            % GetNodesInNodeGroup (or pretty much any other method) is called.  Once this
            % has occurred, subsequent calls to GetNodesInNodeGroup after connection will
            % also throw an error.
            
            % Hack to get the scheduler hostname due to the lack of a "Name" property in v2.
            try
                allHeadNodes = obj.HPCSchedulerConnection.GetNodesInNodeGroup('HeadNodes');
            catch err
                ex = MException(message('parallel:cluster:HPCServerNoHeadNodesGroup'));
                ex = ex.addCause(err);
                throw(ex);
            end
            
            % Expect more than 0 head nodes
            if allHeadNodes.Count < 1
                error(message('parallel:cluster:HPCServerNoHeadNodes'));
            end
            % Use the first in the array for now.
            dotnetHostname = allHeadNodes.Item(0); % zero-based indexing for .NET types
            % convert from System.String to a char array
            hostname = char(dotnetHostname);
        end
        
        %---------------------------------------------------------------
        % getVersionFromScheduler
        %---------------------------------------------------------------
        % Get the scheduler version from the server connection.
        function version = getVersionFromScheduler(obj)
            try
                dotnetSchedulerVersion = obj.HPCSchedulerConnection.GetServerVersion().Major;
            catch err
                ex = MException(message('parallel:cluster:HPCServerCannotRetrieveVersion'));
                ex = ex.addCause(err);
                throw(ex);
            end
            
            % convert from .NET to MATLAB type
            version = double(dotnetSchedulerVersion);
        end
        
        %---------------------------------------------------------------
        % getTotalCores
        %---------------------------------------------------------------
        % Gets the total number of cores in the cluster.
        % Note that this value represents the actual number of cores in the cluster
        % and does not provide any information about how many cores actually belong
        % to compute nodes.
        function totalCores = getTotalCores(obj)
            counters = obj.HPCSchedulerConnection.GetCounters;
            totalCores = double(counters.TotalCores);
        end
        
        %---------------------------------------------------------------
        % cancelJobsByID
        %---------------------------------------------------------------
        % Cancel the specified job on HPC Server.  jobID is a vector of HPC Server job IDs.
        function cancelJobsByID(obj, jobID)
            if isempty(jobID)
                return;
            end
            
            numJobsToCancel = length(jobID);
            allErrors = {};
            allErroredJobIds = -ones(numJobsToCancel, 1);
            for i = 1:numJobsToCancel
                currJobID = jobID(i);
                try
                    hpcsJob = obj.getJobByID(currJobID);
                    
                    if obj.isJobStateOKToCancel(hpcsJob.State)
                        % And cancel the job
                        cancellationMessage = '';
                        obj.HPCSchedulerConnection.CancelJob(currJobID, cancellationMessage);
                    end
                catch err
                    % Save up the errors for a MultipleException later
                    allErroredJobIds(i) = currJobID;
                    allErrors = [allErrors; err]; %#ok<AGROW>
                end
            end
            
            % Now throw the errors that were caught during the cancellation
            if ~isempty(allErrors)
                actualErroredJobIds = allErroredJobIds(allErroredJobIds ~= -1);
                err = MException(message('parallel:cluster:HPCServerFailedToCancelOnScheduler', sprintf( ' %d', actualErroredJobIds )));
                for ii = 1:numel(allErrors)
                    err = err.addCause(allErrors(ii));
                end
                throw(err);
            end
        end
        
        %---------------------------------------------------------------
        % createJobOnScheduler
        %---------------------------------------------------------------
        % Create a new job on the scheduler, applying the job XML Description file
        % and Job Template, if appropriate
        function schedulerJob = createJobOnScheduler(obj)
            % Create a new job
            schedulerJob = obj.HPCSchedulerConnection.CreateJob();
            % If an XML file is defined, then restore the job from the XML.
            % DO THIS IMMEDIATELY AFTER CREATING THE JOB: RestoreFromXml OVERWRITES
            % ALL JOB PROPERTIES.
            if ~isempty(obj.JobDescriptionFile)
                try
                    schedulerJob.RestoreFromXml(obj.JobDescriptionFile);
                catch err
                    ex = MException(message('parallel:cluster:HPCServerFailedToCreateJobFromXML', obj.JobDescriptionFile));
                    ex = ex.addCause(err);
                    throw(ex);
                end
            end
            % Set the job template, if available.
            if ~isempty(obj.JobTemplate)
                try
                    schedulerJob.SetJobTemplate(obj.JobTemplate);
                catch err
                    ex = MException(message('parallel:cluster:HPCServerFailedToUseJobTemplate', obj.JobTemplate));
                    ex = ex.addCause(err);
                    throw(ex);
                end
            end
        end
    end
    
    methods  (Access = protected)
        %---------------------------------------------------------------
        % getJobByID
        %---------------------------------------------------------------
        function job = getJobByID(obj, jobID)
            try
                job = obj.HPCSchedulerConnection.OpenJob(jobID);
            catch err
                ex = MException(message('parallel:cluster:HPCServerUnableToRetrieveJob', jobID));
                ex = ex.addCause(err);
                throw(ex);
            end
        end
        
        %---------------------------------------------------------------
        % submitNonSOAJob
        %---------------------------------------------------------------
        function [schedulerJobIDs, schedulerTaskIDs, schedulerJobName] = submitNonSOAJob(obj, ...
                matlabExe, matlabArgs, ...
                jobEnvironmentVariables, taskLocationEnvironmentVariableName, ...
                taskLocations, taskLogLocations, ...
                username, password, numThreads)
            % Job Name is used only for SOA v2 jobs
            schedulerJobName = '';
            numTasks = numel(taskLocations);
            matlabCommand = sprintf('"%s" %s', matlabExe,  matlabArgs);
            
            hpcsJob = obj.createJobOnScheduler;
            % Don't set the min and max resource usage nor the IsExclusive property.
            % Defer to the values in the job template instead. If no job template
            % is explicitly specified, then the scheduler will use the default job template.
            
            % For each task add the task to the job and set the environment variables
            schedulerTaskIDs = zeros(numTasks, 1);
            for i = 1:numTasks
                % Create the task
                t = hpcsJob.CreateTask();

                % Set the number of cores needed to NumThreads
                t.MinimumNumberOfCores = numThreads;
                t.MaximumNumberOfCores = numThreads;
                
                % Now set the common environment variables
                for k = 1:size(jobEnvironmentVariables, 1)
                    t.SetEnvironmentVariable(jobEnvironmentVariables{k, 1}, jobEnvironmentVariables{k, 2});
                end
                % Additional task-specific environment variables
                t.SetEnvironmentVariable(taskLocationEnvironmentVariableName, taskLocations{i});
                
                % Redirect stdout and stderr to the log
                t.StdOutFilePath = taskLogLocations{i};
                t.StdErrFilePath = taskLogLocations{i};
                
                % Neither the job XML nor job template defines IsExclusive on a task, so set it here.
                % This ensures that multiple tasks from the same job can run on the same node.
                t.IsExclusive = obj.DoNotUseResourcesExclusively;
                t.CommandLine = matlabCommand;
                
                % Now add the task to the job
                hpcsJob.AddTask(t);
                % NOTE - HPC Server doesn't actually know the ID of it's task at this point so
                % we will assume it grows monotonically from 1
                schedulerTaskIDs(i) = i;
            end
            
            % The V2 API has a bug such that if the password is an zero length string,
            % the credentials will not be retrieved from the cache, and the user is
            % prompted to enter their password every single time.  Setting the password
            % to [] (i.e. a null string) means that the cached credentials are correctly
            % used.
            if strcmpi(password, '')
                password = [];
            end
            % Set the credentials for the job and submit the job
            obj.HPCSchedulerConnection.SubmitJob(hpcsJob, username, password);
            
            schedulerJobIDs = double(hpcsJob.Id);
        end
        
        %---------------------------------------------------------------
        % submitSOAJob
        %---------------------------------------------------------------
        function [schedulerJobIDs, schedulerTaskIDs,schedulerJobName] = submitSOAJob(obj, ...
                matlabExe, matlabArgs, ...
                jobEnvironmentVariables, ...
                jobID, jobLocation, taskIDs, taskLocations, taskLogLocations, ...
                username, password)

            assert(~isempty(obj.SOAConnection), ...
                'parallel:cluster:HPCServerNoSOAConnection', ...
                'Cannot submit SOA jobs because there is no SOA connection.');
            
            % Scheduler's task IDs are always empty for SOA jobs
            schedulerTaskIDs = [];
            
            try
                [schedulerJobIDs, schedulerJobName] = obj.SOAConnection.submitJob(...
                    jobID, jobLocation, taskIDs, taskLocations, taskLogLocations, obj.JobTemplate, ...
                    matlabExe, matlabArgs, jobEnvironmentVariables, username, password);
            catch err
                ex = MException(message('parallel:cluster:HPCServerSubmitSOAFailed'));
                ex = ex.addCause(err);
                throw(ex);
            end
       end
    end
end

