
%

%   Copyright 2013-2023 The MathWorks, Inc.

classdef ( Hidden ) MJSCloudCluster < parallel.Cluster

    % Profile/Construction properties
    properties ( Transient, PCTGetSet, GetAccess = public, SetAccess = private, PCTConstraint = 'string', Hidden )
        %Identifier Unique cluster identifier
        %   A unique string that identifies the cluster.
        Identifier='';
    end

    properties ( Transient, PCTGetSet, PCTConstraint = 'string' )

        %LicenseNumber License number to use when running jobs with this cluster
        %   License number specifies the license number to use on the
        %   workers when the cluster is using online licensing.
        %   This string must be the same as one of the licenses listed in
        %   your MathWorks account.
        %
        %   LicenseNumber cannot be defined if
        %   RequiresOnlineLicensing is false.
        %    
        %   See also
        %   parallel.cluster.MJSComputeCloud.RequiresOnlineLicensing
        LicenseNumber = '';

    end

    % Common public properties, declared Abstract as these might be PCTGetSet in
    % subclasses
    properties ( Abstract )
        
        %NumWorkersRequested Number of workers requested for this cluster
        %   NumWorkersRequested indicates the number of workers assigned to the
        %   cluster for running your jobs. This will specify the maximum
        %   value of NumWorkers.
        %
        %   See also parallel.cluster.MJSComputeCloud.NumWorkers
        NumWorkersRequested
        
    end
    
    properties
        
        %RequiresOnlineLicensing True if the cluster is using online licensing
        %   RequiresOnlineLicensing indicates whether the cluster uses
        %   online licensing.  You must provide your MathWorks account
        %   login information if this value is true.  If you have more than one
        %   MATLAB Parallel Server license associated with your
        %   MathWorks account you will also need to supply the LicenseNumber to
        %   use.  (read-only)
        %
        %   See also parallel.cluster.MJSComputeCloud.LicenseNumber.
        RequiresOnlineLicensing = true;
        
    end
    
    properties ( Dependent, SetAccess = private )
        
        %BusyWorkers Workers currently running tasks
        %   The BusyWorkers property value indicates which workers are
        %   currently running tasks for the cluster.
        %   (read-only)
        %
        %   As workers complete tasks and assume new ones, the lists of
        %   workers in BusyWorkers and IdleWorkers can change rapidly. If
        %   you examine these two properties at different times, you might
        %   see the same worker on both lists if that worker has changed
        %   its status between those times. If a worker stops unexpectedly,
        %   the cluster's knowledge of that as a busy or idle worker does
        %   not get updated until the cluster runs the next job and tries
        %   to send a task to that worker
        % 
        %   See also parallel.cluster.MJSComputeCloud.NumWorkers,
        %            parallel.cluster.MJSComputeCloud.IdleWorkers,
        %            parallel.cluster.MJSComputeCloud.NumBusyWorkers, 
        %            parallel.cluster.MJSComputeCloud.NumIdleWorkers, 
        %            parallel.job.MJSIndependentJob.NumWorkersRange,
        %            parallel.job.MJSCommunicatingJob.NumWorkersRange.
        BusyWorkers;
        
        %IdleWorkers Workers that are free to run tasks
        %   The IdleWorkers property value indicates which workers are
        %   currently not working on any tasks and are available to the
        %   cluster for the performance of job tasks.
        %   (read-only)
        %
        %   As workers complete tasks and assume new ones, the lists of
        %   workers in BusyWorkers and IdleWorkers can change rapidly. If
        %   you examine these two properties at different times, you might
        %   see the same worker on both lists if that worker has changed
        %   its status between those times. If a worker stops unexpectedly,
        %   the cluster's knowledge of that as a busy or idle worker does
        %   not get updated until the cluster runs the next job and tries
        %   to send a task to that worker.
        % 
        %   See also parallel.cluster.MJSComputeCloud.NumWorkers,
        %            parallel.cluster.MJSComputeCloud.BusyWorkers,
        %            parallel.cluster.MJSComputeCloud.NumBusyWorkers, 
        %            parallel.cluster.MJSComputeCloud.NumIdleWorkers, 
        %            parallel.job.MJSIndependentJob.NumWorkersRange,
        %            parallel.job.MJSCommunicatingJob.NumWorkersRange.
        IdleWorkers;
        
        %HasSecureCommunication True if the cluster is using secure communication
        %   The HasSecureCommunication property indicates whether secure
        %   communication is being used between the job manager and the
        %   workers. The mjs_def file sets the parameter that controls
        %   secure communication when the mjs process starts on the
        %   cluster nodes.  
        %   (read-only)
        %
        %   See also parallel.cluster.MJSComputeCloud.Username.
        HasSecureCommunication;
                
        %RequiresClientCertificate True if the cluster requires the client to have a certificate
        %   The RequiresClientCertificate property indicates whether the
        %   job manager requires clients to have a certificate for
        %   establishing secure communication. The mjs_def file sets
        %   the parameter that controls requiring client certificates
        %   when the mjs process starts on the cluster nodes.
        %
        %   Client certificates can only be used when also using
        %   secure communication.
        %   (read-only)
        %
        %   See also parallel.cluster.MJSComputeCloud.HasSecureCommunication
        RequiresClientCertificate;

        %Name Name of this cluster 
        %   The descriptive name of an MJSComputeCloud cluster is set when
        %   the cluster is created in Cloud Center.  If you have downloaded
        %   and imported your cluster's profile from Cloud Center then you
        %   will have a profile of the same name.  You can use this profile
        %   to create a cluster object with the parcluster function or the
        %   cluster constructor.
        %
        %    See also PARCLUSTER, 
        %             parallel.cluster.MJSComputeCloud.MJSComputeCloud.
        Name;
              
        %NumBusyWorkers Number of workers currently running tasks
        %   The NumBusyWorkers property value indicates how many workers
        %   are currently running tasks for the cluster.
        %   (read-only)
        %
        %   The value of NumBusyWorkers can range from 0 up to the total
        %   number of workers registered with the cluster.
        %
        %   See also parallel.cluster.MJSComputeCloud.NumWorkers,
        %            parallel.cluster.MJSComputeCloud.BusyWorkers,
        %            parallel.cluster.MJSComputeCloud.IdleWorkers, 
        %            parallel.cluster.MJSComputeCloud.NumIdleWorkers, 
        %            parallel.job.MJSIndependentJob.NumWorkersRange,
        %            parallel.job.MJSCommunicatingJob.NumWorkersRange.
        NumBusyWorkers;
        
        %NumIdleWorkers Number of workers that are free to run tasks
        %   The NumIdleWorkers property value indicates how many workers
        %   are currently not working on any tasks and are available to the
        %   cluster for the performance of job tasks. If the NumIdleWorkers
        %   is equal to or greater than the lower value of NumWorkersRange
        %   of the job at the front of the queue, that job can start
        %   running. (read-only)
        %
        %   The value of NumIdleWorkers can range from 0 up to the total
        %   number of workers registered with the cluster.
        %
        %   See also parallel.cluster.MJSComputeCloud.NumWorkers,
        %            parallel.cluster.MJSComputeCloud.BusyWorkers,
        %            parallel.cluster.MJSComputeCloud.IdleWorkers, 
        %            parallel.cluster.MJSComputeCloud.NumBusyWorkers, 
        %            parallel.job.MJSIndependentJob.NumWorkersRange,
        %            parallel.job.MJSCommunicatingJob.NumWorkersRange.
        NumIdleWorkers;
        
        %Username User who is using the cluster
        %   The Username property value indicates the user who created the
        %   cluster object or who is using the object to access jobs in its
        %   queue.
        %
        %   See also parallel.cluster.MJSComputeCloud.HasSecureCommunication
        Username;
        
        %State Current state of the cluster
        %   A cluster can be in state offline, starting, waitingforworkers, online,
        %   stopping, deleted, communicationerror, or error.
        %   (read-only)
        State;

        %ShutdownAt Shutdown time or event
        %   The event or time at which the cluster will shut down.
        ShutdownAt;

    end

    properties ( Dependent )
                
        %ClusterMatlabRoot Specifies the path to MATLAB that the cluster is using
        %   ClusterMatlabRoot specifies the path name to MATLAB for the
        %   cluster to use for starting MATLAB worker processes.
        %   (read-only)
        %
        %   See also parallel.cluster.MJSComputeCloud.JobStorageLocation
        ClusterMatlabRoot;
        
        %JobStorageLocation Location where job and task data is stored
        %   (read-only)
        %   
        %   See also parallel.cluster.MJSComputeCloud.ClusterMatlabRoot
        JobStorageLocation;

        %OperatingSystem Operating system of the nodes used by the cluster
        %   OperatingSystem is always unix.
        %   (read-only)
        OperatingSystem;
       
    end

    properties ( Dependent, SetAccess = immutable )
    
        %Host Host on which this cluster is running
        %
        %   See also PARCLUSTER,
        %            parallel.cluster.MJSComputeCloud.Name.
        Host;

    end
    
    properties ( Transient, Access = protected )

        % The Support providing access to underlying functionality
        CloudSupport = parallel.internal.cloud.CloudSupport.empty;
        
        % Local cache of cluster state
        ClusterState = parallel.internal.types.MJSComputeCloudStates.Stopped;

    end

    properties ( Transient, GetAccess = protected, SetAccess = private )
        
        % Error message to show as part of cluster display (if necessary)
        StashedErrorMessage = '';
        
    end
    
    properties (Dependent, Hidden)
        
        PromptForPassword;
        
    end

    properties (Transient, Hidden)

        % Switch to control use of PeerLookup or RMI 
        UsePeerLookup = true;
        
        ClusterLogLevel = 2;
        
        ClusterLogSpec;

    end

    properties ( Dependent, Hidden )
        WorkerProxiesPoolConnections;
        RequiresMathWorksHostedLicensing;
    end
    
    % API2(orig) compatibility
    properties ( Dependent, Hidden )
        IsUsingSecureCommunication
    end

    % Testing override
    properties ( Hidden )
        WaitDelay = 10;
        WorkerWaitTimeout = 60*10; % seconds
        StateCheckPeriod = 5; % seconds
    end
    
    properties ( Constant, GetAccess = private )
        %FullMementoProperties A string array that defines the properties
        % needed, in addition to those listed in MinimalMementoProperties,
        % for a full cluster memento (i.e. one that allows a cluster object
        % to be constructed that can find the original cluster's jobs). A
        % property should only be included if it is required in the memento
        % for this class and every sub-class.
        %
        % A sub-class can define additional properties to be included in
        % the memento in a constant, private property of the same name. A
        % sub-class should not include properties that are included by any
        % of its super-classes.  A super-class should only define a
        % property if it should be included in the memento of each of its
        % sub-classes.
        %
        % The full list of properties required by any class can be gathered
        % using hDetermineMementoProperties.
        FullMementoProperties = "LicenseNumber";
    end

    methods
        function v = get.IsUsingSecureCommunication( obj )
            v = obj.HasSecureCommunication;
        end
        function v = get.RequiresMathWorksHostedLicensing( obj )
            v = obj.RequiresOnlineLicensing;
        end
        function set.RequiresMathWorksHostedLicensing( obj, ~ )
            throwAsCaller( obj.buildCannotModify( 'RequiresMathWorksHostedLicensing' ) );
        end
    end
    
% Getters and Setters
% -------------------------------------------------------------------------
    methods

        function currentState = get.State( obj )
        % Query the Cloud Center for the current state of this cluster
            obj.refreshState();
            currentState = obj.ClusterState.Name;
        end

        function shutdownAt = get.ShutdownAt( obj )
            obj.refreshState();
            shutdownAt = parallel.cluster.MJSCloudCluster.hFormatTerminationPolicy( ...
                obj.getClusterDetails() );
        end

        function val = get.ClusterMatlabRoot( obj )
            try
                val = obj.getClusterProps( 'ClusterMatlabRoot' );
            catch E %#ok<NASGU>
                val = '';
            end
        end
        function set.ClusterMatlabRoot( obj, ~ )
            throwAsCaller( obj.buildCannotModify( 'ClusterMatlabRoot' ) );
        end


        function val = get.Host( obj )
            try
                val = obj.getClusterProps( 'Host' );
            catch E %#ok<NASGU>
                clusterDetails = obj.getClusterDetails();
                val = clusterDetails.ClusterHeadNodeDNSName;
            end
        end

        function set.RequiresOnlineLicensing(obj, ~)
            throwAsCaller( obj.buildCannotModify( 'RequiresOnlineLicensing' ) );
        end

        function val = get.NumBusyWorkers( obj )
            try
                val = obj.getClusterProps( 'NumBusyWorkers' );
            catch E %#ok<NASGU>
                val = 0;
            end
        end

        function val = get.NumIdleWorkers( obj )
            try
                val = obj.getClusterProps( 'NumIdleWorkers' );
            catch E %#ok<NASGU>
                val = 0;
            end
        end

        % The MJSSupport.getWorkers() method does not handleJavaException,
        % so we must.
        function bw = get.BusyWorkers( obj )

            try
                bw = obj.CloudSupport.getWorkers( obj, 'BusyWorkers' );
            catch E
                throw( distcomp.handleJavaException( obj, E ) );
            end
        end

        function iw = get.IdleWorkers( obj )

            try
                iw = obj.CloudSupport.getWorkers( obj, 'IdleWorkers' );
            catch E
                throw( distcomp.handleJavaException( obj, E ) );
            end
        end

        function val = get.Name( obj )
            clusterDetails = obj.getClusterDetails();
            val = clusterDetails.ClusterName;
        end

        function val = get.HasSecureCommunication( obj )
            try
                val = obj.getClusterProps( 'HasSecureCommunication' );
            catch E %#ok<NASGU>
                val = [];
            end
        end

        function val = get.RequiresClientCertificate( obj )
            try
                val = obj.getClusterProps( 'RequiresClientCertificate' );
            catch E %#ok<NASGU>
                val = [];
            end
        end

        function val = get.WorkerProxiesPoolConnections( obj )
            try
                val = obj.getClusterProps( 'WorkerProxiesPoolConnections' );
            catch E %#ok<NASGU>
                val = [];
            end
        end

        function val = get.JobStorageLocation( obj )
            try
                propsToGet = {'Name','Host'};
                propVals = obj.getClusterProps( propsToGet );

                cloudConsoleJobManagerName = propVals{1};
                host = propVals{2};
                val = parallel.cluster.MJSCloudCluster.hFormatJobStorageLocation( ...
                        cloudConsoleJobManagerName, host );
            catch E %#ok<NASGU>
                val = '';
            end
        end
        function set.JobStorageLocation( obj, ~ )
            throwAsCaller( obj.buildCannotModify( 'JobStorageLocation' ) );
        end

        function val = get.OperatingSystem( obj )
            try
                val = obj.getClusterProps( 'ClusterOperatingSystem' );
            catch E %#ok<NASGU>
                val = '';
            end
        end
        function set.OperatingSystem( obj, ~ )
            throwAsCaller( obj.buildCannotModify( 'OperatingSystem' ) );
        end

        function username = get.Username( ~ )
            currentUser = parallel.internal.mwa.ensureLoggedIn();
            username = currentUser.getUsername();
        end
        
        function prompt = get.PromptForPassword( obj )
            if obj.isConnected()
                prompt = obj.getClusterProps( 'PromptForPassword' );
            else
                prompt = true;
            end
        end
        function set.PromptForPassword( obj, promptForPassword )
            try
                parallel.internal.customattr.checkConstraint( 'logicalscalar', obj, 'PromptForPassword', promptForPassword );
                obj.setClusterProps( 'PromptForPassword', promptForPassword );
            catch E
                throwAsCaller( distcomp.handleJavaException( obj, E ) );
            end
        end
        
        function v = get.ClusterLogLevel( obj )
            if obj.isConnected()
                v = obj.getClusterProps( 'ClusterLogLevel' );
            else
                v = 2;
            end
        end
        function set.ClusterLogLevel( obj, logLevel )
            try
                validateattributes( logLevel, {'numeric'}, {'scalar', '>=', 0, '<=', 6}, 'setClusterLogLevel' );
                obj.setClusterProps( 'ClusterLogLevel', logLevel );
            catch E
                throwAsCaller( distcomp.handleJavaException( obj, E ) );
            end
        end
        
        function v = get.ClusterLogSpec( obj )
            if obj.isConnected()
                v = obj.getClusterProps( 'ClusterLogSpec' );
            else
                v = '';
            end
        end
        function set.ClusterLogSpec( obj, logSpec )
            logSpec = convertStringsToChars( logSpec );
            try
                validateattributes( logSpec, {'char'}, {'scalartext'}, 'setClusterLogSpec' );
                obj.setClusterProps( 'ClusterLogSpec', logSpec );
            catch E
                throwAsCaller( distcomp.handleJavaException( obj, E ) );
            end
        end
    end


% Overrides
% -------------------------------------------------------------------------
    methods ( Access = protected )

        function jobs = getJobs( obj )
        % Get jobs. Note that this method really shouldn't throw or display
        % errors if the cluster isn't started as it is used for the summary
        % output.
        
            try
                jobs = obj.CloudSupport.getJobs( obj );
            catch E
                throwAsCaller( distcomp.handleJavaException( obj, E ) );
            end
        end

        function varargout = getJobsByState( obj )

            try
                [varargout{1:nargout}] = obj.CloudSupport.getJobsByState( obj );
            catch E
                throwAsCaller( distcomp.handleJavaException( obj, E ) );
            end
        end

        function job = buildIndependentJob( obj )

            import parallel.internal.types.Variant

            % All independent jobs are not interactive.
            executionMode = parallel.internal.types.ExecutionMode.Batch;
            job = obj.buildGenericJob(Variant.IndependentJob, executionMode);
        end

        function job = buildCommunicatingJob( obj, variant, executionMode )

            job = obj.buildGenericJob(variant, executionMode);

        end

        function [connMgr, strategy] = buildConnectionManager( obj )
            
            try
                [connMgr, strategy] = obj.CloudSupport.buildConnectionManager();
            catch E
                throw( distcomp.handleJavaException( obj, E ) );
            end
        end

        function tf = isEquivalentWorkers( obj1, obj2 )
        % isEquivalentWorkers TRUE if Host and Name match.
            tf = isequal( obj1.Host, obj2.Host ) && ...
                 isequal( obj1.Name, obj2.Name );
        end

        function tf = isEquivalentStorage( obj1, obj2 )
        % isEquivalentStorage TRUE if Host and Name match.
            tf = isequal( obj1.Host, obj2.Host ) && ...
                 isequal( obj1.Name, obj2.Name );
        end

        function num = getNumConcurrentTasks( obj, job )
            try
                num = job.hGetProperty( 'NumLabs' );
                while num == 0 % jobmanager hasn't filled in the details
                    pause(0.5);
                    num = job.hGetProperty( 'NumLabs' );
                end
            catch E
                throw( distcomp.handleJavaException( obj, E ) );
            end
        end
        
        function onInvalidatingClusters( obj )
            % If the cluster is using online licensing at the
            % time of logout, invalidate the cluster object and all of its
            % children (jobs/tasks/workers).
            if obj.RequiresOnlineLicensing
                obj.CloudSupport.invalidateAllChildren();
                obj.delete();
            end
        end
    end

% Protected helpers
% -----------------------------------------------------------

    methods ( Access = protected )

        function connected = isConnected( obj )
            connected = obj.CloudSupport.isConnected();
        end

        function clusterDetails = getClusterDetails( obj )
            % Some properties don't change so we don't want to make a call
            % to webservices in those cases. However, if we don't have any
            % cached cluster details yet, we must make the first call to
            % get them.
            if isempty( obj.CloudSupport.ClusterDetails )
                obj.refreshState()
            end
            clusterDetails = obj.CloudSupport.ClusterDetails;
        end

        function newjob = buildChild( obj, variant, jobId, jobSId )
            newjob = obj.CloudSupport.buildChild( obj, variant, jobId, jobSId );
        end

        function v = getClusterProps( obj, names )
        % Simple wrapper to include handleJavaException
            try
                v = obj.CloudSupport.getClusterProperties( names );
            catch E
                throwAsCaller( distcomp.handleJavaException( obj, E ) );
            end
        end

        function setClusterProps( obj, names, values )
            try
                obj.CloudSupport.setClusterProperties( names, values );
            catch E
                throwAsCaller( distcomp.handleJavaException( obj, E ) );
            end
        end

        function refreshState( obj )
            try
                obj.ClusterState = obj.CloudSupport.refreshState();
            catch E
                obj.StashedErrorMessage = E.message;
                obj.ClusterState = parallel.internal.types.MJSComputeCloudStates.Error;
            end
       
        end

        function [values, handled] = preVectorisedGet( obj, ~, handled, values )
            obj.refreshState();
        end      
        
        [values, handled] = doVectorisedGet( obj, propsToAccess, handled, existingValues );

    end

% Private helpers
% -----------------------------------------------------------

    methods ( Access = private )
        
        job = buildGenericJob( obj, variant, executionMode );

        function shutdownOnEvent( obj, eventEnum )
            parallel.internal.cloud.shutdownCloudClusterOnEvent( obj.Identifier, eventEnum );
        end

        function shutdownAtDateNum( obj, dateNumber )
            obj.shutdownAtDateTime( datetime( dateNumber, 'ConvertFrom', 'datenum', 'TimeZone', 'local' ) );
        end

        function shutdownAtDateVec( obj, dateVector )
            obj.shutdownAtDateTime( datetime( dateVector, 'TimeZone', 'local' ) );
        end

        function shutdownAtDateTime( obj, shutdownTime )
            % Ensure that the given datetime has a timezone. If none is
            % specified, use the local one.
            if isempty( shutdownTime.TimeZone )
                shutdownTime.TimeZone = 'local';
            end
            
            % Calculate the duration in seconds between now and the requested shutdown time.
            numSeconds = round( seconds( shutdownTime - datetime( 'now', 'TimeZone', 'local' ) ) );
            if numSeconds == 0
                obj.stop();
            else
                obj.shutdownAfter( numSeconds );
            end
        end

        function shutdownAfter( obj, numSeconds )
            if numSeconds < 0
                error( message( 'parallel:cluster:CloudInvalidShutdownAt' ) );
            end
            
            % Cloud center places a limit of java.lang.Integer.MAX_VALUE on the cluster
            % duration.
            if numSeconds > java.lang.Integer.MAX_VALUE
                error( message( 'parallel:cluster:CloudInvalidShutdownAtFuture' ) );
            end

            % The cloud center client takes a duration in seconds
            parallel.internal.cloud.shutdownCloudClusterAfter( obj.Identifier, numSeconds );
        end
        
    end

% Public documented methods
% -------------------------------------------------------------------------
    methods ( Access = public )

        function start( obj )
            %start Start this cluster
            %   start(cluster) starts the specified MJS cluster in the cloud if
            %   it is not already running. If the cluster is already running or
            %   in the process of shutting down, then start(cluster) returns
            %   immediately and the state of the cluster is not changed.
            %
            %   Example:
            %   % Create a cloud cluster using the default profile
            %   myCluster = parcluster;
            %   % Start the cluster
            %   start(myCluster);
            %   % Wait for it to be ready to accept job submissions
            %   wait(myCluster);
            %
            %   See also parallel.cluster.MJSCloudCluster/shutdown
            %            parallel.cluster.MJSCloudCluster/wait
            obj.ensureLicenseNumberIsSetOrError();

            % In case of an authorization failure we want to try again.
            obj.startCloudCluster();
        end

        function shutdown( obj, varargin )
            %shutdown Shut down this cluster
            %   shutdown(cluster) shuts down the cluster immediately.
            %
            %   shutdown(cluster, 'At', D) shuts down the cluster at the
            %   time specified by the datenum, datevec or datetime D.
            %   D is interpreted in the local time zone unless D is a
            %   datetime with TimeZone specified.
            %
            %   shutdown(cluster, 'After', E) shuts down the cluster after the
            %   specified event E has occurred. E can be 'never' or 'idle'. A
            %   cluster is considered to be idle if there are no currently
            %   running jobs, no queued jobs, and no new jobs or tasks have
            %   been created or modified for 15 minutes.
            %
            %   shutdown(cluster, 'After', N) shuts down the cluster after N
            %   hours measured from the time this method is called.
            %
            %   Examples:
            %
            %   % Create a cloud cluster using the default profile
            %   cloudCluster = parcluster;
            %
            %   % Set the cluster to shut down 1 hour from now.
            %   shutdown(cloudCluster, 'After', 1);
            %
            %   % Set the cluster to shut down at a specified time.
            %   shutdown(cloudCluster, 'At', datenum('2013-02-22 19:00'));
            %
            %   % Set the cluster to shut down at a specified datetime.
            %   % The datetime can (optionally) have a TimeZone specified.
            %   shutdown(cloudCluster, 'At', datetime(2013, 2, 22, 19, 0, 0, 'TimeZone', 'local'));
            %
            %   % Allow the cluster to run indefinitely.
            %   shutdown(cloudCluster, 'After', 'never');
            %
            %   % Shut down the cluster when it is idle
            %   shutdown(cloudCluster, 'After', 'idle')
            %
            %   See also datenum, datevec, datetime, parallel.cluster.MJSComputeCloud/shutdownAt
            
            if nargin > 1
                modifier = varargin{1};
                argument = varargin{2};
                validModifiers = { 'At', 'After' };
                validatestring( modifier, validModifiers, 'shutdown' );
                if strcmpi( modifier, 'After' )
                    if ischar( argument )
                        validPolicies = iGetValidTerminationPolicies( obj );
                        allValidPolicyNames = {validPolicies.Name};
                        validPoliciesStr = [sprintf('''%s'', ', allValidPolicyNames{1:end-1}) ...
                                                sprintf('''%s''', allValidPolicyNames{end})];
                        try
                            terminationPolicy = parallel.internal.types.TerminationPolicy.fromName( argument );
                        catch E
                            if strcmp( E.identifier, 'parallel:cluster:CloudInvalidTerminationPolicy' )
                                error( message( 'parallel:cloud:InvalidShutdownArgument', argument, ...
                                    validPoliciesStr ) );
                            else
                                rethrow( E );
                            end
                        end
                        if ~ismember( terminationPolicy, validPolicies )
                            error( message( 'parallel:cloud:InvalidShutdownArgument', terminationPolicy.Name, ...
                                validPoliciesStr ) );
                        end
                        obj.shutdownOnEvent( terminationPolicy );
                    else
                        % The 'after' modifier specifies the number of hours from now to shut down the
                        % cluster. It must be an integer value.
                        % First validate that the argument is numeric, then check for Inf.
                        validateattributes( argument, {'numeric'}, {}, 2 );
                        if isinf( argument )
                            obj.shutdownOnEvent( parallel.internal.types.TerminationPolicy.Never );
                        else
                            validateattributes( argument, {'numeric'}, {'integer', 'scalar'}, 2 );
                            numSeconds = argument*60*60;
                            obj.shutdownAfter( numSeconds );
                        end
                    end
                elseif strcmpi( modifier, 'At' )
                    % The 'at' modifier specifies a time at which the cluster should be shut down.
                    % This should either be a MATLAB date number or string or a MATLAB datetime
                    if isdatetime( argument )
                        validateattributes( argument, {'datetime'}, {'scalar'}, 'shutdown', 'time', 2 );
                        obj.shutdownAtDateTime( argument );
                    else
                        validateattributes( argument, {'double'}, {'nonnan'}, 'shutdown', 'time', 2 );
                    
                        if isinf( argument )
                            obj.shutdownOnEvent( parallel.internal.types.TerminationPolicy.Never );
                            return
                        end
                        if isscalar( argument )
                            obj.shutdownAtDateNum( argument );
                        else
                            obj.shutdownAtDateVec( argument );
                        end
                    end
                end
            else
                obj.stop();
            end
        end

        function varargout = wait( obj, requestedState, timeout )
            %wait Wait for this cluster to change state
            %   wait(C) blocks execution in the client MATLAB session until the
            %   single cluster identified by the scalar cluster object C reaches
            %   the 'online' state. This occurs when the cluster is ready to
            %   accept job submissions and all workers have registered.
            %
            %   wait(C, STATE) blocks execution in the client session until the
            %   cluster object C changes state to STATE. For a cluster object the
            %   valid states are 'online', 'waitingforworkers', and 'offline'.
            %
            %   OK = wait(C, STATE, TIMEOUT) blocks execution in the client
            %   session until the cluster object C changes state to STATE or
            %   until TIMEOUT seconds elapsed, whichever happens first. OK is
            %   TRUE if STATE has been reached or a terminal state such as
            %   'error', or FALSE in case of a timeout.
            %
            %   Example:
            %   % Create a cloud cluster using the default profile
            %   cloudCluster = parcluster;
            %   start(cloudCluster);
            %
            %   % Wait until the head node has started and the cluster is 
            %   ready to accept job submissions. For multiple node clusters, 
            %   workers on nodes other than the head node may not have started yet. 
            %   wait(cloudCluster, 'waitingforworkers');
            %
            %   % Wait until the head node and all workers have started and
            %   the cluster is ready to accept job submissions.
            %   wait(cloudCluster, 'online');
            %
            %   See also parallel.cluster.MJSComputeCloud/start,
            %            parallel.cluster.MJSComputeCloud/shutdown.
            if nargin < 2
                requestedState = parallel.internal.types.MJSComputeCloudStates.Ready;
            else
                waitableStates = parallel.internal.types.MJSComputeCloudStates.ClusterWaitableStates;
                allWaitableStateNames = {waitableStates.Name};
                allWaitableStatesStr = [sprintf('''%s'', ', allWaitableStateNames{1:end-1}) ...
                                        sprintf('''%s''', allWaitableStateNames{end})];
                try
                    requestedState = parallel.internal.types.MJSComputeCloudStates.fromName( requestedState );
                catch E
                    if strcmp( E.identifier, 'parallel:cluster:MJSComputeCloud:InvalidStateName' )
                        error( message( 'parallel:cloud:InvalidWaitState', requestedState, ...
                            allWaitableStatesStr ) );
                    else
                        ex = MException( message( 'parallel:cloud:InvalidWaitArgument', allWaitableStatesStr ) );
                        ex = ex.addCause( E );
                        throw( ex );
                    end
                end
                if ~requestedState.isStateClusterWaitable()
                    error( message( 'parallel:cloud:InvalidWaitState', requestedState.Name, ...
                        allWaitableStatesStr ) );
                end
            end
            if nargin < 3
                timeout = Inf;
            else
                validateattributes( timeout, {'numeric'}, {'integer', 'scalar', 'positive'}, 2 );
            end  

            % There can be 0 or 1 output arguments.
            nargoutchk(0, 1);
            
            ok = obj.waitFor( requestedState, timeout );
            if nargout > 0
                varargout{1} = ok;
            end
        end
    end

    methods ( Access = private )

        function ok = waitFor( obj, requestedState, timeout )
            waiter = parallel.internal.cloud.CloudClusterStateWaiter( obj.WaitDelay, requestedState, obj );
            ok = waiter.wait( timeout );
        end

        function stop( obj )
            if obj.hIsClusterStoppable()
                parallel.cluster.MJSCloudCluster.hStopNoCheck( obj.Identifier );
            end
        end

    end

% Public hidden methods
% -------------------------------------------------------------------------
    methods ( Access = public, Hidden )

        function pctVersionNumber = hGetPCTVersionNumber( obj )
            pctVersionNumber = obj.CloudSupport.getVersionNumber();
        end
        
        function tf = hIsClusterAvailable( obj )
            obj.refreshState();
            tf = obj.ClusterState.isStateClusterAvailable();
        end

        function tf = hIsClusterStoppable( obj )
            obj.refreshState();
            tf = obj.ClusterState.isStateClusterStoppable();
        end
        
        function hSetProperty( obj, names, values )
        % Only for PCTGetSet properties. Add any non-default behaviour here
            if ~iscell( names )
                names  = { names };
                values = { values };
            end
            licenseNumberProperty = {'LicenseNumber'};
            for ii = 1:length( names )
                if ismember( names{ii}, licenseNumberProperty ) && ...
                        ~strcmp( values{ii}, obj.LicenseNumber )
                    % If the license number is different to the currently
                    % stored number then check its validity before setting
                    % and make sure we stash the corresponding entitlement.
                    try
                        obj.LicenseEntitlement = obj.checkLicenseNumber( values{ii} );
                    catch E
                        throwAsCaller( E );
                    end
                end
                obj.hSetPropertyNoCheck( names{ii}, values{ii} );
            end
        end

        function v = hGetClusterProps( obj, names )
        % Get properties directly from the underlying cluster
            v = obj.getClusterProps( names );
        end
        
        function num = hGetAvailableNumWorkers( obj )
            clusterDetails = obj.getClusterDetails();
            num = clusterDetails.NumberOfAvailableWorkers;
        end

        function num = hGetTotalNumWorkers( obj )
            clusterDetails = obj.getClusterDetails();
            num = clusterDetails.NumberOfWorkers;
        end

        function state = hGetCloudConsoleState( obj )
            clusterDetails = obj.getClusterDetails();
            state = clusterDetails.ClusterState;
            state = parallel.internal.types.CloudCenterClusterState.fromName(state);
        end

        function tf = hIsResizable( obj )
            clusterDetails = obj.getClusterDetails();
            tf = clusterDetails.IsResizable;
        end

        function hRefreshState( obj )
            obj.refreshState();
        end

        function getClusterLogs( obj, saveLocation )
        % Get the MJS logs from the job manager and save as a zip file
        % in the saveLocation.
            
            if nargin < 2
                saveLocation = '.';
            end
            filename = [obj.Name '-logs.zip'];
            filename = fullfile( saveLocation, filename );

            try
                obj.CloudSupport.getClusterLogs( filename );
            catch E
                error(message('parallel:cluster:MJSComputeCloud:FailedToGetLogs',...
                      obj.Name, E.getReport));
            end
        end
        
        function [p, q, r, f] = hEnumerateJobsInStates(obj)
            
            % This function is only used in display, so we don't want to
            % error. The State field will be set to 'communicationerror'
            try
                [p, q, r, f] = obj.CloudSupport.enumerateJobsInStates();
            catch E %#ok<NASGU>
                [p, q, r, f] = deal(0,0,0,0);
                return;
            end
        end

        function clusterState = hGetCachedClusterState( obj )
        % In display we don't want to make multiple remote calls if we
        % can help it. This helper allows display to get the cached
        % state, if it _knows_ that the state has previously been refreshed.
            clusterState = obj.ClusterState;
        end

        function host = hGetCachedHost( obj )
            % Get the cluster host from cached cluster details
            host = obj.CloudSupport.ClusterDetails.ClusterHeadNodeDNSName;
        end

        function shutdownTime = hGetShutdownTime( obj )
            obj.refreshState(); % Needed to update cluster details
            clusterDetails = obj.getClusterDetails();
            shutdownTime = parallel.cluster.MJSCloudCluster.hGetShutdownTimeFromDetails( clusterDetails );
        end
        
        function hSetClusterID( obj, identifier )
            % Cloud clusters only get this information from cloud center
            % when we connect to the MJS for the first time, so we need a
            % way to set this information on the cluster.
            obj.Identifier = identifier;
            obj.CloudSupport.hSetClusterID( identifier );
        end

        function hSetClusterDetails( obj, clusterDetails )
            % The cluster details may be updated during display if we
            % connect to an online cluster and reset its termination
            % policy for example.
            obj.CloudSupport.hSetClusterDetails( clusterDetails );
        end

        function provider = hCreateJobStatusProvider( obj )
        % hCreateJobStatusProvider - return an update provider for Job Monitor
        % to pull updates.
            provider = obj.CloudSupport.hCreateJobStatusProvider( obj );
        end

        function factory = hCreatePoolEndpointFactory( obj )
            factory = obj.CloudSupport.createEndpointFactory(class(obj));
        end

        function [taskIDs, taskSIds] = hBuildTasksVec( obj, varargin )
            [taskIDs, taskSIds] = obj.CloudSupport.buildTasksVec( varargin{:} );
        end
        
        function support = hGetSupport( obj )
            % Return the underlying Support. For use by tests.
            support = obj.CloudSupport.hGetUnderlyingSupport();
        end

        function [isNetworkCluster, latency, dataSend, dataReceive] = hTestClusterConnection( obj )
            [isNetworkCluster, latency, dataSend, dataReceive] = parallel.cluster.MJS.hDoTestClusterConnection( obj );
        end
    end

    methods ( Access = public, Hidden, Static )

        function jobStorageLocation = hFormatJobStorageLocation( name, host )
            dataLocForDisplay = strcat( name, '@', host );
            m = message( 'parallel:cluster:MJSJobStorageLocation', dataLocForDisplay );
            jobStorageLocation = m.getString();
        end

        function policyStr = hFormatTerminationPolicy( clusterDetails )

            terminationPolicy = clusterDetails.TerminationPolicy;

            % Convert the termination policy enum to the corresponding MATLAB enum
            mlPolicy = parallel.internal.types.TerminationPolicy.fromCloudCenterName( terminationPolicy );

            % If the policy is a fixed duration, we want to display the time that the
            % cluster is set to expire. So we get this from the cluster details and
            % convert it to a MATLAB format.
            if mlPolicy == parallel.internal.types.TerminationPolicy.Fixed
                % If the cluster is shutting down, mlDateTime will be
                % datetime.empty() and policyStr will be ''.
                mlDateTime = parallel.cluster.MJSCloudCluster.hGetShutdownTimeFromDetails( clusterDetails );
                policyStr = char( mlDateTime );
            else
                policyStr = mlPolicy.Name;
            end
        end

        function shutdownTime = hGetShutdownTimeFromDetails( clusterDetails )
            expiresOn = clusterDetails.ExpiresOn;
            
            % Convert from uint64 seconds since epoch to MATLAB datetime. If expiresOn is empty then
            % the shutdownTime is expected to have a 0x0 datetime object.
            if isempty(expiresOn)
                shutdownTime = datetime.empty();
            else
                shutdownTime = datetime(expiresOn,...
                                        'ConvertFrom', 'posixtime', 'TimeZone', 'local');
            end
        end

        function hStopNoCheck( clusterID )
        % Stop this cluster without checking its current state.
        % We do check the validity of the clusterID, to prevent unhelpful errors
        % from Cloud Center.
            if isempty( clusterID )
                % If the cluster has never been brought online the clusterID will be
                % empty. Do nothing in that case.
                return
            end
            try
                % To make the shutdown more graceful, we check to see if there is a pool
                % running on this cluster and shut down that first if there is. See g1242648.
                parallel.cluster.MJSCloudCluster.hDeletePoolIfRunning( clusterID );
                parallel.internal.cloud.stopCloudCluster( clusterID );
            catch E
                % It's possible that the cluster might have been deleted already by
                % some other means (perhaps on another machine). In that case, cloud
                % center will throw an error about not being able to find the cluster.
                % We can safely ignore that error, but we should rethrow any other
                % error.
                if ~strcmp( E.identifier, 'parallel:cluster:cloudcenterclient:ClusterNotFound' )
                    throw( E )
                end
            end
        end

        function hDeletePoolIfRunning(identifier)
            pool = gcp('nocreate');
            if ~isempty(pool) && ... % if there's a pool
                    isa(pool.Cluster, 'parallel.cluster.MJSCloudCluster') && ... % on a cloud cluster
                    strcmp(pool.Cluster.Identifier, identifier) % and it's _this_ cloud cluster
                delete(pool);
            end
        end

    end
    
    methods ( Access = private )
        function delete( obj )            
            delete@handle( obj );
        end
        
    end

    methods ( Access = public )
        function obj = MJSCloudCluster( clusterType, cProf )
            obj@parallel.Cluster( clusterType, cProf );
            
            % Listen for MATLAB logout events since an MJSCloudCluster
            % always uses online licensing
            obj.listenForMatlabLogout();
        end      
    end
    
    methods ( Static, Hidden )
        function [host, name] = hGetHostAndName( clusterDetails, usePeerLookup )
            % hGetHostAndName retrieves the host and name properties of a
            % cluster using the provided clusterDetailsFunc
            if usePeerLookup
                host = clusterDetails.PeerLookupUri;
            else
                host = clusterDetails.ClusterHeadNodeDNSName;
            end
            name = clusterDetails.JobManagerName;
        end

        function hStopCloudNoCheck(identifier)
            % Shuts down the cloud cluster specified by the cluster identifier.
            parallel.cluster.MJSCloudCluster.hStopNoCheck(identifier);
            % Now the cluster is offline we notify listeners
            cloudStateNotifier = parallel.internal.cloud.CloudStateNotifier.getInstance();
            cloudStateNotifier.notifyOffline();
        end

        function hErrorIfClusterShuttingDown(clusterId)
            % Verifies whether the cluster is marked for shut down.

            if iClustersShuttingDownOperation('exists', clusterId)
                iClustersShuttingDownOperation('remove', clusterId); % first, remove the cluster from the list

                dctSchedulerMessage(6, 'Cluster "%s" was stopped by the user.', clusterId);

                % if the cluster is marked for termination, abort the current operation
                throw(MException(message('parallel:cloud:ForcefullyStopped')));
            end
        end
    end
    
    methods ( Abstract, Access = protected )
        startCloudCluster( obj, token );
    end
end
%#ok<*ATUNK> custom attributes

function result = iClustersShuttingDownOperation(operation, item)
persistent clustersShuttingDown;
if isempty(clustersShuttingDown)
    clustersShuttingDown = {};
end

result = [];

switch operation
    case 'add'
        clustersShuttingDown{end + 1} = item;
    case 'remove'
        clustersShuttingDown(strcmp(clustersShuttingDown, item)) = [];
    case 'exists'
        result = any(strcmp(clustersShuttingDown, item));
end
end

function policies = iGetValidTerminationPolicies( ~ )
policies = [parallel.internal.types.TerminationPolicy.Idle, ...
    parallel.internal.types.TerminationPolicy.Never];
end
