% JobStateHelper Help CJS schedulers calculate job state when the underlying
% scheduler cannot help.

% Copyright 2011-2016 The MathWorks, Inc.

classdef ( Hidden, Sealed ) JobStateHelper

    properties ( Constant, GetAccess = private )
        % We cache the times relating to calls to hGetStateFromTasks. We don't
        % cache the result as it's irrelevant. The cache values are in the form:
        % { firstCheckTime, latestCheckTime }
        % where the time values are obtained using TIC.
        TaskCheckTimeCache = containers.Map();

        % Call hGetStateFromTasks at most as frequently as this:
        TaskCheckIntervalSeconds = 60;

        % If hGetStateFromTasks doesn't return a terminal state within this time
        % from our first check, we'll deem the job to be 'failed'.
        AssumeFailedTimeoutSeconds = 120;
    end

    methods ( Static, Access = private )
        function jobState = getJobStateForUnknownJob( job, jobState, uuid, checkTimeCache )
        % getJobStateForUnknownJob - get here for a job which we have never been
        % asked about before. In this case, we must get the state from the
        % tasks, and seed the cache if necessary.

            stateFromTasks = job.hGetStateFromTasks();
            if stateFromTasks.isTerminal()
                % Great, let the job become terminal, don't need to put anything
                % in cache since we're not going to be asked about this job
                % again.
                jobState = stateFromTasks;
            else
                % Not terminal, so cache the information and timestamps and
                % leave things for now.
                currentTime = tic;
                cacheInfo = { currentTime, currentTime };
                checkTimeCache( uuid ) = cacheInfo; %#ok<NASGU> handle
            end
        end

        function jobState = getJobStateForKnownJob( job, jobState, uuid, checkTimeCache )
        % getJobStateForKnownJob - get here for a job which has already been
        % queried at least once before. Only re-check hGetStateFromTasks at most
        % once per TaskCheckIntervalSeconds. If more than
        % AssumeFailedTimeoutSeconds has elapsed since the first time we checked
        % and the job still doesn't appear to be Finished, then we're going to
        % force it to be Failed.
            import parallel.internal.cluster.JobStateHelper
            import parallel.internal.types.States

            currentTime = tic;
            cachedInfo              = checkTimeCache( uuid );
            secondsSinceFirstCheck  = toc( cachedInfo{1} );
            secondsSinceLatestCheck = toc( cachedInfo{2} );

            if secondsSinceLatestCheck > JobStateHelper.TaskCheckIntervalSeconds || ...
                    secondsSinceFirstCheck > JobStateHelper.AssumeFailedTimeoutSeconds

                % Check again (Note that we always force a check before setting
                % state to be Failed)
                stateFromTasks = job.hGetStateFromTasks();
                if stateFromTasks.isTerminal()
                    % Great, we're done.
                    jobState = stateFromTasks;
                else
                    % update last check time
                    cachedInfo = { cachedInfo{1}, currentTime };
                    checkTimeCache( uuid ) = cachedInfo;
                end
            end

            if ~jobState.isTerminal() && ...
                    secondsSinceFirstCheck > JobStateHelper.AssumeFailedTimeoutSeconds
                % Too much time has elapsed, force the job to be Failed.
                jobState = States.Failed;
            end

            % Remove from the cache if the state is terminal
            if jobState.isTerminal()
                checkTimeCache.remove( uuid );
            end
        end
    end

    methods ( Static )
        function jobState = getJobStateFromTasks( job, jobState, uuid )
        %getJobState return job state when scheduler is ignorant
        %   jobState = JobStateHelper.getJobState( jobObj, jobState, uuid )
        %   calculates the job state based on:
        %   job - the actual job object
        %   jobState - State of job on disk
        %   uuid - some unique key (such as LSF ID).
        %
        %   When returning 'finished' or 'failed', we ensure that we leave
        %   nothing behind in the cache.

        % We get here when the scheduler has nothing to say about a particular
        % job, and the state is either 'queued' or 'running'. We need to decide
        % whether the job should be considered 'finished' or 'failed'.

            import parallel.internal.cluster.JobStateHelper
            import parallel.internal.types.States

            checkTimeCache = JobStateHelper.TaskCheckTimeCache;

            assert( jobState == States.Queued || jobState == States.Running );

            if checkTimeCache.isKey( uuid )
                % Seen this job before
                jobState = JobStateHelper.getJobStateForKnownJob( ...
                    job, jobState, uuid, checkTimeCache );
            else
                % We know nothing of this job
                jobState = JobStateHelper.getJobStateForUnknownJob( ...
                    job, jobState, uuid, checkTimeCache );
            end
        end
    end
end
