classdef CONDORMATLAB
        methods(Static)
            % validate the input data
            % OUTPUT:
            % 0: no error
            % 1: invalid inputs
            function [status] = MLCheckInput(XStart, XEnd, NumberOfPoints, exeFile)
                    status = 0;
                 try
                    if(XStart > XEnd)

                        disp('Start point must be less than or equal end point.');
                        status = 1;
                        return;
                    end
                     % check the lower bound and upper bound for NumberOfPoints
                    if(NumberOfPoints <=0 || NumberOfPoints > 10000000)

                        disp('Number of points must be a positive integer from 1 to 10,000,000');
                        status = 1;
                        return;

                    end
                    if(mod(NumberOfPoints,1) ~=0)

                        disp('Number of points must be a positive integer.');
                        status = 1;
                        return;
                    end

                    if(exist(exeFile) == 0)
                        disp('ERROR: Can not find the executable file.');
                        status = 1;
                        return;

                    end

                 catch err
                    disp('ERROR: invalid input data.');
                    msg = getReport(err);
                    disp(msg);
                    status = 1;
                 end
            end
            
            % clear all old data from the last time
            function [status] = MLClearOldData()
                    status = 0;
                    try
                    cover = 'CoverMe';
                    appFiles = 'CondorMatlab_App';
                    % delete CoverMe files
                     delete(['*',cover,'*']);
                     % delete Setting File and XML log fileshist
                     delete(['*', appFiles, '*']);
                     % delete log file
                     if(exist('DrawGraph.log','file') ~= 0)
                        delete('DrawGraph.log');
                     end
                    % delete error file
                     if(exist('DrawGraph.err', 'file') ~= 0)
                         delete('DrawGraph.err');
                     end

                    % delete input files
                    if(exist('inp', 'file') == 7)
                         rmdir('inp','s');
                    end

                    % delete output files
                    if(exist('out', 'file') == 7)
                         rmdir('out','s');
                    end

                    catch err
                        disp('ERROR: Cannot delete temporary files.');
                        msg = getReport(err);
                        disp(msg);
                        status = 1;
                    end
            end
            
            function [status] = MLInitGlobalParams_DrawGraph(NumberOfPoints)
                status = 0;
                try
                    global MAX_POINT;
                    MAX_POINT = 1000; % the maximum number of points in 1 chunk
                    global TOTAL_POINTS;
                    TOTAL_POINTS = NumberOfPoints;
                    global CHECK_POINT; % the number of jobs that application will submit in the next round
                    CHECK_POINT = 300;
                    global SETTING_FILE;
                    SETTING_FILE = 'CondorMatlab_AppSettings.xml';
                    global LOG_FILE;
                    LOG_FILE = 'CondorMatlab_AppLog';
                    global JOB_LIST;
                    JOB_LIST = containers.Map();
                    global LOG_LAST_POS;
                    % the position of file pointers in the log files
                    LOG_LAST_POS = containers.Map();
                    % contains the runtime of the complete jobs
                    global RUNTIMES;
                    RUNTIMES = zeros(1, NumberOfPoints);
                    global FOLDER;
                    FOLDER = CONDORMATLAB.MLGeneratePath();
                    s = mkdir(FOLDER);
                    if(s == 0)
                        disp('ERROR: Cannot create folder.');
                        status = 1;
                        return;
                    end
                catch err
                    disp('Cannot initialize global params.');
                    status = 1;
                    msg = getReport(err);
                    disp(msg);
                end
            end
            
            
            % get incomplete jobs from 1 chunk
            function [xArray, status] = MLGetIncompleteJobFromOneChunk(chunkNumber, LOG_FILE, FOLDER)
                status = 0;
                xArray = [];
                
                logFile = [FOLDER, '/', LOG_FILE, num2str(chunkNumber), '.xml'];
                if(exist(logFile, 'file') == 0)
                    disp('file not found');
                    return;
                end
                try
                    xmlDoc = xmlread(logFile);
                    % go to root node
                    xmlRoot = xmlDoc.getDocumentElement;
                    % check attribute Status, if Status = complete
                    % then exit
                    attStatus = xmlRoot.getAttribute('Status');
                    if(strcmp(attStatus, 'complete') == 1)
                       
                        return;
                    end
                                        
                    % get all job nodes
                    allJobs = xmlRoot.getElementsByTagName('Job');
                    LENGTH = allJobs.getLength;
                    % get values from incomplete jobs
                    xArray = zeros(1, LENGTH);
                    index = 1;
                    for i = 0: LENGTH-1
                        currentNode = allJobs.item(i);
                        % get node Status
                        nodeStatus = currentNode.getChildNodes.item(3).getTextContent;
                        % if node Status is not complete
                        if(strcmp(nodeStatus, '1') ~= 1)
                            % then add value to the array
                            xValue = str2double(currentNode.getChildNodes.item(5).getTextContent);
                            xArray(index) = xValue;
                            index = index + 1;
                        end
                    end
                    xArray = xArray(1:index-1);
                    
           
                catch err
                    status = 1;
                    disp('ERROR: Cannot collect incomplete jobs!');
                    % display message when we have exceptions
                    msg = getReport(err);
                    disp(msg);
                end
            
            end
            
            
            
            % log the application settings
            function [status] = MLGenerateAppSettings(NumberOfPoints, MaxPoint, CheckPoint, Function, SettingFile, Folder)
                    try
                         docNode = com.mathworks.xml.XMLUtils.createDocument('Settings');
                         docRootNode = docNode.getDocumentElement;

                         % create node MaxPoint
                         thisElement = docNode.createElement('MaxPoint');
                         thisElement.appendChild(docNode.createTextNode(num2str(MaxPoint)));
                         docRootNode.appendChild(thisElement); 

                         % create node CheckPoint
                         thisElement = docNode.createElement('CheckPoint');
                         thisElement.appendChild(docNode.createTextNode(num2str(CheckPoint)));
                         docRootNode.appendChild(thisElement); 

                         % create node NOP=NumberOfPoints
                         thisElement = docNode.createElement('NOP');
                         thisElement.appendChild(docNode.createTextNode(num2str(NumberOfPoints)));
                         docRootNode.appendChild(thisElement);

                         % create node Function=Function
                         thisElement = docNode.createElement('Function');
                         thisElement.appendChild(docNode.createTextNode(Function));
                         docRootNode.appendChild(thisElement); 

                         % write data to file 
                         xmlwrite([Folder,'/', SettingFile],docNode);

                    catch err
                            status = 1;
                            disp('ERROR: Cannot create application settings!');
                            % display message when we have exceptions
                            msg = getReport(err);
                            disp(msg);
                            return;
                    end
                        status = 0;
            end
                       
            % generate all application log files
            function [status, TOTAL_CHUNK] = MLGenerateAllLogFiles(TOTAL_POINT, MAX_POINT, CHECK_POINT, X_START, X_END, LOG_FILE, FOLDER)
                TOTAL_CHUNK = 0;
                allElements = linspace(X_START, X_END, TOTAL_POINT);
                allElements = CONDORMATLAB.MLSwapElements(allElements);
                progressBar = waitbar(0, 'Generating input data. Please wait...');
                try
                
                    % if we have only 1 chunk
                    if(TOTAL_POINT <= MAX_POINT)
                        
                       status = CONDORMATLAB.MLGenerateLogFile(allElements, 1,  LOG_FILE, FOLDER);
                       waitbar(1, progressBar, 'Completed: 100%');
                       delete(progressBar);
                       if(status ==1)
                           return;
                       end
                       TOTAL_CHUNK = 1;
                    else
                       currentChunk = 1;
                       currentIndex = MAX_POINT;
                       currentArray = allElements(1:currentIndex);
                       % first chunk

                       status = CONDORMATLAB.MLGenerateLogFile(currentArray, currentChunk, LOG_FILE, FOLDER);
                       if(status == 1)
                           delete(progressBar);
                           return;
                       end
                       waitbar(currentIndex/TOTAL_POINT, progressBar, [sprintf('Completed: %.2f', currentIndex*100/TOTAL_POINT), '%']);
                       % middle chunks   
                       while(TOTAL_POINT > MAX_POINT + CHECK_POINT * currentChunk)
                          currentArray = allElements(currentIndex + 1: currentIndex + CHECK_POINT);
                          currentChunk = currentChunk + 1;
                          status = CONDORMATLAB.MLGenerateLogFile(currentArray, currentChunk, LOG_FILE, FOLDER);
                          if(status == 1)
                              delete(progressBar);
                              return;
                          end
                          % update currentIndex
                          currentIndex = currentIndex + CHECK_POINT;
                          % redraw progress bar
                          waitbar(currentIndex/TOTAL_POINT, progressBar, [sprintf('Completed: %.2f', currentIndex*100/TOTAL_POINT), '%']);

                       end

                       % last chunk
                       currentArray = allElements(currentIndex + 1:end);
                       currentChunk = currentChunk + 1;
                       status = CONDORMATLAB.MLGenerateLogFile(currentArray, currentChunk, LOG_FILE, FOLDER);
                       if(status == 1)
                         delete(progressBar);
                         return;
                       end
                       % redraw progress bar
                       waitbar(1, progressBar,'Completed: 100%');
                       delete(progressBar);
                       TOTAL_CHUNK = currentChunk;
                    end
                catch err
                    status = 1;
                    disp('ERROR: Cannot create input data!');
                    % display message when we have exceptions
                    msg = getReport(err);
                    disp(msg);
                    delete(progressBar);
                end
            end
            
            
            % generate application log file
            function [status] = MLGenerateLogFile(XArray, chunkID, logFile, Folder)
                status = 0;
                try
                    docNode = com.mathworks.xml.XMLUtils.createDocument('Jobs');
                    docRootNode = docNode.getDocumentElement;
                    
                    % mark this file as incomplete
                    docRootNode.setAttribute('Status', 'incomplete');
                    
                    for i = 1: length(XArray)
                       data = XArray(i);
                       [status, job] = CONDORMATLAB.MLGenerateJobNode(data, docNode);
                       docRootNode.appendChild(job);
                    end
                    % write data to file
                    fileName = [logFile, num2str(chunkID), '.xml'];
                    xmlwrite([Folder, '/', fileName],docNode);                    
                catch err
                    status = 1;
                    disp('ERROR: Cannot create application log file!');
                    % display message when we have exceptions
                    msg = getReport(err);
                    disp(msg);
                end
            end
            
            % generate application node
            function [status, job] = MLGenerateJobNode(XValue, document)
                status = 0;
                try
                job = document.createElement('Job');
                    id = document.createElement('ID');
                    job.appendChild(id);
                
                    status = document.createElement('Status');
                    status.appendChild(document.createTextNode('NA'));
                    job.appendChild(status);
                    
                    x = document.createElement('X');
                    x.appendChild(document.createTextNode(num2str(XValue)));
                    job.appendChild(x);
                    
                    y = document.createElement('Y');
                    job.appendChild(y);
                    
                    startTime = document.createElement('StartTime');
                    job.appendChild(startTime);
                    
                    finishTime = document.createElement('FinishTime');
                    job.appendChild(finishTime);
                catch err
                    status = 1;
                    disp('ERROR: Cannot create job node!');
                    % display message when we have exceptions
                    msg = getReport(err);
                    disp(msg);
                end
                
            end
            
            % Swap the elements of an array, so we can get the special elements first. 
            % For example: 
            %  input array: [1 2 3 4 5 6 7 8]
            %  output array: [1 4 8 2 6 3 5 7]
            % INPUT:
            % - inList: input array
            % OUTPUT:
            % - outList: an array. This is the new array that contains all the elements but in a new order.
            function [outList] = MLSwapElements(inList)
                n = length(inList);
                checkedList = zeros(1,n);
                count = 2;
                outList = zeros(1, n);
                outList(1) = inList(1);
                outList(2) = inList(n);
                INDEX = 3;
                checkedList(1) = 1;
                checkedList(n) = 1;

                while(sum(checkedList) < n)
                   part = int32(n/count);
                   if(part == 0)
                       break;
                   end
                   range = int32(n/count);
                   while(part <= n)
                       if(checkedList(part) == 0)
                           outList(INDEX) = inList(part);
                           INDEX = INDEX + 1;
                           checkedList(part) = 1;
                       end
                       part = range + part;
                   end
                   count = count * 2;

                end
                outList = outList(1:INDEX-1);
            end

            
            
            % Stores elements of input array into files. Each file contains 1 element in real value. 
            % For example: xArray = [1 2 3], the output will be: file inp.0 has number 1, 
            % file inp.1 has number 2, file inp.2 has number 3.
            % INPUT:
            % - xArray: input array which has real-type elements.
            % OUTPUT:
            % Boolean type. True if all the processes have completed, else return false. 
            function [status] = MLWriteData(xArray, chunkIndex, FOLDER)
                    status = 0; % true
                try
                    % create input and output folders
                    mkdir([FOLDER, '/chunk', int2str(chunkIndex), '/inp']);
                    mkdir([FOLDER, '/chunk', int2str(chunkIndex), '/out']);
                    fname = [FOLDER, '/chunk',int2str(chunkIndex),'/inp/inp'];

                    for i = 1: length(xArray)
                        fid = fopen([fname,'.',int2str(i-1)], 'w');
                        fprintf(fid, '%f ', xArray(i));
                        fclose(fid);
                    end
                catch err
                    disp('Cannot write input data to files');
                    status = 1; % false
                end
            end
           
            % generate Condor submit file
            % INPUT:
            % - exeFile: executable file in Condor submit file
            % - num: number of Condor jobs
            % - chunkIndex: the index of the chunk
            function [status] = CDGenerateCondorSubmitFile(exeFile, num, chunkIndex, FOLDER)
            status = 0;
            try
                % Condor input file name
                inp = [FOLDER,'/chunk', int2str(chunkIndex),'/inp/inp'];
                % Condor error file name
                errorFileName = [FOLDER,'/chunk', int2str(chunkIndex) ,'/DrawGraph.err'];
                % Condor output file
                outputFileName = [FOLDER,'/chunk', int2str(chunkIndex), '/out/out'];
                % Condor log file name
                logFileName = [FOLDER,'/chunk',int2str(chunkIndex), '/DrawGraph.log']; 
                % write necessary data into Condor submit file (CoverMe.sub)
                fid = fopen([FOLDER, '/chunk', int2str(chunkIndex),'/CoverMe.sub'], 'w');
                % config executable file info. Ex: executable=a.out
                fprintf(fid, 'executable=%s\r\n', exeFile);
                % config universe info. Ex: universe=vanilla
                fprintf(fid, 'universe=%s\r\n', 'vanilla');
                % config arguments with inputs. Ex: arguments=inp.0
                fprintf(fid, 'arguments=%s.$(Process)\r\n', 'inp');
                % config environment. Ex: getenv=true
                fprintf(fid, 'getenv=%s\r\n', 'true');
                % config transfer files. Ex: when_to_transfer_output=ON_EXIT
                fprintf(fid, 'when_to_transfer_output=%s\r\n', 'ON_EXIT');
                % should_transfer_files=yes
                fprintf(fid, 'should_transfer_files=yes\r\n');
                % transfer_input_files=a.out,inp.0
                fprintf(fid, 'transfer_input_files=%s,%s.$(Process)\r\n',exeFile, inp);
                % config to run the application on Submit2
                % fprintf(fid, 'requirements=%s\r\n', 'Is_Glidein');
                % fprintf(fid, '+WantGlidein=%s\r\n', 'True');
                % config output file name. output=out.0, out.1.... 
                fprintf(fid, 'output=%s.$(Process)\r\n',outputFileName);
                % config error file name. error=err.0, err.1.....
                fprintf(fid, 'error=%s\r\n',errorFileName);
                % config log file. log=log.0, log.1
                fprintf(fid, 'log=%s\r\n', logFileName);
                % disable sending email function. notification=never
                fprintf(fid, 'notification=%s\r\n', 'never');
                % config queue. Ex: queue 1000
                fprintf(fid, 'queue %d', num);
            fclose(fid);
            catch err
                disp('ERROR: Cannot create Condor submit file.');
                status = 1;
            end

            end
            
            function [status, newJobList, logPos, XandProcessID] = CDSubmitDataToCondor(cusFunction, NumberOfPoints, xArray, chunkIndex, FOLDER)
                
                status = 0;
                
                submitFileName = [FOLDER, '/chunk', int2str(chunkIndex),'/CoverMe.sub'];
                status = CONDORMATLAB.CDGenerateCondorSubmitFile(cusFunction, NumberOfPoints, chunkIndex, FOLDER);
                if(status == 1)
                    return;
                end
                
                % submit Condor submit file in the current chunk to Condor
                command = sprintf('condor_submit %s',submitFileName);
                % store status and return values of command 'system' in 2 params
                [cmdStatus, cmdResult] = system(command);
                % if the action fails
                if(cmdStatus ~= 0)
                    % display error message
                   disp(cmdResult);
                   status = 1;
                   return;
                end
                
                % get submitted jobs information
                [status, newJobList, logPos, XandProcessID] = CONDORMATLAB.CDGetSubmittedJobsInformation(xArray, chunkIndex, FOLDER);
                if(status == 1)
                    return;
                end
                
            end
            
            function [status, newJobList, logPos, XandProcessID] = CDGetSubmittedJobsInformation(xArray, chunkIndex, FOLDER)
                % create dictionaries
                status = 0;
                % 0 : submitted
                % 1 : complete
                % 2 : was held
                % 3 : was aborted
                % 4 : was suspended
                % 5 : was evicted
                newJobList = containers.Map();
                XandProcessID = containers.Map();
                
                logFileName = [FOLDER, '/chunk',int2str(chunkIndex), '/DrawGraph.log'];
                % open file
                fid = fopen(logFileName, 'r');
                % read a line in the file
                logPos = 0;
                curLine = fgetl(fid);
                
                while ischar(curLine)
                    logPos = logPos + 1;
                    % if the line contains 'Job submitted from host', 
                      submittedLine = strfind(curLine, 'Job submitted from host');
                      if(isempty(submittedLine) ==0)
                          % get the process id
                          [status, processID] = CONDORMATLAB.CDGetProcessID(curLine);
                          if(status == 1)
                              return;
                          end
                          % add <key, value> to dictionaries
                          job.ID = processID;
                          job.Chunk = chunkIndex;
                          job.Status = 0;
                          [status, processIndex] = CONDORMATLAB.CDGetProcessIndex(processID);
                          job.X = xArray(processIndex);
                          job.Y = 'NA';
                          job.StartTime = 'NA';
                          job.EndTime = 'NA';
                          XandProcessID(num2str(xArray(processIndex))) = job.ID;
                          newJobList(processID) = job;
                      end
                     % read the next line
                     curLine = fgetl(fid);     
                end
                
            end
            
            % extract the process ID in a line
            function [status, processID] = CDGetProcessID(line)
            status = 0;
            processID = '';
            try
            % extract the process number
                % find the start pos
                startPos = strfind(line, '(');
                % find the end pos
                endPos = strfind(line, ')');
                % get the processID
                processID = line(startPos:endPos);
            catch err
                status = 1;
                disp('Cannot get the process id');
            end
            end
            
            function [status, processIndex] = CDGetProcessIndex(processID)
               status = 0;
               processIndex = -1;
               try
                   % find the character .
                   dotPositions = strfind(processID, '.');
                   % get the process code
                   processCode = processID(dotPositions(1)+1: dotPositions(2)-1);
                   % convert process code to number
                   processIndex = str2double(processCode) + 1;
                   
               catch err
                   disp('Cannot get the process number');
                   status = 1;
               end
            end
            
            function [status, newJobList, logPos] = CDSubmitJobsInChunk(chunkIndex, LOG_FILE, cusFunction, FOLDER)
                % 1. get jobs from first chunk
                [xArray, status] = CONDORMATLAB.MLGetIncompleteJobFromOneChunk(chunkIndex, LOG_FILE, FOLDER);
                if(status == 1)
                    return;
                end
                % 2. generate input data
                status = CONDORMATLAB.MLWriteData(xArray, chunkIndex, FOLDER);
                if(status == 1)
                    return;
                end
                % 3. submit jobs, return dictionaries
                [status, newJobList, logPos, XandProcessID] = CONDORMATLAB.CDSubmitDataToCondor(cusFunction, length(xArray), xArray, chunkIndex, FOLDER);
                if(status == 1)
                    return;
                end
                
                % display message
                disp(['Submitted ', int2str(newJobList.Count), ' jobs to Condor.']);
                
                % 4. update log file with processID and X value
                status = CONDORMATLAB.MLUpdateProcessIDInLogFile(chunkIndex, LOG_FILE, XandProcessID, FOLDER);
                if(status == 1)
                    return;
                end
                
            end
            
            
            function status = MLUpdateProcessIDInLogFile(chunkIndex, LOG_FILE, XandProcessID, FOLDER)
                
                fileName = [FOLDER, '/', LOG_FILE, num2str(chunkIndex), '.xml'];
                % open file
                 xDoc = xmlread(fileName);
                 % root Node
                 xRoot = xDoc.getDocumentElement;
                % get the selected job
                jobs = xRoot.getElementsByTagName('Job');
                for i = 0: jobs.getLength - 1
                    curJob = jobs.item(i);
                    % find the selected job
                    key = char(curJob.getChildNodes.item(5).getTextContent);
                    
                    value = XandProcessID(key);
                    
                    % update processID
                    curJob.getChildNodes.item(1).setTextContent(value);
                end
                % save file
                xmlwrite(fileName, xDoc);
                 % remove space from that file
                 status = CONDORMATLAB.MLRemoveSpace(fileName);
                 if(status == 1)
                    return;
                 end
                
            end
            
            
            % Check the log files to detect the completed Condor jobs.
            % search the line: (1) Normal termination
            % INPUT:
            % - n : number of log files. With n, we will have n log files from �log.0� to �log.(n-1)�.
            % - doneList: the completed log files so far.
            % OUTPUT: 
            % - outCompletedJobs: This is the current completedJobs which mark all the completed jobs.
            % - outUncompletedJobs: This is the current uncompletedJobs which mark all
            % the completed jobs but don't have correct output values.
            % STATUS:    
                % 0 : submitted
                % 1 : complete
                % 2 : was held
                % 3 : was aborted
                % 4 : was suspended
                % 5 : was evicted
            function [status, outJobList, outLOG_LAST_POS, outRUNTIMES, outRUNTIMES_INDEX] = CDCheckLogFileInChunk(chunkIndex, JobList, LOG_LAST_POS, RUNTIMES, RUNTIMES_INDEX, FOLDER)
            % get the index of last line in the last time
                lastIndex = LOG_LAST_POS(num2str(chunkIndex));
                currentIndex = lastIndex;
                status = 0;
            try
                fileName = [FOLDER, '/chunk',int2str(chunkIndex), '/DrawGraph.log'];
                lastLine = '';
                % open file
                fid = fopen(fileName, 'r');
                % read a line in the file
                curLine = fgetl(fid);
                
                while ischar(curLine)
                    % omit lastIndex (th) lines
                    if(lastIndex > 0)
                        lastIndex = lastIndex - 1;
                        curLine = fgetl(fid);
                        continue;
                    end
                        
                    currentIndex = currentIndex + 1;
                    % if the line contains 'Job executing on host', store the
                    % executing time of that jobs
                    submittedLine = strfind(curLine, 'Job executing on host');
                    if(isempty(submittedLine) == 0)
                         % get the process code
                         [status, processID] = CONDORMATLAB.CDGetProcessID(curLine);
                         if(status == 1)
                             break;
                         end
                         % get the associate job
                         job = JobList(processID);
                         % get start time
                         [status, time] = CONDORMATLAB.MLGetTime(curLine);
                         job.StartTime = time;
                         % update the job
                         JobList(processID) = job;
                         
                    else
                        % if the line contains 'Job was held', mark the job as
                        % incomplete
                        heldLine = strfind(curLine, 'Job was held');
                        if(isempty(heldLine) == 0)
                             % get the process code
                             [status, processID] = CONDORMATLAB.CDGetProcessID(curLine);
                             job = JobList(processID);
                             job.Status = 2;
                             JobList(processID) = job;
                        else
                            
                            % if the line contains 'Job was suspended', mark the job as
                            % status 4
                            suspendedLine = strfind(curLine, 'Job was suspended');
                            if(isempty(suspendedLine) == 0)
                             [status, processID] = CONDORMATLAB.CDGetProcessID(curLine);
                             job = JobList(processID);
                             job.Status = 4;
                             JobList(processID) = job;
                            else
                                % if the line contains 'Job was evicted', mark the
                                % job as status 5
                                evictedLine = strfind(curLine, 'Job was evicted');
                                if(isempty(evictedLine)==0)
                                    [status, processID] = CONDORMATLAB.CDGetProcessID(curLine);
                                    job = JobList(processID);
                                    job.Status = 5;
                                    JobList(processID) = job;
                                else

                                    % if the line contains '(1) Normal termination (return value 0)'
                                    completedLine = strfind(curLine, '(1) Normal termination (return value 0)');
                                    if(isempty(completedLine) == 0)
                                             % get the process code
                                            [status, processID] = CONDORMATLAB.CDGetProcessID(lastLine);
                                            % update status of the job
                                            job = JobList(processID);
                                            job.Status = 1;
                                            % get Y value of the job
                                            [status, processNumber] = CONDORMATLAB.CDGetProcessIndex(processID);
                                            [status, job.Y] = CONDORMATLAB.CDGetOutput(chunkIndex, processNumber-1, FOLDER);
                                            % update the finish time of the job
                                            [status, time] = CONDORMATLAB.MLGetTime(lastLine);
                                            
                                            job.EndTime = time;
                                            JobList(processID) = job;
                                            
                                            % add runtime of the job to
                                            % histogram
                                            % GIAI PHAP TINH THE
                                            if(strcmp(num2str(job.StartTime), 'NA'))
                                                RUNTIMES(RUNTIMES_INDEX) = 0;
                                            else
                                                
                                                duration = job.EndTime - job.StartTime;
                                                RUNTIMES(RUNTIMES_INDEX) = duration;
                                            end
                                            RUNTIMES_INDEX = RUNTIMES_INDEX + 1;

                                    else
                                       % check if the job finished but didn't have correct output value
                                       % if job was aborted
                                       aborted = strfind(curLine, 'Job was aborted by the user.');
                                       % if the job finished but didn't have value
                                       error = strfind(curLine, 'Job terminated.');
                                       if(isempty(aborted) == 0 || isempty(error) == 0)
                                            % get the process code
                                            [status, processID] = CONDORMATLAB.CDGetProcessID(curLine);
                                            job = JobList(processID);
                                            job.Status = 3;
                                            JobList(processID) = job;
                                       end
                                    end
                                end
                            end
                        end
                    end
                    % read the next line
                    lastLine = curLine;
                    curLine = fgetl(fid);
                end
                fclose(fid);
            catch  err
                % display message when we have exceptions
                msg = getReport(err);
                disp(msg);
                status = 1;
            end

            outJobList = JobList;
            LOG_LAST_POS(num2str(chunkIndex)) = currentIndex;
            outLOG_LAST_POS = LOG_LAST_POS;
            outRUNTIMES = RUNTIMES;
            outRUNTIMES_INDEX = RUNTIMES_INDEX;
            end
            
            
            function [status, outJobList, outLOG_LAST_POS, outRUNTIMES, outRUNTIMES_INDEX] = CDCheckLogFiles(JobList, LOG_LAST_POS, RUNTIMES, RUNTIMES_INDEX, FOLDER)
                key = LOG_LAST_POS.keys;
                for i = 1:length(key)
                    chunkIndex = str2num(key{i});
                    [status, JobList, LOG_LAST_POS, RUNTIMES, RUNTIMES_INDEX] = CONDORMATLAB.CDCheckLogFileInChunk(chunkIndex, JobList, LOG_LAST_POS, RUNTIMES, RUNTIMES_INDEX, FOLDER);
                end
                outJobList = JobList;
                outLOG_LAST_POS = LOG_LAST_POS;
                outRUNTIMES = RUNTIMES;
                outRUNTIMES_INDEX = RUNTIMES_INDEX;
            end
            
            function [status, outValue] = CDGetOutput(chunkIndex, processNumber, FOLDER)
                 status = 0;
                 outValue = 0;
                 try
                    fileName = [FOLDER, '/chunk',int2str(chunkIndex),'/out/out.', int2str(processNumber)];
                    % open output file to readinp
                    fid = fopen(fileName, 'r');
                    tline = fgetl(fid);
                    % read value line-by-line
                    while ischar(tline)
                        outValue = strread(tline, '%f');
                        tline = fgetl(fid);
                    end
                    fclose(fid);
                 catch err
                     status = 1;
                     msg = getReport(err);
                     disp(msg);
                 end
            end
            
            function [status, totalSoFar, terminatedJobs] = MLDrawCurrentData(JobList, TOTAL_POINT, currentTotal)
                  [status, newX, newY, terminatedJobs] = CONDORMATLAB.MLGetCompleteJobs(JobList);
                  totalSoFar = length(terminatedJobs) + currentTotal;
                  strComplete = [sprintf('Completed: %.2f', totalSoFar *100/TOTAL_POINT), '%'];
                  plot(newX, newY, '.-r','LineStyle','none', 'MarkerFaceColor','r');
                  title(strComplete);
                  xlabel('X value');
                  ylabel('Y value');
                  
            end
            
            % get array
            function [status, newX, newY, terminatedJobs] = MLGetCompleteJobs(JobList)
               LENGTH = JobList.Count;
               newX = zeros(LENGTH);
               newY = zeros(LENGTH);
               % initialize terminatedJobs
               terminatedJobs(LENGTH).ID = 'x';
               terminatedJobs(LENGTH).Chunk = 'x';
               terminatedJobs(LENGTH).Status = 'x';
               terminatedJobs(LENGTH).X = 'x';
               terminatedJobs(LENGTH).Y = 'x';
               terminatedJobs(LENGTH).StartTime = 'x';
               terminatedJobs(LENGTH).EndTime = 'x';
               status = 0;
               count = 1;
               jobIndex = 1;
               try
                   key = JobList.keys;
                   for i = 1:length(key)
                       processID = key{i};
                       job = JobList(processID);
                       if(job.Status == 1)
                           newX(count) = job.X;
                           newY(count) = job.Y;
                           count = count + 1;
                       end
                       % get terminated jobs, jobs have status = complete was held
                       % (2), was aborted (3)
                       if(job.Status == 1 || job.Status == 2 || job.Status ==3)
                           terminatedJobs(jobIndex) = job;
                           jobIndex = jobIndex + 1;
                       end
                   end
                   newX = newX(1:count-1);
                   newY = newY(1:count-1);
                   terminatedJobs = terminatedJobs(1:jobIndex-1);
               catch err
                   status = 1;
                   msg = getReport(err);
                   disp(msg);
               end
            end
            
            % update log liles, also remove terminated jobs from the ROOT
            % HASH TABLE
            function [status, outJOB_LIST] = MLUpdateLogFiles(terminatedJobs, JOB_LIST, LOG_FILE, FOLDER)
                status = 0;
                outJOB_LIST = JOB_LIST;
                if(isempty(terminatedJobs))
                    return;
                end
               
               
               
               try
                   [status, htJobsInChunk] = CONDORMATLAB.MLDivideJobsToChunk(terminatedJobs);

                   keys = htJobsInChunk.keys;
                   for i = 1:length(keys)
                      % get the chunk number in string
                      key = keys{i};
                      % get the hashtable contains jobs of that chunk
                      htJobs = htJobsInChunk(key);
                      [status, JOB_LIST] = CONDORMATLAB.MLUpdateJobStatusInChunk(htJobs, key, LOG_FILE, JOB_LIST, FOLDER);
                      if(status == 1)
                          return;
                      end
                   end
                    
                   outJOB_LIST = JOB_LIST;
                   
                   
                                     
               catch err
                   status = 1;
                   disp('ERROR: Cannot update log files.');
                   msg = getReport(err);
                   disp(msg);
               end
               
            end
            
            function [status, htJobsInChunk] = MLDivideJobsToChunk(terminatedJobs)
                % JobsInChunk is a hashtable: key -> ChunkIndex, value:
                % Hashtable that has key -> processID, value: job
                htJobsInChunk = containers.Map();
                status = 0;
                try
                    for i = 1: length(terminatedJobs)
                        job = terminatedJobs(i);
                        key = num2str(job.Chunk);
                        if(htJobsInChunk.isKey(key))
                            chunk = htJobsInChunk(key);
                            chunk(job.ID) = job;
                            htJobsInChunk(key) = chunk;
                        else
                            chunk = containers.Map();
                            chunk(job.ID) = job;
                            htJobsInChunk(key) = chunk;
                        end
                    end
                   
               catch err
                   status = 1;
                   disp('ERROR: Cannot divide terminated jobs into chunks.');
                   msg = getReport(err);
                   disp(msg);
               end
                
            end
            
            function [status, outJOB_LIST] = MLUpdateJobStatusInChunk(htJobs, chunkIndex, LOG_FILE, JOB_LIST, FOLDER)
                status = 0;
                outJOB_LIST = JOB_LIST;
                try
                 fileName = [FOLDER, '/', LOG_FILE, chunkIndex, '.xml'];
                % open file
                 xDoc = xmlread(fileName);
                 % root Node
                 xRoot = xDoc.getDocumentElement;
                % get the selected job
                jobs = xRoot.getElementsByTagName('Job');
                for i = 0: jobs.getLength - 1
                    curJob = jobs.item(i);
                    key = char(curJob.getChildNodes.item(1).getTextContent);
                    % if the job exists in the hashtable of terminated jobs
                    % in this chunk
                    if(htJobs.isKey(key))
                        job = htJobs(key);
                        % update status
                        curJob.getChildNodes.item(3).setTextContent(num2str(job.Status));
                        % update Y value
                        curJob.getChildNodes.item(7).setTextContent(num2str(job.Y));
                        % update StartTime
                        curJob.getChildNodes.item(9).setTextContent(num2str(job.StartTime));
                        % update EndTime
                        curJob.getChildNodes.item(11).setTextContent(num2str(job.EndTime));
                        
                        % remove that job from the ROOT HASH TABLE
                        JOB_LIST.remove(key);
                    end
                end
                outJOB_LIST = JOB_LIST;
                % save file
                xmlwrite(fileName, xDoc);
                 % remove space from that file
                 status = CONDORMATLAB.MLRemoveSpace(fileName);
                 if(status == 1)
                    return;
                 end
                catch err
                    status = 1;
                    disp('ERROR: Cannot update job status in chunk');
                    msg = getReport(err);
                    disp(msg);
                end
            end
            
            function [status] = MLUpdateJobStatus(job, LOG_FILE, FOLDER)
                status = 0;
                try
                chunkID = job.Chunk;
                fileName = [FOLDER, '/', LOG_FILE, num2str(chunkID), '.xml'];
                % open file
                 xDoc = xmlread(fileName);
                 % root Node
                 xRoot = xDoc.getDocumentElement;
                % get the selected job
                jobs = xRoot.getElementsByTagName('Job');
                for i = 0: jobs.getLength - 1
                    curJob = jobs.item(i);
                    % find the selected job
                    if(strcmp(curJob.getChildNodes.item(1).getTextContent, job.ID))
                        
                        % update status
                        curJob.getChildNodes.item(3).setTextContent(num2str(job.Status));
                        % update Y value
                        curJob.getChildNodes.item(7).setTextContent(num2str(job.Y));
                        % update StartTime
                        curJob.getChildNodes.item(9).setTextContent(num2str(job.StartTime));
                        % update EndTime
                        curJob.getChildNodes.item(11).setTextContent(num2str(job.EndTime));
                    end
                end
                % save file
                xmlwrite(fileName, xDoc);
                 % remove space from that file
                 status = CONDORMATLAB.MLRemoveSpace(fileName);
                 if(status == 1)
                    return;
                 end
                catch err
                    status = 1;
                    disp('Cannot update job status');
                    msg = getReport(err);
                    disp(msg);
                end
                    
            end
            
            function [success] = MLRemoveSpace(filePath)
                success = 0;
                try
                    % Read file.
                    fId = fopen(filePath, 'r');
                    fileContents = fread(fId, '*char')';
                    fclose(fId);

                    % Write new file.
                    fId = fopen(filePath, 'w');
                    % Remove extra lines.
                    fwrite(fId, regexprep(fileContents, '\n\s*\n', '\n'));
                    fclose(fId);
                catch
                    msg = sprintf('ERROR: Error when remove white spaces in the file %s', filePath);
                    disp(msg);
                    success = 1;
                end
            end
            
            
           % extract the date time in a line
            function [status, time] = MLGetTime(line)
            status = 0;
            time = 0;
            try
            % extract the date time value
                % find the character /
                slash = strfind(line, '/');
                % get the process code
                timeStr = line(slash-2: slash+11);
                % set time format
                fmt = 'mm/dd HH:MM:SS';
                time = datenum(timeStr, fmt)*86400;
            catch err
                status = 1;
                disp('ERROR: Cannot get time from current line');
                msg = getReport(err);
                disp(msg);
            end
            end
            
            
            function path = MLGeneratePath()
                format shortg;
                c = clock;
                c = fix(c);
                path = ['CDML', num2str(c(1)), '_', num2str(c(2)), '_', num2str(c(3)), '_', num2str(c(4)),'_', num2str(c(5)), '_', num2str(c(6))];
                format short;
            end
            
            
            %---------------- DISPLAY STATISTICAL INFO ---------------%
            function [status] = MLDislayInformation(TOTAL_CHUNK, LOG_FILE, FOLDER)
               status = 0;
               
               try
                   [status, successJobs, JobInfo] = CONDORMATLAB.MLCollectJobInfo(TOTAL_CHUNK, LOG_FILE, FOLDER);
                   disp(['Complete jobs: ', num2str(successJobs)]);
                   disp('Incomplete jobs: ');
                   keys = JobInfo.keys;
                   for i=1:length(keys)
                      key = keys{i};
                      disp(['Status: ', key, ', Total: ', num2str(JobInfo(key).Count)]); 
                   end
               catch err
                   status = 1;
                   disp('ERROR: Cannot display statistical information');
                   msg = getReport(err);
                   disp(msg);
               end
            end
            
            
            % 
            function [status, successJobs, JobInfo] = MLCollectJobInfo(TOTAL_CHUNK, LOG_FILE, FOLDER)
               status = 0;
               successJobs = 0;
               JobInfo = containers.Map();
               try
                   for i=1:TOTAL_CHUNK + 1
                        [status, successJobs, JobInfo] = CONDORMATLAB.MLCollectJobInfoInChunk(i, successJobs, JobInfo, LOG_FILE, FOLDER);
                        if(status == 1)
                            return;
                        end
                   end
               catch err
                   status = 1;
                   disp('ERROR: Cannot collec job info.');
                   msg = getReport(err);
                   disp(msg);
               end
            end
            
            function [status, outJobs, outJobInfo] = MLCollectJobInfoInChunk(chunkIndex, successJobs, JobInfo, LOG_FILE, FOLDER)
                status = 0;
                outJobs = successJobs;
                outJobInfo = JobInfo;
                try
                    logFile = [FOLDER, '/', LOG_FILE, num2str(chunkIndex), '.xml'];
                    if(exist(logFile, 'file') == 0)
                        status = 1;
                        return;
                    end
                    xDoc = xmlread(logFile);
                    % root Node
                    xRoot = xDoc.getDocumentElement;
                    jobs = xRoot.getElementsByTagName('Job');
                    
                    for i = 0: jobs.getLength - 1
                        % get current node
                        curNode = jobs.item(i);
                        
                        % collect all information of that job
                        job.ID = char(curNode.getChildNodes.item(1).getTextContent);
                        job.Chunk = chunkIndex;
                        job.Status = str2double(curNode.getChildNodes.item(3).getTextContent);
                        job.X = str2double(curNode.getChildNodes.item(5).getTextContent);
                        job.Y = str2double(curNode.getChildNodes.item(7).getTextContent);
                        job.StartTime = 'NA';
                        job.EndTime = 'NA';
                        % if job.Status already is a key of the hashtable
                            if(job.Status ~= 1)
                                key = num2str(job.Status);
                                if(JobInfo.isKey(key))
                                    curHT = JobInfo(key);
                                    curHT(job.ID) = job;
                                % if not, create new hashtable then add the job    
                                else
                                    curHT = containers.Map();
                                    curHT(job.ID) = job;
                                    
                                    JobInfo(key) = curHT;
                                end
                            else
                                % if job.Status = 1, just need to count it
                                successJobs = successJobs + 1;
                            end
                   end
                    
                    
                catch err
                   status = 1;
                   disp('ERROR: Cannot collect job info in chunk');
                   msg = getReport(err);
                   disp(msg);
                end
                outJobs = successJobs;
                outJobInfo = JobInfo;
            end
            
            
            %---------------- HISTOGRAM FUNCTIONS --------------------%
            
            
            
            
            
            % draw histogram about the runtime of complete jobs
            function [status] = MLDrawCompleteJobHistogram(RUNTIMES, RUNTIMES_INDEX)
                status = 0;
                try
                   RUNTIMES = RUNTIMES(1:RUNTIMES_INDEX-1);
                    % find min, max elements, then draw histogram
                    minValue = min(RUNTIMES);
                    maxValue = max(RUNTIMES);
                    % divide the time from min to max into every seconds
                    group = maxValue - minValue;
                    DISTANCE = group + 1;
                    if(DISTANCE > 100)
                        DISTANCE = 100;
                    end
                    ranges = linspace(minValue, maxValue, DISTANCE);
                    % draw histogram
                    figure('name','Histogram');
                    hist(RUNTIMES, ranges);
                    % set title and x-label, y-label, and color
                    title('Run-time distribution of the jobs');
                    xlabel('Run-time(s)');
                    ylabel('Number of jobs');
                    % set color for histogram
                    h = findobj(gca, 'Type', 'patch');
                    set(h, 'FaceColor','g','EdgeColor','w');
                catch err
                    status = 1;
                    disp('ERROR: Cannot display histogram');
                    msg = getReport(err);
                    disp(msg);
                end
            end
            
            
            % draw histogram about running time of all the jobs
            function [status] = MLDrawHistogram(NumberOfPoints, currentChunk)
            status = 0;
            try
                timeList = zeros(1, NumberOfPoints);
                total = 1;

                % get all the times in all chunks
                for i=0:currentChunk
                       [status, timeList, total] = GetTimeInChunk(i, timeList, total);
                       if(status == 1)
                           return;
                       end

                end
                % truncate the array
                timeList = timeList(1:total-1);   
                % find min, max elements, then draw histogram
                minValue = min(timeList);
                maxValue = max(timeList);
                % divide the time from min to max into every seconds
                group = maxValue - minValue;
                DISTANCE = group + 1;
                if(DISTANCE > 100)
                    DISTANCE = 100;
                end
                ranges = linspace(minValue, maxValue, DISTANCE);
                % draw histogram
                figure('name','Histogram');
                hist(timeList, ranges);
                % set title and x-label, y-label, and color
                title('Run-time distribution of the jobs');
                xlabel('Run-time(s)');
                ylabel('Number of jobs');
                % set color for histogram
                h = findobj(gca, 'Type', 'patch');
                set(h, 'FaceColor','g','EdgeColor','w');

            catch err
                 msg = getReport(err);
                 disp(msg);   
            end

            end
            
            
            % get time of complete jobs in a specific chunk
            function [status, timeListOut, totalOut ] = MLGetTimeInChunk(currentChunk, timeListIn, totalIn, LOG_FILE)
            status = 0;
            timeListOut = timeListIn;
            totalOut = totalIn;
            
            try
                    logFile = [LOG_FILE, num2str(currentChunk), '.xml'];
                    if(exist(logFile, 'file') == 0)
                        return;
                    end
                    xDoc = xmlread(logFile);
                    % root Node
                    xRoot = xDoc.getDocumentElement;
                    % get node CompletedJobs
                    CompletedJobsNode = xRoot.getChildNodes.item(1);
                    % get all submitted time
                    sNodes = CompletedJobsNode.getElementsByTagName('StartTime');
                    % get all finish Time
                    fNodes = CompletedJobsNode.getElementsByTagName('EndTime');

                    n = sNodes.getLength;
                    if(n == 0)
                        return;
                    end

                    % read all the node SubmittedTime, FinishTime and store the
                    % runningTime in array timeList
                    for i = 0:n-1
                        % get current value of node 
                        s = str2double(sNodes.item(i).getTextContent);
                        % get current value of node Y
                        f = str2double(fNodes.item(i).getTextContent);
                        % add x and y values to xArray and yArray
                        timeListIn(totalIn) = f - s;
                        totalIn = totalIn + 1;
                    end
                timeListOut = timeListIn;
                totalOut = totalIn;
            catch err
                msg = getReport(err);
                disp(msg);
            end


            end
            
            
        end
end