function pAddDependenciesFromZipfile(zipfileName, attachedFilePaths, dependencyDir)
% pAddDependenciesFromZipFile - Unzip the zip archive into the attached files folder.
%
%   zipFileName - the name of the zip file containing the attached files. Empty if no attached files.
%
%   attachedFilePaths - maps original path roots to the truncated path roots in the zip archive
%       In API2: EITHER 
%       - an array of Strings of the client paths and worker paths
%         interleaved.
%       - an AttachedFileMap
%
%   dependencyDir - path to the attached files folder on the worker.

%   Copyright 2009-2023 The MathWorks, Inc.

% TODO g2989107 - rename this file

% Get the runtime properties of the cluster
root = distcomp.getdistcompobjectroot;
runprop = root.CurrentRunprop;

if ~isempty(zipfileName)
    unzip(zipfileName, dependencyDir);
end

if isa( attachedFilePaths, 'java.lang.String[]' )
    attachedFilePaths = cell( attachedFilePaths );
end

% When called from MATLAB dependencyPaths will be an AttachedFileMap  which
% we'll use to create the dependency map. Instead, when this method is
% called from Java, dependencyPath is a cell array that represents the
% interleaved paths on the client and worker.
if isa( attachedFilePaths, 'parallel.internal.apishared.AttachedFileMap' )
    % Since dependencyPaths *is* an AttachedFileMap, just assign it to
    % runprop.AttachedFilesPathMap.
    tmpMap = runprop.AttachedFilesPathMap;
    tmpMap.mergePathEntries(attachedFilePaths);
    pathsToAdd = iGetPathsToAdd( attachedFilePaths, dependencyDir );
    iInstallAndResolvePackageRegistry( attachedFilePaths, dependencyDir );
else
    % PCT_JT_CLEANUP g2364333 - Remove the else branch, since this code is
    % only executed when this function is called from NewDirectoryDependencyCommand.java. 

    % Work out the folders to add to runprop.AttachedFilesPathMap
    clientList = attachedFilePaths(1:2:end);
    workerList = attachedFilePaths(2:2:end);
    % convert '\' to '/' in case the path is coming from a PC - NOTE PC
    % MATLAB can deal with '/' whereas unix cannot deal with '\' so ALWAYS
    % convert to unix.
    clientList = cellfun( @(x) strrep( x, '\', '/' ), clientList, ...
        'UniformOutput', false );
    tmpMap = runprop.AttachedFilesPathMap;
    cellfun( @tmpMap.put, clientList, workerList );
    
    dctSchedulerMessage( 6, 'Calculating dependency paths' );
    % Need to replace slashes in the dependency paths if we don't have an AttachedFileMap.
    workerList = cellfun( @(x) strrep( x, '\', '/' ), workerList, ...
        'UniformOutput', false );
    pathsToAdd = iCalculatePathsToAdd(dependencyDir, clientList, workerList);
end

% Add the paths only if we're not deployed. This is because addpath throws
% an error in deployed applications if the path-entry to be added is not
% already on the path. Omitting this addpath will not affect a worker's
% ability to execute code files, because deployed applications can only
% execute compiled MATLAB code, which is necessarily already on their path.
% It will however affect the worker's ability to access data files, as they
% will have to specify the full path to the file, rather than just their
% filename. For example, load('data.mat') will not work but
% load('/full/path/to/data.mat') will.
if ~isdeployed
    iAddPaths( pathsToAdd );
end

%--------------------------------------------------------------------------
% Helpers
%--------------------------------------------------------------------------

function pathsToAdd = iGetPathsToAdd( pathMap, dependencyDir )
% iGetPathsToAdd - convert paths from their original client path to the
%   equivalent local path on the worker, and return the local paths

originalPaths = pathMap.keys();
% These are the directories on the client of Packages
clientPkgPaths = pathMap.getPackageClientDirsAndIds();
% Need to remove any Package directories from the originalPaths
nonPkgOriginalPaths = setdiff(originalPaths, clientPkgPaths, 'stable');
pathsToAdd = {};
for ii = 1:numel(nonPkgOriginalPaths)
    clientPath = nonPkgOriginalPaths{ii};
    newPath = pathMap.get(clientPath);
    if newPath.AddToWorkerPath
        % The dependency map maps the file name on the client to the file name
        % on the worker.
        pathsToAdd = [ pathsToAdd fullfile(dependencyDir, newPath.WorkerPath) ]; %#ok<AGROW>
    end
end

function pathsToAdd = iCalculatePathsToAdd(rootdir, filelist, relativefilelist)
% filelist input is a cell array of strings that we need to match up to
% files or directories on the local machine. They might be relative or
% absolute and we need to distinguish.

% Create a cell array of all the paths that we will need to add to the
% current MATLAB path in one go
pathsToAdd = cell(size(filelist));

for i = numel(filelist):-1:1
    name = relativefilelist{i};
    localName = [rootdir filesep name];
    dctSchedulerMessage(6, ['Searching for ' localName ' ...']);
    % check for a relative path to a file or directory
    if exist(localName,'dir')
        % it is a relative path to a directory
        pathsToAdd{i} = localName;
    elseif exist(localName,'file')
        % it is a relative path to a file
        pathsToAdd{i} = fileparts(localName);
    else
        % This is here for compatibility with API-1 and
        % NewDirectoryDependencyCommand.java.
        %
        % it is an absolute path to a file or directory
        % OR
        % it is a relative path to a directory that was empty.
        % A directory could look empty because either,
        %  - a relative or absolute path to a actual empty directory
        %    was specified as a file dependency, or
        %  - a directory was specified as a file dependency, and all
        %    the files in the directory were also specified as file
        %    dependencies by their absolute path.

        % if the directory name ends in a filesep, then the
        % directory's contents will be placed by UNZIP in rootdir,
        % otherwise the directory will be placed in a subdirectory
        % of the appropriate name

        % Get the last part of the filename - this is everything after
        % the last '/', or the whole string if there are no slashes.
        seplocations = strfind(name,'/');
        if ~isempty(seplocations)
            nameEnd = name(seplocations(end)+1:end);
        else
            nameEnd = name;
        end
        localName = [rootdir filesep nameEnd];
        if name(end) == '/'
            % THIS CODE PATH SHOULD NEVER BE CALLED - whilst setting the
            % file dependencies on the job we are checking to see if the
            % element ends with a file separator and removing it.
            dctSchedulerMessage(1, 'File dependency with trailing file separator found - this was NOT expected.');
            % there was a filesep on the end of the directory name
            pathsToAdd{i} = rootdir;
        elseif exist(localName,'dir')
            % there was no filesep on the end of the directory name
            pathsToAdd{i} = localName;
        elseif exist(localName,'file')
            % it is an absolute file name
            pathsToAdd{i} = rootdir;
        else 
            % To get here the filelist entry can not be found locally.
            % This is probably because it was an empty dir and so didn't
            % get added to the zip file. We need to create it here so
            % any files added to this dir can get updated correctly.
            dctSchedulerMessage(1, 'File dependency can not be found - creating dir %s', localName);
            mkdir(localName);
            pathsToAdd{i} = localName;            
        end
    end
end

function iAddPaths( pathsToAdd )
% Add the pathStart of specialSplitPath() to the path rather than the folder
% name itself since special folder path names are not allowed in the path.
% Normal folders paths should be the same as pathStart and be unaffected.
pathsToAdd = cellfun(@parallel.internal.apishared.FilenameUtils.specialSplitPath, pathsToAdd, 'UniformOutput', false );

% Remove any empty elements / files
pathsToAdd(cellfun(@(x) isempty(x) || isfile(x), pathsToAdd)) = [];

pathsToAdd = unique(pathsToAdd, 'stable');
dctSchedulerMessage(4, 'Adding path entries: "%s"', strjoin(pathsToAdd, pathsep));
if ~isempty(pathsToAdd)
    parallel.internal.general.addpathWithFoldersFilteredIfDeployed(pathsToAdd{:});
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% The Path Map contains the serialized Package Registry data. We will install
% that, and "fix" the locations of newly-attached Packages, retaining any
% pre-existing Package locations.
function iInstallAndResolvePackageRegistry(pathMap, dependencyDir)
prData = pathMap.getPackageRegistryData();
if isempty(prData)
    dctSchedulerMessage(3, 'No Package Registry to install.');
    return
end

% TODO g2702048 remove this condition when Packages are enabled everywhere.
if ~matlab.internal.feature("packages")
    dctSchedulerMessage(2, 'Client was in "-packages" mode, but worker does not support Packages.');
    if ~isempty(pathMap.getPackagePathOrder())
        % Note that pre-submission checks ensure that Packages are not attached, but here we warn
        % because the client had Packages on the Path which the worker cannot use.
        warning(message('parallel:job:WorkerDoesNotSupportPackages'));
    end
    return
end

% Install Package Registry, then fix up locations. The serialized Package registry
% has the client locations burned into it. Here we compute a mapping of Package IDs to locations
% here on the worker that can be used to "fix" the worker's Package registry after
% deserialization. 
[clientPkgDirs, pkgIds] = pathMap.getPackageClientDirsAndIds();
workerPkgDirs = strings(size(clientPkgDirs));
keepMapping = true(1, numel(clientPkgDirs));
for ii = 1:numel(clientPkgDirs)
    workerTrail = pathMap.get(clientPkgDirs(ii));
    workerDir = fullfile(dependencyDir, workerTrail.WorkerPath);
    if isfolder(workerDir)
        % We have the Package contents here on the worker.
        dctSchedulerMessage(5, 'Would resolve %s --> %s', pkgIds(ii), workerDir);
        workerPkgDirs(ii) = workerDir;
    else
        % We do not currently have the Package contents here on the worker.
        % Get here during pool add attached files which sends Packages
        % individually, but must send the serialized Registry each time.
        dctSchedulerMessage(5, 'Not resolving %s --> %s as destination missing.', ...
            pkgIds(ii), workerDir);
        keepMapping(ii) = false;
    end
end

pkgIdsAttached = pkgIds(keepMapping);
workerPkgDirs = workerPkgDirs(keepMapping);

dctSchedulerMessage(4, 'About to install Package Registry.');
% Note that currently, installing a Package Registry adds all Packages to the Path, and this can
% result in warnings. We're going to sort out all the Path entries in just a moment anyway, so
% suppress these warnings.
warningSuppressor = ... 
    parallel.internal.general.WarningSuppressor('MATLAB:mpath:nameNonexistentOrNotADirectory'); %#ok<NASGU>
parallel.internal.packages.installSerializedRegistryBytes(prData, ...
    pkgIdsAttached, workerPkgDirs);
warningSuppressor = []; %#ok<NASGU>

% Finally, fix up the order of Packages on the Path. This depends only on the order of Packages back
% at the client, and is independent of whether we actually attached contents or not.
iFixPathOrderForPackages(pathMap.getPackagePathOrder());

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Given the order of Packages on the Path at the client:
% - remove any from the Path that should not be present.
% - ensure the others are on the Path in the correct order.
function iFixPathOrderForPackages(clientPkgPathOrder)

% First, work out which Packages are currently on the Path.
allPathDirs = strsplit(path, pathsep);
[allPkgIds, allPkgLocations] = parallel.internal.packages.listPackages();
pkgLoc2PkgId = dictionary(allPkgLocations, allPkgIds);

% Since all locations are "canonical", we may simply use ISMEMBER here.
pkgLocationsOnPath = allPathDirs(ismember(allPathDirs, allPkgLocations));
pkgIdsOnPath = pkgLoc2PkgId(pkgLocationsOnPath);

% Compute extraneous Packages
isExtraneousPkg = ~ismember(pkgIdsOnPath, clientPkgPathOrder);
dctSchedulerMessage(5, 'Extraneous Packages on the path: %s', strjoin(pkgIdsOnPath(isExtraneousPkg), pathsep));
pathsToRemove = pkgLocationsOnPath(isExtraneousPkg);
dctSchedulerMessage(5, 'Removing extraneous path entries: %s', strjoin(pathsToRemove, pathsep));
% Remove paths if present (and not deployed)
if ~isdeployed && ~isempty(pathsToRemove)
    rmpath(pathsToRemove{:});
end

% Finally, in GUPPI, we simply need to `ADDPATH -end` all the locations corresponding to
% clientPkgPathOrder.
pkgId2PkgLoc = dictionary(allPkgIds, allPkgLocations);
locsForPath = pkgId2PkgLoc(clientPkgPathOrder);
gotLoc = isfolder(locsForPath);
locsToAdd = locsForPath(gotLoc);
locsMissing = locsForPath(~gotLoc);
dctSchedulerMessage(5, 'Adding path entries: %s', strjoin(locsToAdd, pathsep));
dctSchedulerMessage(5, 'Missing path entries: %s', strjoin(locsMissing, pathsep));
if ~isempty(locsToAdd)
    warningSuppressor = ... 
        parallel.internal.general.WarningSuppressor('MATLAB:mpath:packagesMustBeLastOnPath'); %#ok<NASGU>
    parallel.internal.general.addpathWithFoldersFilteredIfDeployed(locsToAdd{:}, '-end');
end
