classdef MATFileService < mps.persistence.internal.MatlabService
% MATFileProvider Cache user data in MATLAB MAT-files.
%   Manage user data caches by storing data in MAT-files in a given folder.
%   
% Store (name,value) pairs in separate, parallel, data structures:
%   name: An array of strings (not char!)
%   value: A cell array.
% Guarantee: name(n) is name for value(n).

% Copyright 2020-2021 The MathWorks, Inc.
    properties
        Folder
    end
    
    properties (Access = private)
        Ext = '.mat';
        NameVar = 'name';
        ValueVar = 'value';
        HashPrefix = '#';
        Stopped = false;
        ConfigFile = '.configFile';
        Config
    end
    
    methods
        function obj = MATFileService(folder, varargin)
        %MATFileService Construct a MATFileService 
        %    mfp = MATFileService(folder) creates a MATFileService that
        %    manages user data caches stored in MATLAB MAT-files in the
        %    given folder. 
        %
        %    Options:
        %       KeepData: If true, do not delete cached data on stop().
        %           May also be an array of strings naming caches to
        %           preserve on stop.
        %
        
            import mps.cache.internal.InputValidator;
        
            p = inputParser;
            p.KeepUnmatched = true;
            addRequired(p,'folder',InputValidator.NonEmptyString);
            addParameter(p,'KeepData', false, ...
                @(a)validateattributes(a, {'logical', 'string'}, ...
                        {'nonempty'}));
            
            parse(p,folder,varargin{:});            
                     
            obj.Folder = p.Results.folder;
            
            obj.Config.KeepData = p.Results.KeepData;
        end
        
        function delete(obj)
            try
                stop(obj);
            catch me
                % TODO: Ignore "directory not found" errors.
                if ~strcmp(me.identifier, '')
                    rethrow(me);
                end
            end
        end
        
        function tf = activeService(obj)
            tf = ~obj.Stopped;
        end
        
        function [status,msg] = attach(obj)
            if strlength(obj.Folder) > 0 && ...
               exist(obj.Folder,'dir') ~= 7
                status = 0;
                msg = getString(...
                    message('MPS:MATLAB:Provider:AttachToExistingFolder', ...
                            "MAT-file", obj.Folder));
            else
                status = 1;
                msg = "";
            end
        end
        
        function [status,msg] = detach(obj)
            status = 1;
            msg = "";
            if obj.Stopped
                status = 0;
                msg = getString(...
                    message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                            "MAT-File", obj.Folder));
                return;
            end
        end
                        
        function [status,msg] = start(obj)
        % start Create the folder that will store the service's cache
        % files.
            if strlength(obj.Folder) > 0 && ...
               exist(obj.Folder,'dir') == 7
                status = 0;
                msg = getString(...
                    message('MPS:MATLAB:Provider:StartInNewFolder', ...
                            "MAT-file", obj.Folder));
            else
                [status,msg] = mkdir(obj.Folder);
                writeConfiguration(obj);
            end
        end
        
        function [status,msg] = stop(obj)
        % stop Remove all cache files in the service's folder.
        
            if obj.Stopped
                status = 0;
                msg = getString(...
                    message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                            "MAT-File", obj.Folder));
                return;
            end
        
            readConfiguration(obj);
            if obj.Config.KeepData == false
                [status,msg] = rmdir(obj.Folder,'s');
            elseif isstring(config.KeepData)
                d = dir(obj.Folder);
                % First cut for files to delete is all the files that are
                % not on the list to preserve.
                doomed = intersect(string({d.name}), obj.Config.KeepData);
                doomed = fullfile(obj.Folder, doomed);
                
                % But! Also save any composite key hashes associated with
                % caches the user wants to keep. Fortunately (sure, like it
                % is a concidence) the file name of a composite key hash 
                % associated with a cache ends with the file name of the
                % cache. endsWith to the rescue!
                pardoned = endsWith(doomed, obj.Config.KeepData);
                pardoned = [ pardoned ...
                    fullfile(obj.Folder, strcat(obj.ConfigFile, obj.Ext)) ];
                doomed = doomed(~pardoned);
                
                for k=1:numel(doomed)
                    delete(doomed(k));
                end
                status = 1;                
            end
            
            % Set Stopped to true so that:
            %   1) delete does not attempt to stop().
            %   2) Subsequent stop() errors.
            %   3) Data remains preserved according to KeepData.
            %   4) Cache operations fail.
            %
            % stop() means stop(). The service is finished for the day.
            obj.Stopped = true;
        end
        
        function [status,msg] = ping(obj)
            status = 0;
            if obj.Stopped == false
                status = 1;
                msg = message('MPS:MATLAB:Provider:FolderServicePong', ...
                    "MAT-File", obj.Folder);
            else
                msg = message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                    "MAT-File", obj.Folder);
            end
            msg = getString(msg);
        end
        
        function [status,msg] = version(obj)  
            if obj.Stopped == false
                status = 1;
                msg = version; % MATLAB version
            else
                status = 0;
                msg = getString(...
                    message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                            "MAT-File", obj.Folder));
            end
        end
        
        function reset(~)
        % reset Reset internal connection data.
        % Release and then reacquire internal resources.
        % For now, a no-op for MATFileService
        end
        
        function release(~)
        % reset Release internal connection data.
        % Close open file handles, etc.
        % For now, a no-op for MATFileService
        end        
        
        function tf = hasHash(obj, cache, hash)
        % Does the given cache exist on this service?
            file = hashFile(obj, cache, hash);
            tf = exist(file,'file') == 2;
        end
        
        function k = keys(obj, cache)
        % keys Return the keys stored in this cache by the service. 
        %
        % k = keys(obj, cache) returns a string matrix of all the keys
        % used in the given cache. Note that some keys may not be valid
        % MATLAB variable names. Returns an empty string if there are no
        % keys, or if the cache doesn't exist.
        
            if obj.Stopped
                error(message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                    "MAT-File", obj.Folder));
            end
        
            k = string.empty;
            
            file = fullfile(obj.Folder, strcat(cache, obj.Ext));
            
            if exist(file,'file') == 2
                data = load(file,obj.NameVar);
                k = data.(obj.NameVar);
            end
        end
        
        function sz = bytes(obj, cache, varargin)
           if obj.Stopped
                error(message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                    "MAT-File", obj.Folder));
            end            
            file = fullfile(obj.Folder, strcat(cache, obj.Ext)); 
            sz = nan(1,numel(varargin)); % Don't exist? Size == NaN

            if exist(file,'file') == 2
                m = matfile(file);
                names = m.(obj.NameVar);
                for v=1:numel(varargin)
                    pos = find(strcmp(varargin{v},names));
                    if isempty(pos)
                        continue;
                    end
                    % eval required because of subsref magic used to extract
                    % part of the "value" variable. Dynamic indexing cannot
                    % accept both field name and index into the value of
                    % the field.
                    getVar = sprintf("m.%s(1,%d)",obj.ValueVar,pos);
                    var = eval(getVar); %#ok<NASGU>
                    w = whos('var');
                    sz(v) = w.bytes;
                end
            end
        end
        
        function count = countRemoteKeys(obj, cache)
        % countRemoteKeys Return number of keys stored by the service.
        
            if obj.Stopped
                error(message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                    "MAT-File", obj.Folder));
            end        
        
            count = 0;
            
            file = fullfile(obj.Folder, strcat(cache, obj.Ext)); 
            if exist(file,'file') == 2
                m = matfile(file);
                % An oddity of implementation: the names are stored as a string
                % array, for more efficient access. But size(matfile(),...)
                % can't determine the size of a string array. However, we know
                % there are exactly the same number of keys as values (by
                % definition -- an invariant -- so we can count the number of
                % values instead). And size values are stored a cell array,
                % which size(matfile(), ...) can see into, we can retrieve the
                % count without loading some monsterous variable.
                count = size(m,obj.ValueVar,2);
            end
        end
        
        function [value, name] = hashGet(obj, cache, hash, name)
            
            if obj.Stopped
                error(message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                    "MAT-File", obj.Folder));
            end
            
            file = hashFile(obj, cache, hash);
            value = cell.empty;
            if nargin > 3
                % If names provided, read only the values of those names
                % from the hash. Don't error on non-existing names.
                data = load(file,obj.NameVar);
                if isfield(data, obj.NameVar)
                    kList = data.(obj.NameVar);
                    [exist, locK] = ismember(name, kList);
                    if any(exist)
                        data = load(file, obj.ValueVar);
                        value = data.(obj.ValueVar);
                        pos = locK(exist);
                        value = value(pos);
                    end
                    name = name(exist);
                else
                    name = "";
                end
            else
                % Read all names and values.
                data = load(file, obj.NameVar, obj.ValueVar);
                value = data.(obj.ValueVar);
                name = data.(obj.NameVar);
            end
            name = string(name);
        end

        function count = hashRemove(obj, cache, hashName, name)
        % hashRemove Remove named values (and the names) from the cache.
        %   count = hashRemove(obj, cache, hashName, name) removes
        %   the (name,value) pairs with names in the input name list.
        
            if obj.Stopped
                error(message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                    "MAT-File", obj.Folder));
            end 
            
            count = 0;
            
            % cache and hashName must not be empty
            
            % name and value must be the same size. name must be a 1xN
            % string array or cell array of chars. value must be a cell
            % array of MATLAB values.
            
            file = hashFile(obj, cache, hashName);
            data = struct;
            if exist(file,'file') == 2
                data = load(file,obj.NameVar);
            end  
            
            if isfield(data, obj.NameVar)
                % Get the list of keys.
                [found, pos] = ismember(name, data.(obj.NameVar));
                
                % Load all the data in the MAT file. Possibly requiring a
                % lot of time and space.
                v = load(file,obj.ValueVar);
                data.value = v.value;
                
                % Remove existing keys
                if any(found) 
                    count = count + sum(found);
                    goodbye = pos(found);
                    data.value(goodbye) = [];
                    data.name(goodbye) = [];
                    
                    % Write remaining names and values
                    save(file,'-struct','data','-v7.3');                    
                end
            end  
        end
        
        function count = hashSet(obj, cache, hashName, name, value)
        % hashSet Set name/value pairs in the named hash. 
        %   count = hashSet(obj, cache, hashName, name, value) sets the
        %   name/value pairs in name and value, which must be the same size,
        %   into the hash named hashName in the cache named cache.
 
            if obj.Stopped
                error(message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                    "MAT-File", obj.Folder));
            end
            count = 0;
            
            % cache and hashName must not be empty
            
            % name and value must be the same size. name must be a 1xN
            % string array or cell array of chars. value must be a cell
            % array of MATLAB values.
            
            file = hashFile(obj, cache, hashName);
            data = struct;
            if exist(file,'file') == 2
                data = load(file,obj.NameVar);
            end
            
            % If the hash exists, update it. 
            if isfield(data, obj.NameVar)
                % Get the list of keys.
                [found, locK] = ismember(name, data.(obj.NameVar));
                
                % Load all the data in the MAT file. Possibly requiring a
                % lot of time and space.
                v = load(file,obj.ValueVar);
                data.value = v.value;
                
                % Update existing keys
                if any(found) 
                    count = count + sum(found);
                    
                    % The keys exist, so we don't write the names into the
                    % cache. Instead, we update the value corresponding to
                    % each key.
                    posOld = locK(found);
                    posNew = find(found);
                    v = v.value;
                    for n=1:numel(posOld)
                        v(1,posOld(n)) = value(posNew(n));
                    end    
                    data.value(found) = v;
                    
                    % Remove the name/value pairs we used to update exiting
                    % keys.
                    new = ~found;
                    name = name(new);
                    value = value(new);
                end
                
                % Add new keys to the end of the name list and the new
                % values to the end of the value list. 
                if ~isempty(name)
                    sz = size(data.(obj.NameVar),2); % Number of columns
                    % For storage efficiency, esp. when growing these
                    % variables, store name and value as cell arrays.
                    n = data.(obj.NameVar);
                    v = data.(obj.ValueVar);
                    n(1,sz+1:sz+numel(name)) = name;
                    v(1,sz+1:sz+numel(value)) = value;
                    data.(obj.NameVar) = n;
                    data.(obj.ValueVar) = v;
                end
            else
                data.(obj.NameVar) = name;
                data.(obj.ValueVar) = value;
            end
            % Write all names and values
            save(file,'-struct','data','-v7.3');
        end
        
        function count = write(obj, cache, varargin)
        % write Write data to the named cache.
        %    write(mfp, cache, name1, value1, ...) stores the N values
        %    under their corresponding names in a MATLAB MAT-file.
            
            if obj.Stopped
                error(message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                    "MAT-File", obj.Folder));
            end
            
            % Validate varargin length -- must be even and 4 or greater.
            if mod(numel(varargin),2) ~= 0 || numel(varargin) < 2
                error(message('MPS:MATLAB:KVStore:BadInputsToWrite', ...
                    numel(varargin)+2));
            end
            
            % Extract keys and values into separate MATLAB arrays: keys
            % into an array of strings, values into a cell array.
            
            names = string(varargin(1:2:end-1));
            values = varargin(2:2:end);
        
            % Create a matlab.io.MatFile object to manipulate the MAT-file.
            % More efficient than loading/saving all the data at once.
            file = fullfile(obj.Folder, strcat(cache, obj.Ext));
            data = struct;
            if exist(file,'file') == 2
                data = load(file,obj.NameVar);
            end
            
            % write is an "append" operation. If a key exists, we must
            % update its value. If the key doesn't exist, we add it and its
            % value.
            
            % Update existing variables
            count = 0;
            if isfield(data, obj.NameVar)
                v = load(file,obj.ValueVar);
                v = v.value;
                [found, locK] = ismember(names, data.(obj.NameVar));
                if any(found)
                    posOld = locK(found);
                    v(posOld) = values(found);

                    % Remove existing variables from the input arrays
                    new = ~found;
                    names = names(new);
                    values = values(new);
                    
                    count = count + sum(found);
                end
                data.value = v;
            end
            
            % Add new variables -- if there are any (update might have done
            % all the work for us)
            if ~isempty(names)
                sz = 0;
                n = string.empty;
                v = {};
                if isfield(data, obj.NameVar)
                    sz = numel(data.(obj.NameVar));
                    n = data.(obj.NameVar);
                    v = data.(obj.ValueVar);
                end
                data.(obj.NameVar) = [ n names ];
                v(1,sz+1:sz+numel(names)) = values;
                data.(obj.ValueVar) = v;
            
                % Update count to reflect number of variables written.
                count = count + numel(names);
            end
            save(file,'-struct','data','-v7.3');
        end
        
        function varargout = read(obj, cache, varargin)
        % read Read named variables from the cache.
        %
        % data = read(obj, cache, v1, ... vN) reads the values of named
        % variables v1 through vN from the cache, returning their values in
        % the cell array data. 
        %
        % If a variable name is valid, but the variable does not exist,
        % read returns [] (empty numeric array) in the corresponding
        % position.
        
            if obj.Stopped
                error(message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                    "MAT-File", obj.Folder));
            end
            
            file = fullfile(obj.Folder, strcat(cache, obj.Ext));
            
            % Extract the variable names.
            if exist(file,'file') ~= 2
                varargout = cell(1,numel(varargin));
                return;
            end
            
            data = struct;
            if exist(file,'file') == 2
                data = load(file,obj.NameVar);
            end
            
            variables = string(varargin);
            
            % Find existing variables -- remove all zeros from pos so we
            % can use it as an index into value. (MATLAB doesn't like zero
            % as an index.)
            [found, pos] = ismember(variables, data.(obj.NameVar));
            pos = pos(found);
            
            % Tolerate non-existent variables -- return [].
            v = load(file,obj.ValueVar);
            varargout = cell(1,numel(varargin));
            if any(found)
                varargout(found) = v.value(pos);
            end
        end
        
        function removed = remove(obj, cache, varargin)
        % remove Remove named variables from the cache.
        %   
        % n = remove(obj, cache, v1, v2, ..., vN) removes the N variables
        % v1 ... vN from the given cache, returning the number of variables
        % removed. 
        %
        % It is an error if the cache does not exist. But non-existent
        % variables are treated tolerantly, and do not cause an error.
        
            if obj.Stopped
                error(message('MPS:MATLAB:Provider:FolderServiceStopped', ...
                    "MAT-File", obj.Folder));
            end
            
            removed = 0;
            % cache name may be a scalar string or a char vector. 
            file = fullfile(obj.Folder, strcat(cache, obj.Ext));
 
            % Load all the variables into a structure. Remove the
            % appropriate fields from the structure. Save the structure
            % back into the MAT-file, truncating it so only the variables
            % in the structure remain.
            if exist(file,'file') ~= 2
                removed = int64(0);
                return;
            end
            
            data = load(file,obj.NameVar);
            if isfield(data, obj.NameVar)
                names = data.(obj.NameVar);

                variables = varargin;
                % ismember will fail on { ["name"] } -- that is a cell
                % array of strings. In that case, convert to a string
                % matrix.
                if isstring(variables{1})
                    variables = string(variables);
                end

                % Find existing variables -- remove the ones we find.
                [found, pos] = ismember(variables, names);
                if ~isempty(pos)
                    v = load(file,obj.ValueVar);
                    v = v.(obj.ValueVar);
                    % We'll keep all the variables that we didn't find --
                    % that is, all the existing variables that the caller
                    % didn't ask us to remove.
                    removed = sum(found);
                    goodbye = pos(found);
                    names(goodbye) = [];
                    v(goodbye) = [];
                    data.(obj.NameVar) = names;
                    data.(obj.ValueVar) = v;
                    save(file,'-struct','data','-v7.3');
                end
            end
            removed = int64(removed);
       end
        
    end
    
    methods (Access = protected)
        
        function file = hashFile(obj, cacheName, hashName)
        % Create the name of the file that stores hashed composite keys.
        % The name won't conflict with any cache names because it a)
        % contains a cache name and b) starts with an invalid MATLAB
        % identifier character (and all cache names must be valid MATLAB
        % identifiers).
            file = fullfile(obj.Folder, ...
              strcat(obj.HashPrefix, hashName, '_', cacheName, obj.Ext));
        end
        
        function readConfiguration(obj)
        % (Obviously). Read a configuration file. Configuration files
        % are MAT-files that store MATLAB variables. The names of the
        % variables are names of configuration options. Load the variables
        % into the MATFileService's Config structure.
            cfile = fullfile(obj.Folder, strcat(obj.ConfigFile, obj.Ext));
            obj.Config = struct();
            if exist(cfile,'file') == 2
                obj.Config = load(cfile);
            end
        end
        
        function writeConfiguration(obj)
        % Save MATFileService configuration to a MAT-file in the MATFile
        % service's folder. We know we won't overwrite any caches because
        % the name of the configuration file starts with an invalid MATLAB
        % identifier character, and all cache names must be valid MATLAB
        % identifiers.
        %
        % Save the configuration structure's fields into the MAT-file as
        % separate variables. 
            cfile = fullfile(obj.Folder, strcat(obj.ConfigFile, obj.Ext));
            % Awkward. But that's the API.
            config = obj.Config;
            save(cfile,'-struct','config');
        end
    end
    
end

