%% Use parfor to Train Multiple Deep Learning Networks
% This example shows how to use a |parfor| loop to perform a parameter
% sweep on a training option.
 
%%
% Deep learning training often takes hours or days, and searching for good
% training options can be difficult. With parallel computing, you can 
% speed up and automate your search for good models. If you have access to
% a machine with multiple graphical processing units (GPUs), you can complete this example on a local copy of
% the data set with a local parpool. If you want to use more resources, you
% can scale up deep learning training to the cloud. This example shows how
% to use a parfor loop to perform a parameter sweep on the training option
% |MiniBatchSize| in a cluster in the cloud. You can modify the script to
% do a parameter sweep on any other training option. Also, this example
% shows how to obtain feedback from the workers during computation using
% |DataQueue|. You can also send the script as a batch job to the cluster,
% so you can continue working or close MATLAB and fetch the results later.
%% Load the Data Set from the Cloud
% Load the training and test data sets from the cloud using |imageDatastore|.
% Split the training data set into training and validation sets, and keep the
% test data set to test the best network from the parameter sweep.
%%
imds = imageDatastore('s3://cifar10cloud/cifar10/train', ...
    'IncludeSubfolders',true, ...
    'LabelSource','foldernames');

imdsTest = imageDatastore('s3://cifar10cloud/cifar10/test', ...
    'IncludeSubfolders',true, ...
    'LabelSource','foldernames');

[imdsTrain,imdsValidation] = splitEachLabel(imds,0.9);
%%
% Train the network with augmented image data, by creating an
% |augmentedImageDatastore| object. Use random translations and horizontal
% reflections. Data augmentation helps prevent the network from overfitting
% and memorizing the exact details of the training images.


imageSize = [32 32 3];
pixelRange = [-4 4];
imageAugmenter = imageDataAugmenter( ...
    'RandXReflection',true, ...
    'RandXTranslation',pixelRange, ...
    'RandYTranslation',pixelRange);
augmentedImdsTrain = augmentedImageDatastore(imageSize,imdsTrain, ...
    'DataAugmentation',imageAugmenter, ...
    'OutputSizeMode','randcrop');
%% Define Network Architecture
% Define a network architecture for the CIFAR-10 data set. To simplify the code, use
% convolutional blocks that convolve the input. The pooling layers downsample the spatial dimensions.
%%
imageSize = [32 32 3];
netDepth = 2; % netDepth controls the depth of a convolutional block
netWidth = 16; % netWidth controls the number of filters in a convolutional block

layers = [
    imageInputLayer(imageSize)
    
    convolutionalBlock(netWidth,netDepth)
    maxPooling2dLayer(2,'Stride',2)
    convolutionalBlock(2*netWidth,netDepth)
    maxPooling2dLayer(2,'Stride',2)
    convolutionalBlock(4*netWidth,netDepth)
    averagePooling2dLayer(8)
    
    fullyConnectedLayer(10)
    softmaxLayer
    classificationLayer
    ];
%% Train Several Networks Simultaneously
% Specify the mini-batch sizes on which to do a parameter sweep. Allocate
% variables for the resulting networks and accuracy.
%%
miniBatchSizes = [64 128 256 512];
numMiniBatchSizes = numel(miniBatchSizes);
trainedNetworks = cell(numMiniBatchSizes,1);
accuracies = zeros(numMiniBatchSizes,1);
%%
% Perform a parallel parameter sweep training several networks inside a
% |parfor| loop and varying the mini-batch size. The workers in the cluster
% train the networks simultaneously and send the trained networks and
% accuracies back when the training is complete. If you want to check that the training
% is working, set |Verbose| to |true| in the training options. Note that the workers
% compute independently, so the command line output is not in the same
% sequential order as the iterations.
parfor idx = 1:numMiniBatchSizes
    
    miniBatchSize = miniBatchSizes(idx);
    initialLearnRate = 1e-1 * miniBatchSize/256; % Scale the learning rate according to the mini-batch size.
    
    % Define the training options. Set the mini-batch size.
    options = trainingOptions('sgdm', ...
        'MiniBatchSize',miniBatchSize, ... % Set the corresponding MiniBatchSize in the sweep.
        'Verbose',false, ... % Do not send command line output.
        'InitialLearnRate',initialLearnRate, ... % Set the scaled learning rate.
        'L2Regularization',1e-10, ...
        'MaxEpochs',30, ...
        'Shuffle','every-epoch', ...
        'ValidationData',imdsValidation, ...
        'LearnRateSchedule','piecewise', ...
        'LearnRateDropFactor',0.1, ...
        'LearnRateDropPeriod',25);
    
    % Train the network in a worker in the cluster.
    net = trainNetwork(augmentedImdsTrain,layers,options);
    
    % To obtain the accuracy of this network, use the trained network to
    % classify the validation images on the worker and compare the predicted labels to the
    % actual labels.
    YPredicted = classify(net,imdsValidation);
    accuracies(idx) = sum(YPredicted == imdsValidation.Labels)/numel(imdsValidation.Labels);
    
    % Send the trained network back to the client.
    trainedNetworks{idx} = net;
end
%%
% After |parfor| finishes, |trainedNetworks| contains the resulting
% networks trained by the workers. Display the trained networks and their
% accuracies.
trainedNetworks
accuracies
%%
% Select the best network in terms of accuracy. Test its performance
% against the test data set.
[~, I] = max(accuracies);
bestNetwork = trainedNetworks{I(1)};
YPredicted = classify(bestNetwork,imdsTest);
accuracy = sum(YPredicted == imdsTest.Labels)/numel(imdsTest.Labels)

%% Helper Functions
% Define a function to create a convolutional block in the
% network architecture.
%%
function layers = convolutionalBlock(numFilters,numConvLayers)
layers = [
    convolution2dLayer(3,numFilters,'Padding','same')
    batchNormalizationLayer
    reluLayer
    ];

layers = repmat(layers,numConvLayers,1);
end

%%
% Copyright 2018 The MathWorks, Inc.