%% Try Multiple Pretrained Networks for Transfer Learning
% This example shows how to configure an experiment that replaces layers of
% different pretrained networks for transfer learning. Transfer learning is
% commonly used in deep learning applications. You can take a pretrained
% network and use it as a starting point to learn a new task. Fine-tuning a
% network with transfer learning is usually much faster and easier than
% training a network with randomly initialized weights from scratch. You
% can quickly transfer learned features to a new task using a smaller
% number of training images.
%
% There are many pretrained networks available in Deep Learning
% Toolbox(TM). These pretrained networks have different characteristics
% that matter when choosing a network to apply to your problem. The most
% important characteristics are network accuracy, speed, and size. Choosing
% a network is generally a tradeoff between these characteristics. To
% compare the performance of different pretrained networks for your task,
% edit this experiment and specify which pretrained networks to use.
%
% This experiment requires the Deep Learning Toolbox Model _for GoogLeNet
% Network_ support package and the Deep Learning Toolbox Model _for
% ResNet-18 Network_ support package. Before you run the experiment,
% install these support packages by calling the
% <docid:nnet_ref#mw_7cbd0577-4371-4eb3-829c-a9447220d89d |googlenet|> and
% <docid:nnet_ref#mw_591a2746-7267-4890-8390-87ae4dc7204c |resnet18|>
% functions and clicking the download links. For more information on other
% pretrained networks that you can download from the Add-On Explorer, see
% <docid:nnet_ug#bvf9ych-1 Pretrained Deep Neural Networks>.

% Copyright 2019-2021 The MathWorks, Inc.

%% Open Experiment
% First, open the example.
% *<docid:nnet_ref#mw_11ce840f-eed7-4975-9c28-baeebd79f092 Experiment
% Manager>* loads a project with a preconfigured experiment that you can
% inspect and run. To open the experiment, in the *Experiment Browser*
% pane, double-click the name of the experiment
% (|TransferLearningExperiment|).
%
% <<../exp_mgr_transfer_learning_config.png>>

% Project Name: 'FlowerTransferLearningProject'

%%
% Built-in training experiments consist of a description, a table of
% hyperparameters, a setup function, and a collection of metric functions
% to evaluate the results of the experiment. For more information, see
% <docid:nnet_ref#mw_92d8f99c-f283-4bc5-a5d7-7d779c4831f7 Configure Deep
% Learning Experiment>.
%
% The *Description* field contains a textual description of the experiment.
% For this example, the description is:
%
%  Perform transfer learning by replacing layers in a pretrained network.
%
% The *Hyperparameters* section specifies the strategy (|Exhaustive Sweep|)
% and hyperparameter values to use for the experiment. When you run the
% experiment, Experiment Manager trains the network using every combination
% of hyperparameter values specified in the hyperparameter table. In this
% example, the hyperparameter |NetworkName| specifies the network to train
% and the value of the training option |'miniBatchSize'|.
%
% The *Setup Function* configures the training data, network architecture,
% and training options for the experiment. The input to the setup function
% is a structure with fields from the hyperparameter table. The setup
% function returns three outputs that you use to train a network for image
% classification problems. In this example, the setup function:
%
% * Loads a pretrained network corresponding to the hyperparameter
% |NetworkName|.
%   
%   networkName = params.NetworkName;
%   
%   switch networkName
%       case "squeezenet"
%           net = squeezenet;
%           miniBatchSize = 128;
%       case "googlenet"
%           net = googlenet;
%           miniBatchSize = 128;
%       case "resnet18"
%           net = resnet18;
%           miniBatchSize = 128;
%       case "mobilenetv2"
%           net = mobilenetv2;
%           miniBatchSize = 128;
%       case "resnet50"
%           net = resnet50;
%           miniBatchSize = 128;
%       case "resnet101"
%           net = resnet101;
%           miniBatchSize = 64;
%       case "inceptionv3"
%           net = inceptionv3;
%           miniBatchSize = 64;
%       case "inceptionresnetv2"
%           net = inceptionresnetv2;
%           miniBatchSize = 64;
%       otherwise
%           error("Undefined network selection.");
%   end
%   
% * Downloads and extracts the Flowers data set, which is about 218 MB. For
% more information on this data set, see
% <docid:nnet_ug#mw_d3130147-29e6-47eb-b6f3-9a11a4c8b659 Image Data Sets>.
%   
%   url = "http://download.tensorflow.org/example_images/flower_photos.tgz";
%   downloadFolder = tempdir;
%   filename = fullfile(downloadFolder,"flower_dataset.tgz");
%   
%   imageFolder = fullfile(downloadFolder,"flower_photos");
%   if ~exist(imageFolder,"dir")
%       disp("Downloading Flower Dataset (218 MB)...")
%       websave(filename,url);
%       untar(filename,downloadFolder)
%   end
%   
%   imds = imageDatastore(imageFolder, ...
%       IncludeSubfolders=true, ...
%       LabelSource="foldernames");
%   
%   [imdsTrain,imdsValidation] = splitEachLabel(imds,0.9);
%   inputSize = net.Layers(1).InputSize;
%   augimdsTrain = augmentedImageDatastore(inputSize,imdsTrain);
%   augimdsValidation = augmentedImageDatastore(inputSize,imdsValidation);
%
% * Replaces the learnable layers of the pretrained network to perform
% transfer learning. The helper function |findLayersToReplace|, which is
% listed in <#6 Appendix 2> at the end of this example, determines the
% layers in the network architecture to replace for transfer learning. For
% more information on the available pretrained networks, see
% <docid:nnet_ug#bvf9ych-1 Pretrained Deep Neural Networks>.
%   
%   lgraph = layerGraph(net);  
%   [learnableLayer,classLayer] = findLayersToReplace(lgraph);  
%   numClasses = numel(categories(imdsTrain.Labels));
%   
%   if isa(learnableLayer,"nnet.cnn.layer.FullyConnectedLayer")
%       newLearnableLayer = fullyConnectedLayer(numClasses, ...
%           Name="new_fc", ...
%           WeightLearnRateFactor=10, ...
%           BiasLearnRateFactor=10);
%   elseif isa(learnableLayer,"nnet.cnn.layer.Convolution2DLayer")
%       newLearnableLayer = convolution2dLayer(1,numClasses, ...
%           Name="new_conv", ...
%           WeightLearnRateFactor=10, ...
%           BiasLearnRateFactor=10);
%   end
%   
%   lgraph = replaceLayer(lgraph,learnableLayer.Name,newLearnableLayer);
%   
%   newClassLayer = classificationLayer(Name="new_classoutput");
%   lgraph = replaceLayer(lgraph,classLayer.Name,newClassLayer);
% 
% * Defines a <docid:nnet_ref#bu59f0q trainingOptions> object for the
% experiment. The example trains the network for 10 epochs, using an
% initial learning rate of 0.0003 and validating the network every 5
% epochs.
%   
%   validationFrequencyEpochs = 5;
%   
%   numObservations = augimdsTrain.NumObservations;
%   numIterationsPerEpoch = floor(numObservations/miniBatchSize);
%   validationFrequency = validationFrequencyEpochs * numIterationsPerEpoch;
%   
%   options = trainingOptions("sgdm", ...
%       MaxEpochs=10, ...
%       MiniBatchSize=miniBatchSize, ...
%       InitialLearnRate=3e-4, ...
%       Shuffle="every-epoch", ...
%       ValidationData=augimdsValidation, ...
%       ValidationFrequency=validationFrequency, ...
%       Verbose=false);
% 
% To inspect the setup function, under *Setup Function*, click *Edit*. The
% setup function opens in MATLAB(R) Editor. In addition, the code for the
% setup function appears in <#5 Appendix 1> at the end of this example.
%
% The *Metrics* section specifies optional functions that evaluate the
% results of the experiment. This example does not include any custom
% metric functions.

%% Run Experiment
% When you run the experiment, Experiment Manager trains the network
% defined by the setup function six times. Each trial uses a different
% combination of hyperparameter values. By default, Experiment Manager runs
% one trial at a time. If you have Parallel Computing Toolbox(TM), you can
% run multiple trials at the same time. For best results, before you run
% your experiment, start a parallel pool with as many workers as GPUs. For
% more information, see
% <docid:nnet_ug#mw_4858679a-7447-4736-9c57-7b8dc706648d Use Experiment
% Manager to Train Networks in Parallel> and
% <docid:distcomp_ug#mw_57e04559-0b60-42d5-ad55-e77ec5f5865f GPU Support by
% Release>.
%
% * To run one trial of the experiment at a time, on the Experiment Manager
% toolstrip, click *Run*.
% * To run multiple trials at the same time, click *Use Parallel* and then
% *Run*. If there is no current parallel pool, Experiment Manager starts
% one using the default cluster profile. Experiment Manager then executes
% multiple simultaneous trials, depending on the number of parallel workers
% available.
%
% A table of results displays the accuracy and loss for each trial.
% While the experiment is running, click *Training Plot* to display the
% training plot and track the progress of each trial.
% Click *Confusion Matrix* to display the confusion matrix for the
% validation data in each completed trial.
%
% <<../exp_mgr_transfer_learning_results.png>>
%
% When the experiment finishes, you can sort the results table by column,
% filter trials by using the *Filters* pane, or record observations by
% adding annotations. For more information, see
% <docid:nnet_ref#mw_ffdb175d-cd27-4954-8fa6-e006dbb6dfd0 Sort, Filter, and
% Annotate Experiment Results>.
%
% To test the performance of an individual trial, export the trained
% network or the training information for the trial. On the *Experiment
% Manager* toolstrip, select *Export* > *Trained Network* or *Export* >
% *Training Information*, respectively. For more information, see
% <docid:nnet_ref#bu6sn4c-trainedNet Trained Network> and
% <docid:nnet_ref#bu6sn4c-traininfo Training Information>.

%% Close Experiment
% In the *Experiment Browser* pane, right-click the name of the project and
% select *Close Project*. Experiment Manager closes all of the experiments
% and results contained in the project.

%% Appendix 1: Setup Function
% This function configures the training data, network architecture, and
% training options for the experiment.
%
% *Input*
%
% * |params| is a structure with fields from the Experiment Manager
% hyperparameter table.
%
% *Output*
%
% * |augimdsTrain| is an augmented image datastore for the training data.
% * |lgraph| is a layer graph that defines the neural network architecture.
% * |options| is a |trainingOptions| object.
%
% <include>exp_mgr_transfer_learning_setup_function.m</include>

%% Appendix 2: Find Layers to Replace
% This function finds the single classification layer and the
% preceding learnable (fully connected or convolutional) layer of the layer
% graph |lgraph|.
%   
% <include>exp_mgr_findLayersToReplace.m</include>