%% Create Continuous Actor-Critic Agent from Actor and Critic
% Create an environment with a continuous action space, and obtain its observation and action specifications. For this example, load the double integrator continuous action space environment used in the example Train DDPG Agent to Control Double Integrator System.

% load predefined environment
env = rlPredefinedEnv("DoubleIntegrator-Continuous");

% obtain observation specifications
obsInfo = getObservationInfo(env)
% obtain action specifications
actInfo = getActionInfo(env)

% In this example, the action is a scalar input representing a force ranging from -2 to 2 Newton, so it is a good idea to set the upper and lower limit of the action signal accordingly. This must be done when the network representation for the actor has an nonlinear output layer than needs to be scaled accordingly to produce an output in the desired range.

% make sure action space upper and lower limits are finite
actInfo.LowerLimit=-2;
actInfo.UpperLimit=2;

%
% The actor and critic networks are initialized randomly. You can ensure reproducibility by fixing the seed of the random generator. To do that, uncomment the following line.

% rng(0)

%
% Create a critic representation. Actor-critic agents use a value representation for the critic. For continuous observation spaces, you can use either a deep neural network or a custom basis representation. For this example, create a deep neural network as the underlying approximator.

% create the network to be used as approximator in the critic
% it must take the observation signal as input and produce a scalar value
criticNet = [
    imageInputLayer([obsInfo.Dimension 1],'Normalization','none','Name','state')
    fullyConnectedLayer(10,'Name', 'fc_in')
    reluLayer('Name', 'relu')
    fullyConnectedLayer(1,'Name','out')];

% set some training options for the critic
criticOpts = rlRepresentationOptions('LearnRate',8e-3,'GradientThreshold',1);

% create the critic representation from the network
critic = rlValueRepresentation(criticNet,obsInfo,'Observation',{'state'},criticOpts);

%
% Actor-critic agents use a stochastic actor representation. For stochastic actors operating in continuous action spaces, you can use only a deep neural network as the underlying approximator.
% The observation input (here called myobs) must accept a two-dimensional vector, as specified in obsInfo. The output (here called myact) must also be a two-dimensional vector (twice the number of dimensions specified in actInfo). The elements of the output vector represent, in sequence, all the means and all the standard deviations of every action (in this example, there is only one mean value and one standard deviation).
% The fact that standard deviations must be nonnegative while mean values must fall within the output range means that the network must have two separate paths. The first path is for the mean values, and any output nonlinearity must be scaled so that it can produce outputs in the output range. The second path is for the standard deviations, and you must use a softplus or ReLU layer to enforce nonnegativity.

% input path layers (2 by 1 input and a 1 by 1 output)
inPath = [ 
    imageInputLayer([obsInfo.Dimension 1], 'Normalization','none','Name','state')
    fullyConnectedLayer(10,'Name', 'ip_fc')  % 10 by 1 output
    reluLayer('Name', 'ip_relu')             % nonlinearity
    fullyConnectedLayer(1,'Name','ip_out') ];% 1 by 1 output

% path layers for mean value (1 by 1 input and 1 by 1 output)
% use scalingLayer to scale the range
meanPath = [
    fullyConnectedLayer(15,'Name', 'mp_fc1') % 15 by 1 output
    reluLayer('Name', 'mp_relu')             % nonlinearity
    fullyConnectedLayer(1,'Name','mp_fc2');  % 1 by 1 output
    tanhLayer('Name','tanh');                % output range: (-1,1)
    scalingLayer('Name','mp_out','Scale',actInfo.UpperLimit) ]; % output range: (-2N,2N)

% path layers for standard deviation (1 by 1 input and output)
% use softplus layer to make output non negative
sdevPath = [
    fullyConnectedLayer(15,'Name', 'vp_fc1') % 15 by 1 output
    reluLayer('Name', 'vp_relu')             % nonlinearity
    fullyConnectedLayer(1,'Name','vp_fc2');  % 1 by 1 output
    softplusLayer('Name', 'vp_out') ];       % output range: (0,+Inf)

% concatenate two inputs (along dimension #3) to form a single (2 by 1) output layer
outLayer = concatenationLayer(3,2,'Name','mean&sdev');

% add layers to layerGraph network object
actorNet = layerGraph(inPath);
actorNet = addLayers(actorNet,meanPath);
actorNet = addLayers(actorNet,sdevPath);
actorNet = addLayers(actorNet,outLayer);

% connect layers: the mean value path output must be connected to the first input of the concatenation layer
actorNet = connectLayers(actorNet,'ip_out','mp_fc1/in');   % connect output of inPath to meanPath input
actorNet = connectLayers(actorNet,'ip_out','vp_fc1/in');   % connect output of inPath to sdevPath input
actorNet = connectLayers(actorNet,'mp_out','mean&sdev/in1');% connect output of meanPath to mean&sdev input #1
actorNet = connectLayers(actorNet,'vp_out','mean&sdev/in2');% connect output of sdevPath to mean&sdev input #2

% plot network 
plot(actorNet)

%
% Specify some options for the actor and create the stochastic actor representation using the deep neural network actorNet.

% set some training options for the actor
actorOpts = rlRepresentationOptions('LearnRate',8e-3,'GradientThreshold',1);

% create the actor using the network
actor = rlStochasticActorRepresentation(actorNet,obsInfo,actInfo,...
    'Observation',{'state'},actorOpts);

%
% Specify agent options, and create an AC agent using actor, critic, and agent options.

agentOpts = rlACAgentOptions('NumStepsToLookAhead',32,'DiscountFactor',0.99);
agent = rlACAgent(actor,critic,agentOpts)

%
% To check your agent, use getAction to return the action from a random observation.

getAction(agent,{rand(2,1)})

% You can now test and train the agent within the environment.
% Copyright 2019 The MathWorks, Inc.
%