%% Create Q-Value Function Critic from Deep Neural Network
% Create an observation specification object (or alternatively use getObservationInfo to extract the specification object from an environment). For this example, define the observation space as a continuous four-dimensional space, so that a single observation is a column vector containing four doubles.

obsInfo = rlNumericSpec([4 1]);

%
% Create an action specification object (or alternatively use getActionInfo to extract the specification object from an environment). For this example, define the action space as a continuous two-dimensional space, so that a single action is a column vector containing two doubles.

actInfo = rlNumericSpec([2 1]);

%
% Create a deep neural network to approximate the Q-value function. The network must have two inputs, one for the observation and one for the action. The observation input (here called myobs) must accept a four-element vector (the observation vector defined by obsInfo). The action input (here called myact)  must accept a two-element vector (the action vector defined by actInfo). The output of the network must be a scalar, representing the expected cumulative long-term reward when the agent starts from the given observation and takes the given action.

% observation path layers
obsPath = [featureInputLayer(4, 'Normalization','none','Name','myobs') 
    fullyConnectedLayer(1,'Name','obsout')];

% action path layers
actPath = [featureInputLayer(2, 'Normalization','none','Name','myact') 
    fullyConnectedLayer(1,'Name','actout')];

% common path to output layers
comPath = [additionLayer(2,'Name', 'add')  fullyConnectedLayer(1, 'Name', 'output')];

% add layers to network object
net = addLayers(layerGraph(obsPath),actPath); 
net = addLayers(net,comPath);

% connect layers
net = connectLayers(net,'obsout','add/in1');
net = connectLayers(net,'actout','add/in2');

% plot network
plot(net)

%
% Create the critic with rlQValueRepresentation, using the network, the observations and action specification objects, as well as the names of the network input layers.

critic = rlQValueRepresentation(net,obsInfo,actInfo, ...
    'Observation',{'myobs'},'Action',{'myact'})

%
% To check your critic, use the getValue function to return the value of a random observation and action, using the current network weights.

v = getValue(critic,{rand(4,1)},{rand(2,1)})

% You can now use the critic (along with an with an actor) to create an agent relying on a Q-value function critic (such as an rlQAgent, rlDQNAgent, rlSARSAAgent, or rlDDPGAgent agent).
% Copyright 2019 The MathWorks, Inc.