%% Obtain Multi-Output Q-Value Function Estimates
% Create an environment and obtain observation and action information.

env = rlPredefinedEnv('CartPole-Discrete');
obsInfo = getObservationInfo(env);
actInfo = getActionInfo(env);
numObs = obsInfo.Dimension(1);
numDiscreteAct = numel(actInfo.Elements);

% Create a deep neural network for a multi-output Q-value function representation.

criticNetwork = [
    featureInputLayer(4,'Normalization','none','Name','state')
    fullyConnectedLayer(50, 'Name', 'CriticStateFC1')
    reluLayer('Name','CriticRelu1')
    fullyConnectedLayer(20,'Name','CriticStateFC2')
    reluLayer('Name','CriticRelu2')
    fullyConnectedLayer(numDiscreteAct,'Name','output')];

% Create a representation for your critic using the recurrent neural network.

criticOptions = rlRepresentationOptions('LearnRate',1e-3,'GradientThreshold',1);
critic = rlQValueRepresentation(criticNetwork,obsInfo,actInfo,...
    'Observation','state',criticOptions);

% Obtain value function estimates for each possible discrete action using random observations.

val = getValue(critic,{rand(4,1)})

% val contains two value function estimates, one for each possible discrete action.
% You can also obtain value function estimates for a batch of observations. For example, obtain value function estimates for a batch of 10 observations.

batchVal = getValue(critic,{rand(4,1,10)});
size(batchVal)

% batchVal contains two value function estimates for each observation in the batch.
% Copyright 2019 The MathWorks, Inc.
%