%% Weighted Standard Deviation and Variance of Tall Vectors
% Calculate weighted standard deviation and variance of a tall array using a vector of weights. This is one example of how you can use matlab.tall.reduce to work around functionality that tall arrays do not support yet.
%
% Create two tall vectors of random data. tX contains random data, and tP contains corresponding probabilities such that sum(tP) is 1. These probabilities are suitable to weight the data.

rng default
tX = tall(rand(1e4,1));
p = rand(1e4,1);
tP = tall(normalize(p,'scale',sum(p)));

%
% Write an identity function that returns outputs equal to the inputs. This approach skips the transform step of matlab.tall.reduce and passes the data directly to the reduction step, where the reduction function is repeatedly applied to reduce the size of the data.

type identityTransform.m

%
% Next, write a reduction function that operates on blocks of the tall vectors to calculate the weighted variance and standard deviation.

type weightedStats.m

%
% Use matlab.tall.reduce to apply these functions to the blocks of data in the tall vectors.

[tX_var_weighted, tX_std_weighted] = matlab.tall.reduce(@identityTransform, @weightedStats, tX, tP)

% Copyright 2018 The MathWorks, Inc.