%% Count Flights by Airline
% Use mapreduce to count the number of flights made by each unique airline carrier in a data set.
% Create a datastore using the airlinesmall.csv data set. This 12-megabyte data set contains 29 columns of flight information for several airline carriers, including arrival and departure times. In this example, select UniqueCarrier (airline name) as the variable of interest. Specify the 'TreatAsMissing' name-value pair so that the datastore treats 'NA' values as missing, and specify the 'MissingValue' name-value pair to replace missing values with zeros.

ds = tabularTextDatastore('airlinesmall.csv','TreatAsMissing','NA',...
    'MissingValue',0);
ds.SelectedVariableNames = 'UniqueCarrier';
ds.SelectedFormats = '%C';

%
% Preview the data.

preview(ds)

%
% Run mapreduce on the data. The map and reduce functions count the number of instances of each airline carrier name in each block of data, then combine those intermediate counts into a final count. This method leverages the intermediate sorting by unique key performed by mapreduce. The functions countMapper and countReducer are included at the end of this script.

outds = mapreduce(ds, @countMapper, @countReducer);
readall(outds)

%
% The map function countMapper leverages the fact that the data is categorical. The countcats and categories functions are used on each block of the input data to generate key/value pairs of the airline name and associated count.

function countMapper(data, info, intermKV)
% Counts unique airline carrier names in each block.
a = data.UniqueCarrier;
c = num2cell(countcats(a));
keys = categories(a);
addmulti(intermKV, keys, c)
end

% The reduce function countReducer reads in the intermediate data produced by the map function and adds together all of the counts to produce a single final count for each airline carrier.

function countReducer(key, intermValIter, outKV)
% Combines counts from all blocks to produce final counts.
count = 0;
while hasnext(intermValIter)
    data = getnext(intermValIter);
    count = count + data;
end
add(outKV, key, count)
end

% Copyright 2020 The MathWorks, Inc.