%% Specify Custom Tokens Using Regular Expressions
% The tokenizedDocument function, by default, splits words and tokens containing symbols. For example, the function splits the text "ver:2" into multiple tokens.

str = "Upgraded to ver:2 rev:3.";
documents = tokenizedDocument(str)

% To prevent the function from splitting tokens that have particular patterns, specify those patterns using the 'RegularExpressions' option.
% Specify regular expressions to detect tokens denoting version and revision numbers: strings of digits appearing after "ver:" and "rev:" respectively.

documents = tokenizedDocument(str,'RegularExpressions',["ver:\d+" "rev:\d+"])

% Custom tokens, by default, have token type "custom". View the token details. The column Type contains the token types.

tdetails = tokenDetails(documents)

% To specify your own token types, input the regular expressions as a table with the regular expressions in a column named Pattern and the token types in a column named Type.

T = table;
T.Pattern = ["ver:\d+" "rev:\d+"]';
T.Type = ["version" "revision"]'

% Tokenize the text using the table of custom tokens and view the token details.

documents = tokenizedDocument(str,'RegularExpressions',T);
tdetails = tokenDetails(documents)

% Copyright 2018 The MathWorks, Inc.