%% Specify Custom Tokens
% The tokenizedDocument function, by default, splits words and tokens that contain symbols. For example, the function splits "C++" and "C#" into multiple tokens.

str = "I am experienced in MATLAB, C++, and C#.";
documents = tokenizedDocument(str)

% To prevent the function from splitting tokens that contain symbols, specify custom tokens using the 'CustomTokens' option.

documents = tokenizedDocument(str,'CustomTokens',["C++" "C#"])

% The custom tokens have token type "custom". View the token details. The column Type contains the token types.

tdetails = tokenDetails(documents)

% To specify your own token types, input the custom tokens as a table with the tokens in a column named Token, and the types in a column named Type. To assign a custom type to a token that doesn't include symbols, include in the table too. For example, create a table that will assign "MATLAB", "C++", and "C#" to the "programming-language" token type.

T = table;
T.Token = ["MATLAB" "C++" "C#"]';
T.Type = ["programming-language" "programming-language" "programming-language"]'

% Tokenize the text using the table of custom tokens and view the token details.

documents = tokenizedDocument(str,'CustomTokens',T);
tdetails = tokenDetails(documents)

% Copyright 2018 The MathWorks, Inc.