%% Calculate Document Log-Probabilities
% To reproduce the results in this example, set rng to 'default'.

rng('default')

% Load the example data. The file sonnetsPreprocessed.txt contains preprocessed versions of Shakespeare's sonnets. The file contains one sonnet per line, with words separated by a space. Extract the text from sonnetsPreprocessed.txt, split the text into documents at newline characters, and then tokenize the documents.

filename = "sonnetsPreprocessed.txt";
str = extractFileText(filename);
textData = split(str,newline);
documents = tokenizedDocument(textData);

% Create a bag-of-words model using bagOfWords.

bag = bagOfWords(documents)

% Fit an LDA model with 20 topics. To suppress verbose output, set 'Verbose' to 0.

numTopics = 20;
mdl = fitlda(bag,numTopics,'Verbose',0);

% Compute the document log-probabilities of the training documents and show them in a histogram.

logProbabilities = logp(mdl,documents);
figure
histogram(logProbabilities)
xlabel("Log Probability")
ylabel("Frequency")
title("Document Log-Probabilities")

% Identify the three documents with the lowest log-probability. A low log-probability may suggest that the document may be an outlier.

[~,idx] = sort(logProbabilities);
idx(1:3)
documents(idx(1:3))

% Copyright 2012 The MathWorks, Inc.