lil_matrix is a SciPy sparse-matrix representation for building and modifying mostly-zero matrices efficiently. It fits co-occurrence analysis, where most word pairs never appear together and storing every zero wastes memory and computation. section 5.4.2
Co-occurrence analysis
Replace a dense NumPy matrix with lil_matrix((n, n), dtype=np.float64) while counting words within a five-token window. The workflow can then focus visualization on the 30 most common co-occurring pairs and translate common words into English. section 5.4.2
Listing 5.3 Code snippet for co-occurrence analysis
# Preprocessing function
def preprocess(text):
text = re.sub(r'[^\w\s]', '', text).lower()
words = text.split()
words = [word for word in words if word not in
stopwords.words('portuguese')]
return words
# Co-occurrence function
def co_occurrence_matrix(corpus, window_size=5):
vocab = set(corpus)
vocab = list(vocab)
vocab_index = {word: i for i, word in enumerate(vocab)}
n = len(vocab)
co_occurrence_matrix = np.zeros((n, n))
for row in data['review_comment_message']:
tokens = preprocess(row)
for i, word in enumerate(tokens):
for j in range(max(i - window_size, 0), min(i + window_size +
1, len(tokens))):
if i != j:
co_occurrence_matrix[vocab_index[word],
vocab_index[tokens[j]]] += 1
return co_occurrence_matrix, vocab_index
# Calculate co-occurrence matrix
corpus = [word for row in data['review_comment_message'] for word in
preprocess(row)]
co_matrix, vocab_index = co_occurrence_matrix(corpus)
# Visualize the co-occurrence matrix using a heatmap
plt.figure(figsize=(15, 15))
sns.heatmap(co_matrix, xticklabels=vocab_index.keys(),
yticklabels=vocab_index.keys(), cmap="YlGnBu")
plt.show()
Compare the original dense co-occurrence implementation with the sparse replacement to see which allocation and counting steps require optimization. section 5.4.2
Listing 5.4 A more optimal code snippet for co-occurrence analysis
<SOME IMPORTS HERE>
from scipy.sparse import lil_matrix
<DATA LOADING>
# Preprocessing function
def preprocess(text):
text = re.sub(r'[^\w\s]', '', text).lower()
words = text.split()
words = [word for word in words if word not in stopwords_set]
return words
# Co-occurrence function
def co_occurrence_matrix(corpus, vocab, window_size=5):
vocab_index = {word: i for i, word in enumerate(vocab)}
n = len(vocab)
co_occurrence_matrix = lil_matrix((n, n), dtype=np.float64)
for row in data['review_comment_message']:
tokens = preprocess(row)
for i, word in enumerate(tokens):
for j in range(max(i - window_size, 0), min(i + window_size +
1, len(tokens))):
if i != j and word in vocab_index and tokens[j] in
vocab_index:
co_occurrence_matrix[vocab_index[word],
vocab_index[tokens[j]]] += 1
return co_occurrence_matrix, vocab_index
# Calculate co-occurrence matrix
corpus = [word for row in data['review_comment_message'] for word in
preprocess(row)]
word_counts = Counter(corpus)
top_n_words = 30 # Adjust this number according to your needs
most_common_words = [word for word, count in
word_counts.most_common(top_n_words)]
co_matrix, vocab_index = co_occurrence_matrix(corpus, most_common_words)
# English translations of the words (this part was added by ChatGPT when we
asked directly for translations)
translations = {
'produto': 'product',
'prazo': 'deadline',
<MORE TRANSLATIONS HERE>
}
# Create a list of translated words for visualization
translated_words = [translations.get(word, word) for word in
vocab_index.keys()]
# Visualize the co-occurrence matrix using a heatmap
plt.figure(figsize=(15, 15))
sns.heatmap(co_matrix.toarray(), xticklabels=translated_words,
yticklabels=translated_words, cmap='coolwarm', annot=False)
plt.title("Co-occurrence Matrix Heatmap with English Translations")
plt.xlabel("Words")
plt.ylabel("Words")
plt.show()
Inspect the optimized workflow, including the SciPy import, sparse allocation, vocabulary indexing, preprocessing, and co-occurrence counting. section 5.4.2
