文本分析——分詞、統(tǒng)計詞頻、詞云

TNFE 發(fā)布于2019-07-30 15:34 / 1646人閱讀

摘要：導(dǎo)入包導(dǎo)入文件和數(shù)據(jù)數(shù)據(jù)預(yù)處理分詞并去除停用詞算法該類會將文本中的詞語轉(zhuǎn)換為詞頻矩陣，矩陣元素表示詞在類文本下的詞頻該類會統(tǒng)計每個詞語的權(quán)值第一個是計算，第二個是將文本轉(zhuǎn)為詞頻矩陣獲取詞袋模型中的所有詞語將矩陣抽取出來，元素表示

導(dǎo)入包

import os
import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import re
from pandas import Series, DataFrame  

import string
import re
import jieba
import jieba.analyse
import datetime
from wordcloud import WordCloud, ImageColorGenerator
import codecs

導(dǎo)入文件和數(shù)據(jù)

gongdan = pd.read_excel("Gongdan.xlsx")

數(shù)據(jù)預(yù)處理

gongdan["content"] = [str(i) for i in gongdan["content"]]
gongdan["content"] = ["".join(re.findall(u"[u4e00-u9fff]+", i)) for i in gongdan["content"]]
indexs = list(gongdan["content"][pd.isnull(gongdan["content"])].index)
gongdan = gongdan.drop(indexs)
indexs = list(gongdan["content"][gongdan["content"]==""].index)
gongdan = gongdan.drop(indexs)

content = gongdan["content"]

cont = "".join(content)
cont = "".join(re.findall(u"[u4e00-u9fa5]+", cont))

分詞并去除停用詞

stopwords = set()
fr = codecs.open("stopwords.txt", "r", "utf-8")
for word in fr:
   stopwords.add(str(word).strip())
fr.close()

jieba.load_userdict("dict.txt")
text = list(jieba.cut(cont, cut_all=False, HMM=True))
text = list(filter(lambda x: x not in stopwords, text))
text = [str(i) for i in text if i != " "]

Tfidf 算法

from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfTransformer  
from sklearn.feature_extraction.text import CountVectorizer  

test = " ".join(text)
tlist = []
tlist.append(test)

vectorizer=CountVectorizer()#該類會將文本中的詞語轉(zhuǎn)換為詞頻矩陣，矩陣元素a[i][j] 表示j詞在i類文本下的詞頻
transformer = TfidfTransformer()#該類會統(tǒng)計每個詞語的tf-idf權(quán)值
tfidf = transformer.fit_transform(vectorizer.fit_transform(tlist))  #第一個fit_transform是計算tf-idf，第二個fit_transform是將文本轉(zhuǎn)為詞頻矩陣  

word=vectorizer.get_feature_names()#獲取詞袋模型中的所有詞語  
weight=tfidf.toarray()#將tf-idf矩陣抽取出來，元素a[i][j]表示j詞在i類文本中的tf-idf權(quán)重  
tfidf_list = {}
for i in range(len(weight)):#打印每類文本的tf-idf詞語權(quán)重，第一個for遍歷所有文本，第二個for便利某一類文本下的詞語權(quán)重  
    for j in range(len(word)):  
        tfidf_list[word[j]] = weight[i][j]

詞云

font_path = "yahei.ttf"

from PIL import Image
back_coloring = np.array(Image.open("circle.jpg"))

wc = WordCloud(font_path=font_path,  # 設(shè)置字體
               background_color="white",  # 背景顏色
               max_words=60,  # 詞云顯示的最大詞數(shù)
               mask=back_coloring,  # 設(shè)置背景圖片
               stopwords=stopwords,
               max_font_size=100,  # 字體最大值
               random_state=42,
               width=1000, height=860, margin=2,# 設(shè)置圖片默認(rèn)的大小,但是如果使用背景圖片的話,那么保存的圖片大小將會按照其大小保存,margin為詞語邊緣距離
#               prefer_horizontal=1,
               )

wc.generate_from_frequencies(tfidf_list)

plt.imshow(wc, interpolation="bilinear")
plt.axis("off")
plt.figure()
wc.to_file("w.png")

# create coloring from image
image_colors = ImageColorGenerator(back_coloring)
# recolor wordcloud and show
# we could also give color_func=image_colors directly in the constructor
plt.imshow(wc.recolor(color_func=image_colors), interpolation="bilinear")
plt.axis("off")
plt.figure()
plt.imshow(back_coloring, cmap=plt.cm.gray, interpolation="bilinear")
plt.axis("off")
plt.show()