31 lines
964 B
Python
31 lines
964 B
Python
import jieba
|
||
|
||
print("=" * 50)
|
||
print("完整的文本预处理流程")
|
||
print("=" * 50)
|
||
|
||
docs = [
|
||
"今天天气真不错！适合出去玩。",
|
||
"Python是一门很棒的编程语言。",
|
||
"人工智能和机器学习是未来的发展方向。",
|
||
"今天在咖啡馆喝了一杯很好喝的拿铁。"
|
||
]
|
||
|
||
|
||
stopwords = set(['的', '了', '在', '是', '我', '有', '和', '就', '不', '人', '都', '一', '一个', '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有', '看', '好', '自己', '这', '！', '。', ','])
|
||
|
||
def preprocess_text(text):
|
||
"""完整的文本预处理流程"""
|
||
words = jieba.cut(text)
|
||
|
||
words = [w for w in words if w not in stopwords and len(w) > 0]
|
||
|
||
words = [w for w in words if w.strip()]
|
||
|
||
return words
|
||
|
||
print("预处理结果：")
|
||
for i, doc in enumerate(docs):
|
||
words = preprocess_text(doc)
|
||
print(f"\nDoc{i+1}: {doc}")
|
||
print(f" → {' / '.join(words)}") |