26 lines
958 B
Python
26 lines
958 B
Python
import json
|
|
|
|
# 读取原始爬虫数据
|
|
with open("movies.json", "r", encoding="utf-8") as f:
|
|
raw_movies = json.load(f)
|
|
|
|
# 过滤quote为空的样本
|
|
valid_movies = [movie for movie in raw_movies if movie["quote"].strip()]
|
|
|
|
# 1. 生成LabelStudio导入文件 quotes_processed.txt
|
|
with open("quotes_processed.txt", "w", encoding="utf-8") as out_f:
|
|
for item in valid_movies:
|
|
line = json.dumps({"text": item["quote"]}, ensure_ascii=False)
|
|
out_f.write(line + "\n")
|
|
|
|
# 2. 生成数据统计文件 process_stats.json
|
|
stats = {
|
|
"原始总样本": len(raw_movies),
|
|
"过滤后有效样本": len(valid_movies),
|
|
"过滤掉空短评样本": len(raw_movies) - len(valid_movies)
|
|
}
|
|
with open("process_stats.json", "w", encoding="utf-8") as f:
|
|
json.dump(stats, f, ensure_ascii=False, indent=2)
|
|
|
|
print("数据清洗完成，已生成 quotes_processed.txt、process_stats.json")
|
|
print(f"统计信息：{stats}") |