diff --git a/数据处理与标注.py b/数据处理与标注.py new file mode 100644 index 0000000..4676eff --- /dev/null +++ b/数据处理与标注.py @@ -0,0 +1,25 @@ +import json + +# 1. 读取原始数据 +with open("movies.json", "r", encoding="utf-8") as f: + movies = json.load(f) + +# 2. 过滤掉quote为空的条目 +filtered_data = [ + { + "id": movie["rank"], + "title": movie["title"], + "quote": movie["quote"] + } + for movie in movies + if movie["quote"].strip() != "" +] + +print(f"过滤前:{len(movies)} 条") +print(f"过滤后:{len(filtered_data)} 条") + +# 3. 保存为Label Studio可导入的格式 +with open("filtered_quotes.json", "w", encoding="utf-8") as f: + json.dump(filtered_data, f, ensure_ascii=False, indent=4) + +print("✅ 已生成 filtered_quotes.json,可直接导入Label Studio") \ No newline at end of file diff --git a/数据采集.py b/数据采集.py new file mode 100644 index 0000000..21bfbe2 --- /dev/null +++ b/数据采集.py @@ -0,0 +1,65 @@ +import requests +from bs4 import BeautifulSoup +import json +import time + +headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36", + "Referer": "https://movie.douban.com/top250" +} + +movies_data = [] + +# 豆瓣Top250前50条,需要爬2页(start=0和start=25) +for start in range(0, 50, 25): + url = f"https://movie.douban.com/top250?start={start}" + print(f"正在爬取第 {start//25 + 1} 页...") + + # 发送请求 + response = requests.get(url, headers=headers) + response.raise_for_status() # 请求失败自动报错 + soup = BeautifulSoup(response.text, "html.parser") + + # 提取所有电影条目 + movie_items = soup.find_all("div", class_="item") + + for idx, item in enumerate(movie_items): + rank = start + idx + 1 + + # 1. 电影名称 + title_tag = item.find("span", class_="title") + title = title_tag.text.strip() if title_tag else "" + + # 2. 主演信息 + actors = "" + bd_div = item.find("div", class_="bd") + if bd_div: + p_tag = bd_div.find("p") + if p_tag: + info_text = p_tag.text.strip() + if "主演:" in info_text: + actors_part = info_text.split("主演:")[-1].strip() + actors = actors_part.split("\n")[0].strip() + + # 3. 短评(适配当前页面的inq标签) + quote_tag = item.find("span", class_="inq") + quote = quote_tag.text.strip() if quote_tag else "" + + # 构造数据 + movie_info = { + "rank": rank, + "title": title, + "actors": actors, + "quote": quote + } + movies_data.append(movie_info) + print(f"已获取:{rank}. {title} | 短评:{quote if quote else '无'}") + + # 每页爬完暂停2秒,避免请求过快被封 + time.sleep(2) + +# 保存到movies.json +with open("movies.json", "w", encoding="utf-8") as f: + json.dump(movies_data, f, ensure_ascii=False, indent=4) + +print("\n✅ 爬取完成!数据已保存到 movies.json") \ No newline at end of file