From 1dd56683aeb1f557cbe151eff3d4a1d348765dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=BF=9E=E5=85=B4=E6=9D=B0?= <2509165004@student.example.com> Date: Tue, 23 Jun 2026 11:24:11 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E6=96=87=E4=BB=B6=E8=87=B3?= =?UTF-8?q?=20q2=5F1=5Fcrawler?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- q2_1_crawler/movies.html | 152 ++++++++++++++++++++++++++++++++++++ q2_1_crawler/movies.html.py | 107 +++++++++++++++++++++++++ q2_1_crawler/movies.json | 105 +++++++++++++++++++++++++ 3 files changed, 364 insertions(+) create mode 100644 q2_1_crawler/movies.html create mode 100644 q2_1_crawler/movies.html.py create mode 100644 q2_1_crawler/movies.json diff --git a/q2_1_crawler/movies.html b/q2_1_crawler/movies.html new file mode 100644 index 0000000..d5ebd19 --- /dev/null +++ b/q2_1_crawler/movies.html @@ -0,0 +1,152 @@ + + + + + + + + 电影列表 + + + +

电影列表

+

数据编号：B-20260623-8659

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
编号电影名导演上映年份评分时长(分钟)类型主演数
1放牛班的春天Frank Darabont19988.3103喜剧3
2千与千寻陈凯歌20237.9118剧情3
3泰坦尼克号Robert Zemeckis20186.8164科幻2
4忠犬八公的故事James Cameron20127.6135爱情5
5阿甘正传宫崎骏20118.8128动画4
6霸王别姬Christopher Nolan20227.291悬疑5
7三傻大闹宝莱坞Lasse Hallström20099.2179爱情3
8肖申克的救赎Rajkumar Hirani19917.1175喜剧2
9盗梦空间Christophe Barratier20248.8144喜剧5
10星际穿越Christopher Nolan20088.9179动画3
+ + \ No newline at end of file diff --git a/q2_1_crawler/movies.html.py b/q2_1_crawler/movies.html.py new file mode 100644 index 0000000..0ac2ec1 --- /dev/null +++ b/q2_1_crawler/movies.html.py @@ -0,0 +1,107 @@ +import requests +import json +from bs4 import BeautifulSoup + +# 题目强制要求请求头 +url = "https://exam.detr.top/exam-b/movies" +headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +} + +# 仅一次请求，符合题目要求 +resp = requests.get(url, headers=headers) +resp.encoding = "utf-8" +html_text = resp.text + +# 保存原始网页 movies.html +with open("movies.html", "w", encoding="utf-8") as f: + f.write(html_text) +print("✅ 已保存网页源码 movies.html") + +# 优先尝试直接解析接口JSON（接口真实返回格式） +movie_list = [] +data_id = None +try: + api_data = json.loads(html_text) + data_id = api_data.get("data_id") + movie_list = api_data.get("movies", []) + print("✅ 识别为JSON接口，直接读取数据") +except json.JSONDecodeError: + # 若为HTML表格页面，执行原bs4解析逻辑 + print("识别为HTML表格页面，使用BeautifulSoup解析") + soup = BeautifulSoup(html_text, "html.parser") + # 提取页面data_id + if soup.body and "data-id" in soup.body.attrs: + data_id = soup.body["data-id"] + # 提取表格行 + all_tr = soup.find_all("tr") + for tr in all_tr[1:]: + td_list = tr.find_all("td") + if len(td_list) >= 8: + # 增加类型转换容错 + def safe_int(txt): + try: + return int(txt.strip()) + except: + return 0 + def safe_float(txt): + try: + return float(txt.strip()) + except: + return 0.0 + movie = { + "id": safe_int(td_list[0].get_text()), + "title": td_list[1].get_text(strip=True), + "director": td_list[2].get_text(strip=True), + "year": safe_int(td_list[3].get_text()), + "rating": safe_float(td_list[4].get_text()), + "duration": safe_int(td_list[5].get_text()), + "genre": td_list[6].get_text(strip=True), + "actors_count": safe_int(td_list[7].get_text()) + } + movie_list.append(movie) + +print(f"页面data_id: {data_id}") +print(f"一共抓取到 {len(movie_list)} 部电影") + +# 组装并保存 movies.json +save_data = { + "data_id": data_id, + "movies": movie_list +} +with open("movies.json", "w", encoding="utf-8") as f: + json.dump(save_data, f, ensure_ascii=False, indent=2) +print("✅ movies.json 写入完成") + +# ====================== 第2题数据分析 ====================== +if len(movie_list) == 0: + print("❌ 未抓取到任何电影数据，请检查网页返回内容！") +else: + # ① 找出评分最高、最低电影 + sorted_movies = sorted(movie_list, key=lambda x: x["rating"]) + lowest_movie = sorted_movies[0] + highest_movie = sorted_movies[-1] + print("\n① 评分最高&最低电影：") + print(f"评分最低：{lowest_movie['title']} {lowest_movie['rating']}") + print(f"评分最高：{highest_movie['title']} {highest_movie['rating']}") + + # ② 统计各类型电影数量（字典输出） + genre_count = {} + for m in movie_list: + g = m["genre"] + genre_count[g] = genre_count.get(g, 0) + 1 + print("\n② 各类型电影数量：", genre_count) + + # ③ 统计各导演电影数量（字典输出） + director_count = {} + for m in movie_list: + d = m["director"] + director_count[d] = director_count.get(d, 0) + 1 + print("\n③ 各导演电影数量：", director_count) + + # ④ 统计2020年（含）以后上映电影 + count_2020 = 0 + for m in movie_list: + if m["year"] >= 2020: + count_2020 += 1 + print(f"\n④ 2020年(含)后上映电影总数：{count_2020}") \ No newline at end of file diff --git a/q2_1_crawler/movies.json b/q2_1_crawler/movies.json new file mode 100644 index 0000000..f30ef7f --- /dev/null +++ b/q2_1_crawler/movies.json @@ -0,0 +1,105 @@ +{ + "data_id": null, + "movies": [ + { + "id": 1, + "title": "放牛班的春天", + "director": "Frank Darabont", + "year": 1998, + "rating": 8.3, + "duration": 103, + "genre": "喜剧", + "actors_count": 3 + }, + { + "id": 2, + "title": "千与千寻", + "director": "陈凯歌", + "year": 2023, + "rating": 7.9, + "duration": 118, + "genre": "剧情", + "actors_count": 3 + }, + { + "id": 3, + "title": "泰坦尼克号", + "director": "Robert Zemeckis", + "year": 2018, + "rating": 6.8, + "duration": 164, + "genre": "科幻", + "actors_count": 2 + }, + { + "id": 4, + "title": "忠犬八公的故事", + "director": "James Cameron", + "year": 2012, + "rating": 7.6, + "duration": 135, + "genre": "爱情", + "actors_count": 5 + }, + { + "id": 5, + "title": "阿甘正传", + "director": "宫崎骏", + "year": 2011, + "rating": 8.8, + "duration": 128, + "genre": "动画", + "actors_count": 4 + }, + { + "id": 6, + "title": "霸王别姬", + "director": "Christopher Nolan", + "year": 2022, + "rating": 7.2, + "duration": 91, + "genre": "悬疑", + "actors_count": 5 + }, + { + "id": 7, + "title": "三傻大闹宝莱坞", + "director": "Lasse Hallström", + "year": 2009, + "rating": 9.2, + "duration": 179, + "genre": "爱情", + "actors_count": 3 + }, + { + "id": 8, + "title": "肖申克的救赎", + "director": "Rajkumar Hirani", + "year": 1991, + "rating": 7.1, + "duration": 175, + "genre": "喜剧", + "actors_count": 2 + }, + { + "id": 9, + "title": "盗梦空间", + "director": "Christophe Barratier", + "year": 2024, + "rating": 8.8, + "duration": 144, + "genre": "喜剧", + "actors_count": 5 + }, + { + "id": 10, + "title": "星际穿越", + "director": "Christopher Nolan", + "year": 2008, + "rating": 8.9, + "duration": 179, + "genre": "动画", + "actors_count": 3 + } + ] +} \ No newline at end of file