上传文件至 q2_1_crawler

q2_1_crawler/movies.html.py

@@ -0,0 +1,107 @@
+import requests
+import json
+from bs4 import BeautifulSoup
+
+# 题目强制要求请求头
+url = "https://exam.detr.top/exam-b/movies"
+headers = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+}
+
+# 仅一次请求，符合题目要求
+resp = requests.get(url, headers=headers)
+resp.encoding = "utf-8"
+html_text = resp.text
+
+# 保存原始网页 movies.html
+with open("movies.html", "w", encoding="utf-8") as f:
+    f.write(html_text)
+print("✅ 已保存网页源码 movies.html")
+
+# 优先尝试直接解析接口JSON（接口真实返回格式）
+movie_list = []
+data_id = None
+try:
+    api_data = json.loads(html_text)
+    data_id = api_data.get("data_id")
+    movie_list = api_data.get("movies", [])
+    print("✅ 识别为JSON接口，直接读取数据")
+except json.JSONDecodeError:
+    # 若为HTML表格页面，执行原bs4解析逻辑
+    print("识别为HTML表格页面，使用BeautifulSoup解析")
+    soup = BeautifulSoup(html_text, "html.parser")
+    # 提取页面data_id
+    if soup.body and "data-id" in soup.body.attrs:
+        data_id = soup.body["data-id"]
+    # 提取表格行
+    all_tr = soup.find_all("tr")
+    for tr in all_tr[1:]:
+        td_list = tr.find_all("td")
+        if len(td_list) >= 8:
+            # 增加类型转换容错
+            def safe_int(txt):
+                try:
+                    return int(txt.strip())
+                except:
+                    return 0
+            def safe_float(txt):
+                try:
+                    return float(txt.strip())
+                except:
+                    return 0.0
+            movie = {
+                "id": safe_int(td_list[0].get_text()),
+                "title": td_list[1].get_text(strip=True),
+                "director": td_list[2].get_text(strip=True),
+                "year": safe_int(td_list[3].get_text()),
+                "rating": safe_float(td_list[4].get_text()),
+                "duration": safe_int(td_list[5].get_text()),
+                "genre": td_list[6].get_text(strip=True),
+                "actors_count": safe_int(td_list[7].get_text())
+            }
+            movie_list.append(movie)
+
+print(f"页面data_id: {data_id}")
+print(f"一共抓取到 {len(movie_list)} 部电影")
+
+# 组装并保存 movies.json
+save_data = {
+    "data_id": data_id,
+    "movies": movie_list
+}
+with open("movies.json", "w", encoding="utf-8") as f:
+    json.dump(save_data, f, ensure_ascii=False, indent=2)
+print("✅ movies.json 写入完成")
+
+# ====================== 第2题数据分析 ======================
+if len(movie_list) == 0:
+    print("❌ 未抓取到任何电影数据，请检查网页返回内容！")
+else:
+    # ① 找出评分最高、最低电影
+    sorted_movies = sorted(movie_list, key=lambda x: x["rating"])
+    lowest_movie = sorted_movies[0]
+    highest_movie = sorted_movies[-1]
+    print("\n① 评分最高&最低电影：")
+    print(f"评分最低：{lowest_movie['title']} {lowest_movie['rating']}")
+    print(f"评分最高：{highest_movie['title']} {highest_movie['rating']}")
+
+    # ② 统计各类型电影数量（字典输出）
+    genre_count = {}
+    for m in movie_list:
+        g = m["genre"]
+        genre_count[g] = genre_count.get(g, 0) + 1
+    print("\n② 各类型电影数量：", genre_count)
+
+    # ③ 统计各导演电影数量（字典输出）
+    director_count = {}
+    for m in movie_list:
+        d = m["director"]
+        director_count[d] = director_count.get(d, 0) + 1
+    print("\n③ 各导演电影数量：", director_count)
+
+    # ④ 统计2020年（含）以后上映电影
+    count_2020 = 0
+    for m in movie_list:
+        if m["year"] >= 2020:
+            count_2020 += 1
+    print(f"\n④ 2020年(含)后上映电影总数：{count_2020}")