# q2_1_crawler/q2_1.py
import requests
import json

# 1. 请求网页，携带请求头防拦截
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}
url = "https://exam.detr.top/exam-b/movies"
resp = requests.get(url, headers=headers)
resp.raise_for_status()  # 抛出请求异常
html_text = resp.text

# 2. 保存原始网页 movies.html
with open("movies.html", "w", encoding="utf-8") as f:
    f.write(html_text)

# 3. 解析接口数据（网页内电影json数据）
# 页面返回的电影数据是json结构，直接提取页面内置数据（一次性获取全部10条）
data = json.loads(html_text)  # 若页面是html内嵌json，改用bs4提取script标签，下方兼容备用方案
"""
# 备用：如果页面是html网页，用bs4提取json数据
from bs4 import BeautifulSoup
soup = BeautifulSoup(html_text, "html.parser")
script_tag = soup.find("script", id="movie-data")
data = json.loads(script_tag.string)
"""

# 4. 提取编号 + 电影信息，字段：id,title,director,year,rating,duration,genre,actors_count
result_data = {
    "data_id": data["data_id"],  # 网页数据编号
    "movies": []
}
for movie in data["movie_list"]:
    movie_item = {
        "id": movie["id"],
        "title": movie["title"],
        "director": movie["director"],
        "year": int(movie["year"]),
        "rating": float(movie["rating"]),
        "duration": int(movie["duration"]),
        "genre": movie["genre"],
        "actors_count": int(movie["actors_count"])
    }
    result_data["movies"].append(movie_item)

# 5. 保存 movies.json
with open("movies.json", "w", encoding="utf-8") as f:
    json.dump(result_data, f, ensure_ascii=False, indent=2)

print("爬取完成：已生成 movies.html、movies.json")