51 lines
1.8 KiB
Python
51 lines
1.8 KiB
Python
# q2_1_crawler/q2_1.py
|
||
import requests
|
||
import json
|
||
|
||
# 1. 请求网页，携带请求头防拦截
|
||
headers = {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||
}
|
||
url = "https://exam.detr.top/exam-b/movies"
|
||
resp = requests.get(url, headers=headers)
|
||
resp.raise_for_status() # 抛出请求异常
|
||
html_text = resp.text
|
||
|
||
# 2. 保存原始网页 movies.html
|
||
with open("movies.html", "w", encoding="utf-8") as f:
|
||
f.write(html_text)
|
||
|
||
# 3. 解析接口数据（网页内电影json数据）
|
||
# 页面返回的电影数据是json结构，直接提取页面内置数据（一次性获取全部10条）
|
||
data = json.loads(html_text) # 若页面是html内嵌json，改用bs4提取script标签，下方兼容备用方案
|
||
"""
|
||
# 备用：如果页面是html网页，用bs4提取json数据
|
||
from bs4 import BeautifulSoup
|
||
soup = BeautifulSoup(html_text, "html.parser")
|
||
script_tag = soup.find("script", id="movie-data")
|
||
data = json.loads(script_tag.string)
|
||
"""
|
||
|
||
# 4. 提取编号 + 电影信息，字段：id,title,director,year,rating,duration,genre,actors_count
|
||
result_data = {
|
||
"data_id": data["data_id"], # 网页数据编号
|
||
"movies": []
|
||
}
|
||
for movie in data["movie_list"]:
|
||
movie_item = {
|
||
"id": movie["id"],
|
||
"title": movie["title"],
|
||
"director": movie["director"],
|
||
"year": int(movie["year"]),
|
||
"rating": float(movie["rating"]),
|
||
"duration": int(movie["duration"]),
|
||
"genre": movie["genre"],
|
||
"actors_count": int(movie["actors_count"])
|
||
}
|
||
result_data["movies"].append(movie_item)
|
||
|
||
# 5. 保存 movies.json
|
||
with open("movies.json", "w", encoding="utf-8") as f:
|
||
json.dump(result_data, f, ensure_ascii=False, indent=2)
|
||
|
||
print("爬取完成：已生成 movies.html、movies.json") |