56 lines
2.1 KiB
Python
56 lines
2.1 KiB
Python
import requests
|
|
import json
|
|
from bs4 import BeautifulSoup
|
|
|
|
def crawl_movies():
|
|
url = "https://exam.detr.top/exam-b/movies"
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
|
}
|
|
|
|
resp = requests.get(url, headers=headers)
|
|
resp.encoding = 'utf-8' # 根据实际编码调整
|
|
html = resp.text
|
|
|
|
# 保存原始网页源码
|
|
with open("movies.html", "w", encoding="utf-8") as f:
|
|
f.write(html)
|
|
|
|
# 解析 HTML 提取数据
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
|
|
# 示例：数据编号（请根据实际网页结构调整选择器）
|
|
data_id_elem = soup.find('span', id='data-id')
|
|
data_id = data_id_elem.text.strip() if data_id_elem else "unknown"
|
|
|
|
# 示例：电影列表（请根据实际网页结构调整选择器）
|
|
movies = []
|
|
movie_elements = soup.find_all('div', class_='movie') # 示例选择器
|
|
|
|
for elem in movie_elements:
|
|
# 以下字段提取均为示例，需实际调整
|
|
movie = {
|
|
'id': elem.get('data-id') or elem.find('span', class_='id').text.strip(),
|
|
'title': elem.find('h3', class_='title').text.strip(),
|
|
'director': elem.find('span', class_='director').text.strip(),
|
|
'year': int(elem.find('span', class_='year').text.strip()),
|
|
'rating': float(elem.find('span', class_='rating').text.strip()),
|
|
'duration': int(elem.find('span', class_='duration').text.strip().replace(' min', '')),
|
|
'genre': [g.text.strip() for g in elem.find_all('span', class_='genre')],
|
|
'actors_count': int(elem.find('span', class_='actors_count').text.strip())
|
|
}
|
|
movies.append(movie)
|
|
|
|
# 保存 JSON
|
|
result = {
|
|
"data_id": data_id,
|
|
"movies": movies
|
|
}
|
|
with open("movies.json", "w", encoding="utf-8") as f:
|
|
json.dump(result, f, ensure_ascii=False, indent=2)
|
|
|
|
print("爬取完成，已生成 movies.html 和 movies.json")
|
|
return result
|
|
|
|
if __name__ == "__main__":
|
|
crawl_movies() |