diff --git a/q2_1_crawler/crawl.py b/q2_1_crawler/crawl.py new file mode 100644 index 0000000..8c30806 --- /dev/null +++ b/q2_1_crawler/crawl.py @@ -0,0 +1,45 @@ +import requests +from bs4 import BeautifulSoup +import json + +# 目标网址与请求头(题目强制要求请求头) +url = "https://exam.detr.top/exam-b-movies" +headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +} + +# 一次性请求获取全部网页数据 +resp = requests.get(url, headers=headers) +resp.encoding = resp.apparent_encoding +html_text = resp.text + +# 保存原始网页到data/movies.html +with open("./data/movies.html", "w", encoding="utf-8") as f: + f.write(html_text) + +# 解析表格电影数据 +soup = BeautifulSoup(html_text, "html.parser") +movie_rows = soup.select("table tr")[1:] # 跳过表头 +movies = [] +data_id = "movie_2026" + +for row in movie_rows: + tds = row.find_all("td") + movie_item = { + "id": tds[0].text.strip(), + "title": tds[0].text.strip(), + "director": tds[1].text.strip(), + "year": int(tds[2].text.strip()), + "rating": float(tds[3].text.strip()), + "duration": int(tds[4].text.strip()), + "genre": tds[5].text.strip(), + "actors_count": int(tds[6].text.strip()) + } + movies.append(movie_item) + +# 保存json文件到data目录 +out = {"data_id": data_id, "movies": movies} +with open("./data/movies.json", "w", encoding="utf-8") as f: + json.dump(out, f, ensure_ascii=False, indent=2) + +print("文件生成完毕") \ No newline at end of file