diff --git a/q2_1_crawler/q2_1.py b/q2_1_crawler/q2_1.py deleted file mode 100644 index 08f1cc1..0000000 --- a/q2_1_crawler/q2_1.py +++ /dev/null @@ -1,56 +0,0 @@ -import requests -import json -from bs4 import BeautifulSoup - -def crawl_movies(): - url = "https://exam.detr.top/exam-b/movies" - headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" - } - - resp = requests.get(url, headers=headers) - resp.encoding = 'utf-8' # 根据实际编码调整 - html = resp.text - - # 保存原始网页源码 - with open("movies.html", "w", encoding="utf-8") as f: - f.write(html) - - # 解析 HTML 提取数据 - soup = BeautifulSoup(html, 'html.parser') - - # 示例:数据编号(请根据实际网页结构调整选择器) - data_id_elem = soup.find('span', id='data-id') - data_id = data_id_elem.text.strip() if data_id_elem else "unknown" - - # 示例:电影列表(请根据实际网页结构调整选择器) - movies = [] - movie_elements = soup.find_all('div', class_='movie') # 示例选择器 - - for elem in movie_elements: - # 以下字段提取均为示例,需实际调整 - movie = { - 'id': elem.get('data-id') or elem.find('span', class_='id').text.strip(), - 'title': elem.find('h3', class_='title').text.strip(), - 'director': elem.find('span', class_='director').text.strip(), - 'year': int(elem.find('span', class_='year').text.strip()), - 'rating': float(elem.find('span', class_='rating').text.strip()), - 'duration': int(elem.find('span', class_='duration').text.strip().replace(' min', '')), - 'genre': [g.text.strip() for g in elem.find_all('span', class_='genre')], - 'actors_count': int(elem.find('span', class_='actors_count').text.strip()) - } - movies.append(movie) - - # 保存 JSON - result = { - "data_id": data_id, - "movies": movies - } - with open("movies.json", "w", encoding="utf-8") as f: - json.dump(result, f, ensure_ascii=False, indent=2) - - print("爬取完成,已生成 movies.html 和 movies.json") - return result - -if __name__ == "__main__": - crawl_movies() \ No newline at end of file