上传文件至 q2_1_crawler

q2_1_crawler/q2_1.py

@@ -0,0 +1,56 @@
+import requests
+import json
+from bs4 import BeautifulSoup
+
+def crawl_movies():
+    url = "https://exam.detr.top/exam-b/movies"
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    }
+    
+    resp = requests.get(url, headers=headers)
+    resp.encoding = 'utf-8'  # 根据实际编码调整
+    html = resp.text
+    
+    # 保存原始网页源码
+    with open("movies.html", "w", encoding="utf-8") as f:
+        f.write(html)
+    
+    # 解析 HTML 提取数据
+    soup = BeautifulSoup(html, 'html.parser')
+    
+    # 示例：数据编号（请根据实际网页结构调整选择器）
+    data_id_elem = soup.find('span', id='data-id')
+    data_id = data_id_elem.text.strip() if data_id_elem else "unknown"
+    
+    # 示例：电影列表（请根据实际网页结构调整选择器）
+    movies = []
+    movie_elements = soup.find_all('div', class_='movie')  # 示例选择器
+    
+    for elem in movie_elements:
+        # 以下字段提取均为示例，需实际调整
+        movie = {
+            'id': elem.get('data-id') or elem.find('span', class_='id').text.strip(),
+            'title': elem.find('h3', class_='title').text.strip(),
+            'director': elem.find('span', class_='director').text.strip(),
+            'year': int(elem.find('span', class_='year').text.strip()),
+            'rating': float(elem.find('span', class_='rating').text.strip()),
+            'duration': int(elem.find('span', class_='duration').text.strip().replace(' min', '')),
+            'genre': [g.text.strip() for g in elem.find_all('span', class_='genre')],
+            'actors_count': int(elem.find('span', class_='actors_count').text.strip())
+        }
+        movies.append(movie)
+    
+    # 保存 JSON
+    result = {
+        "data_id": data_id,
+        "movies": movies
+    }
+    with open("movies.json", "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+    
+    print("爬取完成，已生成 movies.html 和 movies.json")
+    return result
+
+if __name__ == "__main__":
+    crawl_movies()