完成期末练习

q2_1_crawler/q2_1.py

@@ -0,0 +1,48 @@
+import requests
+from bs4 import BeautifulSoup as bs
+import json
+
+url = 'https://exam.detr.top/exam-b/movies'
+headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36 Edg/149.0.0.0',
+    'Referer':'https://exam.detr.top/exam-b/movies'}
+req = requests.get(url, headers=headers)
+req.encoding="utf-8"
+
+data=[]
+
+soup=bs(req.text,"html.parser")
+# print(soup)
+#id, title, director, year, rating, duration, genre, actors-count
+
+items=soup.select('.item-row')
+# movie_list=[]
+for i in range(len(items)):
+    id=items[i].find("td",class_="item-id").get_text()
+    title=items[i].find("td",class_="item-title").get_text().strip()
+    director=items[i].find("td",class_="item-director").get_text().strip()
+    year=items[i].find("td",class_="item-year").get_text()
+    rating=items[i].find("td",class_="item-rating").get_text()
+    duration=items[i].find("td",class_="item-duration").get_text()
+    genre=items[i].find("td",class_="item-genre").get_text()
+    actors_count=items[i].find("td",class_="item-actors-count").get_text()
+
+    data.append({
+        "id":id,
+        "title":title,
+        "director":director,
+        "year":year,
+        "rating":rating,
+        "duration":duration,
+        "genre":genre,
+        "actors_count":actors_count
+        
+    })
+
+print(data)
+
+
+with open('movie.json', 'w', encoding='utf-8') as f:
+    json.dump(data, f, ensure_ascii=False, indent=2)
+
+with open("movie.html","w",encoding='utf-8') as f:
+     json.dump(data, f, ensure_ascii=False, indent=2)