finish

q2_1_crawler/q2_1.py

@@ -1,46 +1,31 @@
+import re
 import requests
+from bs4 import BeautifulSoup as bs
 import json
-from bs4 import BeautifulSoup
 
-headers = {
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-}
+header = {'User-Agent':'Mozilla/5.0'}
+url = 'https://exam.detr.top/exam-b/movies'
 
-url = "https://exam.detr.top/exam-b/movies"
+html = requests.get(url, headers=header).text
+# print(html)
+open('movies.html','w',encoding='utf-8').write(html)
+print(html)
 
-try:
-    response = requests.get(url, headers=headers)
-    response.encoding = 'utf-8'
-    html_content = response.text
-    with open('movies.html', 'w', encoding='utf-8') as f_html:
-        f_html.write(html_content)
-    soup = BeautifulSoup(html_content, 'html.parser')
-    table = soup.find('table')
-    tbody = table.find('tbody')
-    rows = tbody.find_all('tr')
+print("============")
+fd = bs(html, 'html.parser').find('p', class_='meta')
+fd = re.search(r'exam_fingerprint:\s*(\S+)', html).group(1)
+print(fd)
 
-    movies_data = []
+resp = bs(html, 'html.parser').find_all('tr', class_='item-row')
+# print(resp)
+movies = []
 
-    for row in rows:
-        tds = row.find_all('td')
-        if len(tds) >= 8:
-            movie = {
-                "id": tds[0].text.strip(),
-                "title": tds[1].text.strip(),
-                "director": tds[2].text.strip(),
-                "year": int(tds[3].text.strip()),
-                "rating": float(tds[4].text.strip()),
-                "duration": int(tds[5].text.strip()),
-                "genre": tds[6].text.strip(),
-                "actors_count": int(tds[7].text.strip())
-            }
-            movies_data.append(movie)
+for i in resp:
+    c = []
+    for r in i.find_all('td'):
+        c.append(r.text.strip())
+    movies.append({
+        'id':int(c[0]), 'title':c[1], 'director':c[2], 'year':int(c[3]), 'rating':float(c[4]), 'duration':int(c[5]), 'genre':c[6], 'actors_count':int(c[7])
+    })
 
-    with open('movies.json', 'w', encoding='utf-8') as f_json:
-        json.dump(movies_data, f_json, ensure_ascii=False, indent=4)
-
-    print(f"爬取成功！共获取 {len(movies_data)} 条电影数据。")
-    print("文件 movies.html 和 movies.json 已保存。")
-
-except Exception as e:
-    print(f"爬取或解析失败，错误信息：{e}")
+json.dump({"ID":fd, 'movies':movies}, open('movies.json', 'w', encoding='utf-8'), ensure_ascii=False, indent=2)

q4/q4_3.py

@@ -0,0 +1,15 @@
+import json
+import matplotlib.pyplot as plt
+
+movies = json.load(open('movies.json', encoding='utf-8'))['movies']
+# print(movies)
+
+ratings = []
+durations = []
+for m in movies:
+    ratings.append(m['rating'])
+    durations.append(m['duration'])
+
+plt.figure(figsize=(8,5))
+plt.hist(ratings, bins=5,color='blue')
+plt.savefig('q4_3a_hist.png')