正则表达式：爬虫内容提取利器
This commit is contained in:
2509165015
45
0402+2509165015.CSV
Normal file
45
0402+2509165015.CSV
Normal file
@@ -0,0 +1,45 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import csv
|
||||
import time
|
||||
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
||||
movies = []
|
||||
|
||||
for start in range(0, 250, 25):
|
||||
url = f"https://movie.douban.com/top250?start={start}"
|
||||
res = requests.get(url, headers=headers)
|
||||
soup = BeautifulSoup(res.text, "html.parser")
|
||||
items = soup.find_all("div", class_="item")
|
||||
|
||||
for item in items:
|
||||
rank = item.find("em").text
|
||||
title = item.find("span", class_="title").text
|
||||
rating = item.find("span", class_="rating_num").text
|
||||
people = item.find("div", class_="star").find_all("span")[-1].text.replace("人评价", "")
|
||||
quote = item.find("span", class_="inq").text if item.find("span", class_="inq") else "无"
|
||||
info = item.find("p", class_="").text.strip().split("\n")
|
||||
line1 = info[0].strip()
|
||||
line2 = info[1].strip() if len(info) > 1 else ""
|
||||
|
||||
director = line1.split("导演: ")[1].split("主演: ")[0].strip() if "导演: " in line1 else "未知"
|
||||
actor = line1.split("主演: ")[1].strip() if "主演: " in line1 else "未知"
|
||||
parts = line2.split("/") if line2 else []
|
||||
year = parts[0].strip() if len(parts) >= 1 else "未知"
|
||||
area = parts[1].strip() if len(parts) >= 2 else "未知"
|
||||
genre = parts[2].strip() if len(parts) >= 3 else "未知"
|
||||
|
||||
movies.append({
|
||||
"排名": rank, "电影名": title, "评分": rating, "评价人数": people, "经典台词": quote,
|
||||
"导演": director, "主演": actor, "年份": year, "地区": area, "类型": genre
|
||||
})
|
||||
time.sleep(1)
|
||||
print(f"已爬取 {start + 25} 条")
|
||||
with open("douban_top250.csv", "w", encoding="utf-8", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=movies[0].keys())
|
||||
writer.writeheader()
|
||||
writer.writerows(movies)
|
||||
print("✅ CSV 导出完成")
|
||||
|
Can't render this file because it contains an unexpected character in line 7 and column 16.
|
46
0402+2509165015.JSON
Normal file
46
0402+2509165015.JSON
Normal file
@@ -0,0 +1,46 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import json
|
||||
import time
|
||||
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
||||
movies = []
|
||||
|
||||
for start in range(0, 250, 25):
|
||||
url = f"https://movie.douban.com/top250?start={start}"
|
||||
res = requests.get(url, headers=headers)
|
||||
soup = BeautifulSoup(res.text, "html.parser")
|
||||
items = soup.find_all("div", class_="item")
|
||||
|
||||
for item in items:
|
||||
rank = item.find("em").text
|
||||
title = item.find("span", class_="title").text
|
||||
rating = item.find("span", class_="rating_num").text
|
||||
people = item.find("div", class_="star").find_all("span")[-1].text.replace("人评价", "")
|
||||
quote = item.find("span", class_="inq").text if item.find("span", class_="inq") else "无"
|
||||
info = item.find("p", class_="").text.strip().split("\n")
|
||||
line1 = info[0].strip()
|
||||
line2 = info[1].strip() if len(info) > 1 else ""
|
||||
|
||||
director = line1.split("导演: ")[1].split("主演: ")[0].strip() if "导演: " in line1 else "未知"
|
||||
actor = line1.split("主演: ")[1].strip() if "主演: " in line1 else "未知"
|
||||
parts = line2.split("/") if line2 else []
|
||||
year = parts[0].strip() if len(parts) >= 1 else "未知"
|
||||
area = parts[1].strip() if len(parts) >= 2 else "未知"
|
||||
genre = parts[2].strip() if len(parts) >= 3 else "未知"
|
||||
|
||||
movies.append({
|
||||
"排名": rank, "电影名": title, "评分": rating, "评价人数": people, "经典台词": quote,
|
||||
"导演": director, "主演": actor, "年份": year, "地区": area, "类型": genre
|
||||
})
|
||||
|
||||
time.sleep(1)
|
||||
print(f"已爬取 {start + 25} 条")
|
||||
|
||||
with open("douban_top250.json", "w", encoding="utf-8") as f:
|
||||
json.dump(movies, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print("✅ JSON 导出完成")
|
||||
50
0402+2509165015.txt
Normal file
50
0402+2509165015.txt
Normal file
@@ -0,0 +1,50 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import time
|
||||
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
||||
movies = []
|
||||
|
||||
for start in range(0, 250, 25):
|
||||
url = f"https://movie.douban.com/top250?start={start}"
|
||||
res = requests.get(url, headers=headers)
|
||||
soup = BeautifulSoup(res.text, "html.parser")
|
||||
items = soup.find_all("div", class_="item")
|
||||
|
||||
for item in items:
|
||||
rank = item.find("em").text
|
||||
title = item.find("span", class_="title").text
|
||||
rating = item.find("span", class_="rating_num").text
|
||||
people = item.find("div", class_="star").find_all("span")[-1].text.replace("<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>", "")
|
||||
quote = item.find("span", class_="inq").text if item.find("span", class_="inq") else "<22><>"
|
||||
info = item.find("p", class_="").text.strip().split("\n")
|
||||
line1 = info[0].strip()
|
||||
line2 = info[1].strip() if len(info) > 1 else ""
|
||||
|
||||
director = line1.split("<22><><EFBFBD><EFBFBD>: ")[1].split("<22><><EFBFBD><EFBFBD>: ")[0].strip() if "<22><><EFBFBD><EFBFBD>: " in line1 else "δ֪"
|
||||
actor = line1.split("<22><><EFBFBD><EFBFBD>: ")[1].strip() if "<22><><EFBFBD><EFBFBD>: " in line1 else "δ֪"
|
||||
parts = line2.split("/") if line2 else []
|
||||
year = parts[0].strip() if len(parts) >= 1 else "δ֪"
|
||||
area = parts[1].strip() if len(parts) >= 2 else "δ֪"
|
||||
genre = parts[2].strip() if len(parts) >= 3 else "δ֪"
|
||||
|
||||
movies.append({
|
||||
"<22><><EFBFBD><EFBFBD>": rank, "<22><>Ӱ<EFBFBD><D3B0>": title, "<22><><EFBFBD><EFBFBD>": rating, "<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>": people, "<22><><EFBFBD><EFBFBD>̨<EFBFBD><CCA8>": quote,
|
||||
"<22><><EFBFBD><EFBFBD>": director, "<22><><EFBFBD><EFBFBD>": actor, "<22><><EFBFBD><EFBFBD>": year, "<22><><EFBFBD><EFBFBD>": area, "<22><><EFBFBD><EFBFBD>": genre
|
||||
})
|
||||
|
||||
time.sleep(1)
|
||||
print(f"<22><><EFBFBD><EFBFBD>ȡ {start + 25} <20><>")
|
||||
|
||||
with open("douban_top250.txt", "w", encoding="utf-8") as f:
|
||||
for m in movies:
|
||||
f.write(f"<22><>{m['<27><><EFBFBD><EFBFBD>']}<7D><>{m['<27><>Ӱ<EFBFBD><D3B0>']}\n")
|
||||
f.write(f"<22><><EFBFBD>֣<EFBFBD>{m['<27><><EFBFBD><EFBFBD>']} <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>{m['<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>']}\n")
|
||||
f.write(f"<22><><EFBFBD>ݣ<EFBFBD>{m['<27><><EFBFBD><EFBFBD>']} <20><><EFBFBD>ݣ<EFBFBD>{m['<27><><EFBFBD><EFBFBD>']}\n")
|
||||
f.write(f"<22><><EFBFBD>ݣ<EFBFBD>{m['<27><><EFBFBD><EFBFBD>']} <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>{m['<27><><EFBFBD><EFBFBD>']} <20><><EFBFBD>ͣ<EFBFBD>{m['<27><><EFBFBD><EFBFBD>']}\n")
|
||||
f.write(f"̨<>ʣ<EFBFBD>{m['<27><><EFBFBD><EFBFBD>̨<EFBFBD><CCA8>']}\n\n")
|
||||
|
||||
print("? TXT <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>")
|
||||
Reference in New Issue
Block a user