获取豆瓣电影TOP 250并存入文件

# encoding=utf-8

import requests
from bs4 import BeautifulSoup
import csv
import codecs

DOWNLOAD_URL = 'http://movie.douban.com/top250'


def download_page(url):
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0'}
data = requests.get(url, headers=headers).content
return data


def parse_html(html):
#更换解析器,详情 https://www.crummy.com/software/BeautifulSoup/bs4/doc/index.zh.html#id9
soup = BeautifulSoup(html, "lxml")
movies_list_soup = soup.find('ol', attrs={'class': 'grid_view'})
movie_name_list = []
movie_rating_num_list = []
movie_quote_list = []

for movies_li in movies_list_soup.find_all('li'):
movie_name = movies_li.find('div', attrs={'class': 'hd'}).find('span', attrs={'class': 'title'}).getText()
movie_rating = movies_li.find('div', attrs={'class': 'bd'}).find('span', attrs={'class': 'rating_num',
'property': 'v:average'}).getText()
#movie_quote = movies_li.find('div', attrs={'class': 'bd'}).find('span', attrs={'class': 'inq'}).getText()
movie_quote_obj = movies_li.find('div', attrs={'class': 'bd'}).find('span', attrs={'class': 'inq'})
### 发现某部电影评价为空,此处判断评价是否为空
if movie_quote_obj is None:
movie_quote = ''
else:
movie_quote = movie_quote_obj.getText()

movie_name_list.append(movie_name)
movie_rating_num_list.append(movie_rating)
movie_quote_list.append(movie_quote)

next_page = soup.find('span', attrs={'class': 'next'}).find('a')
if next_page:
return movie_name_list, movie_rating_num_list, movie_quote_list, DOWNLOAD_URL + next_page['href']

return movie_name_list, movie_rating_num_list, movie_quote_list, None


def main():
url = DOWNLOAD_URL
#utf_8_sig 解决windows下csv文件乱码问题
with codecs.open('movies_top_250.csv', 'wb', encoding='utf_8_sig') as csvFile:
writer = csv.writer(csvFile)
writer.writerow((u'电影名', u'评分', u'理由'))
while url:
html = download_page(url)
movies, rating, quote, url = parse_html(html)
for i in range(len(movies)):
writer.writerow((movies[i], rating[i], quote[i]))

if __name__ == '__main__':
main()
-------------本文结束感谢阅读-------------
0%