Python之获取豆瓣电影Top250

发表于 | 分类于 ，
字数统计: 448 字

练习html各元素的获取，熟悉BeautifulSoup的使用。
实例：豆瓣top250电影名单获取。

获取html页面

1
2
3
4
5
6
7
8
9
10
11
12
import requests
def download_page(url):
    data = requests.get(url).text
    '''
    这里有效,但据说有时要伪装成浏览器,代码修改如下:
    data = requests.get(url, headers={
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
    }).content
    '''
    return data

User-Agent的获取方法：console敲入命令：navigator.userAgent

解析页面

  • 将html交给BeautifulSoup解析一下

    1
    2
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, "html.parser")

  • 找到数据element

  1. ol.grid_view
    t0
  2. div.hd
    t1
  3. span.title
    t2

  • 出动选择器

    1
    2
    3
    movie_list_soup = soup.find('ol', attrs={'class': 'grid_view'})
    detail = movie_li.find('div', attrs={'class': 'hd'})
    movie_name = detail.find('span', attrs={'class': 'title'}).getText()

  • 循环把一页结果存下来

    1
    2
    3
    4
    5
    6
    movie_name_list = []
    for movie_li in movie_list_soup.find_all('li'):
        detail = movie_li.find('div', attrs={'class': 'hd'})
        movie_name = detail.find('span', attrs={'class': 'title'}).getText()
        movie_name_list.append(movie_name)

  • 拿到下一页的element
    t3

    1
    next_page = soup.find('span', attrs={'class': 'next'}).find('a')

写入文件

1
2
3
4
5
with open('movies.txt', 'w') as fp:
    while url:
        html = download_page(url)
        movies, url = parse_html(html)
        fp.write(u'{movies}\n'.format(movies='\n'.join(movies)))

完整代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
DOWNLOAD_URL = "https://movie.douban.com/top250"
# 获取html页面
def download_page(url):
    data = requests.get(url).content
    '''
    这里有效,但据说有时要伪装成浏览器,代码修改如下:
    data = requests.get(url, headers={
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
    }).content
    '''
    return data
# 解析页面并返回 list_movie 与 下一页的url
def parse_html(html):
    soup = BeautifulSoup(html, "html.parser")
    movie_list_soup = soup.find('ol', attrs={'class': 'grid_view'})
    movie_name_list = []
    for movie_li in movie_list_soup.find_all('li'):
        detail = movie_li.find('div', attrs={'class': 'hd'})
        movie_name = detail.find('span', attrs={'class': 'title'}).getText()
        movie_name_list.append(movie_name)
    next_page = soup.find('span', attrs={'class': 'next'}).find('a')
    if next_page:
        return movie_name_list, DOWNLOAD_URL + next_page['href']
    else:
        return movie_name_list, None
def main():
    url = DOWNLOAD_URL
    with open('movies.txt', 'w') as fp:
        while url:
            html = download_page(url)
            movies, url = parse_html(html)
            fp.write(u'{movies}\n'.format(movies='\n'.join(movies)))    
if __name__ == '__main__':
    main()