MENU

文章目录

Python--爬取豆瓣TOP250电影信息

起因

最近在生产实习,要求自己找一个网站爬一下存储数据到excel表格中

我看电影也是按豆瓣TOP250来筛选的,手动翻页太麻烦,于是爬一下

[2019-09-02更新]后面要做作业答辩,改为存储到mysql数据库中

代码实现

和网上大部分爬取的文章不同,我想要的是每部电影的剧情简介信息

所以需要先获取每部电影的链接,再单独爬取每部电影

全部代码如下:

# -*- coding: utf-8 -*-
'''
@author: soapffz
@fucntion: 豆瓣TOP250电影信息爬取并存储到mysql数据库(多线程)
@time: 2019-09-01
'''

import requests
from fake_useragent import UserAgent
from lxml import etree
from tqdm import tqdm
import threading
import pymysql
from re import split

"""
提示库找不到可复制以下语句解决
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple fake_useragent
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple tqdm
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple threading
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple pymysql
"""


class Top250(object):
    def __init__(self):
        ua = UserAgent()  # 用于生成User-Agent
        self.headers = {"User-Agent": ua.random}  # 获得一个随机的User-Agent
        self.bangdang_l = []  # 存储榜单的页面
        self.subject_url_l = []  # 存储每部电影的链接
        self.connect_mysql()

    def connect_mysql(self):
        # 连接数据库,密码后面可以加数据库名称
        try:
            self.mysql_conn = pymysql.connect(
                'localhost', 'root', 'root', charset='utf8')
            # 得到一个可以执行SQL语句的光标对象,执行完毕返回的结果默认以元组显示
            self.cursor = self.mysql_conn.cursor()
            print("数据库连接成功")
        except Exception as e:
            print("数据库连接出错:{}\n你让我爬完存哪?不爬了,退出程序!".format(e))
            exit(0)
        else:
            self.create_db()

    def create_db(self):
        # 创建数据库和表
        # 检测数据库是否存在,存在则删除,然后创建
        sql_db_dection = "DROP DATABASE IF EXISTS `douban_top250`"
        sql_create_db = "CREATE DATABASE `douban_top250` default charset utf8 COLLATE utf8_general_ci;"
        sql_create_table = """
            CREATE TABLE `movies_info` (
                `电影名称` varchar(255) NOT NULL,
                `导演` varchar(511) NOT NULL,
                `主演信息` varchar(511) NOT NULL,
                `类型` varchar(255) NOT NULL,
                `上映日期` varchar(255) NOT NULL,
                `剧情简介` varchar(511) NOT NULL,
                `排名` varchar(255) NOT NULL,
                `片长` varchar(255) NOT NULL,
                PRIMARY KEY (`排名`)
            )DEFAULT CHARSET=utf8;
            """
        try:
            self.cursor.execute(sql_db_dection)
            self.cursor.execute(sql_create_db)
            self.cursor.execute('use douban_top250;')  # 将当前数据库设置为刚创建的数据库
            self.cursor.execute(sql_create_table)
        except Exception as e:  # 捕获所有异常并打印,python2 是Exception,e
            print("数据库创建出错:{}\n退出程序!".format(e))
            self.mysql_conn.rollback()  # 发生错误时回滚
            self.mysql_conn.close()  # 关闭数据库连接
            exit(0)
        else:
            print("创建数据库和表成功,开始爬取每部电影的链接..")
            self.get_subject_url()

    def get_subject_url(self):
        # 遍历250榜单获取每部电影单独链接
        self.bangdang_l = [
            "https://movie.douban.com/top250?start={}&filter=".format(i) for i in range(0, 250, 25)]
        self.multi_thread(self.bangdang_l, self.crawl_bangdang)
        if len(self.subject_url_l) == 0:
            print("ip被封了,程序退出")
            exit(0)
        else:
            print("{}部TOP电影的链接已获取完毕,开始爬取单部电影,请稍后...".format(
                len(self.subject_url_l)))
            self.multi_thread(self.subject_url_l, self.get_one_movie_info)

    def crawl_bangdang(self, url):
        # 爬取每个榜单页面的电影链接
        try:
            req = requests.get(url, headers=self.headers)
            req.encoding = "utf-8"
            html = etree.HTML(req.text)
            # 获得所有a标签列表
            url_l = html.xpath('//div[@class="hd"]//a/@href')
            self.subject_url_l.extend(url_l)
        except Exception as e:
            print("爬取榜单信息时报错:{}".format(e))
        else:
            print("获取第{}页的榜单数据成功...\n".format(
                int(int(split(r"=|&", url)[1])/25 + 1)))

    def multi_thread(self, url_l, target_func):
        # 多线程爬取函数,传入需要爬取的url列表以及用来爬取的函数
        threads = []
        for i in range(len(url_l)):
            t = threading.Thread(target=target_func, args=(url_l[i],))
            threads.append(t)
        for i in range(len(threads)):
            threads[i].start()
        for i in range(len(threads)):
            threads[i].join()
        print("爬取完毕")

    def get_one_movie_info(self, subject):
        # 爬取单个url的函数
        try:
            req = requests.get(subject, headers=self.headers)
            html = etree.HTML(req.content)
        except Exception as e:
            print("爬取报错:".format(e))
        else:
            # 用来存储单个电影的信息
            info = []
            # 电影名称信息
            movie_name = html.xpath(
                "//span[@property='v:itemreviewed']/text()")
            info.append(" ".join(movie_name))
            # 导演信息
            director = html.xpath("//a[@rel='v:directedBy']//text()")
            info.append(" ".join(director))
            # 主演信息
            actor = html.xpath("//a[@rel='v:starring']//text()")
            info.append(" ".join(actor))
            # 类型
            genre = html.xpath("//span[@property='v:genre']/text()")
            info.append(" ".join(genre))
            # 上映日期
            initialReleaseDate = html.xpath(
                "//span[@property='v:initialReleaseDate']/text()")
            info.append(" ".join(initialReleaseDate))
            # 剧情简介
            reated_info = html.xpath("//span[@class='all hidden']/text()")
            # 有的剧情简介是隐藏的,默认获取隐藏的标签,如果没隐藏获取没隐藏的标签
            if len(reated_info) == 0:
                reated_info = html.xpath(
                    "//span[@property='v:summary']/text()")
            reated_info = "".join([s.strip() for s in reated_info]).strip("\\")
            reated_info = self.transferContent(reated_info)
            info.append(reated_info)
            # 排名
            no = html.xpath("//span[@class='top250-no']/text()")
            if len(no) == 1:
                info.append(no[0].split(".")[-1])
            else:
                info.append("获取失败")
            runtime = html.xpath("//span[@property='v:runtime']/text()")
            if len(runtime) == 1:
                info.append(runtime[0].split("分钟")[0])
            else:
                info.append("获取失败")
            self.db_insert(info)

    def db_insert(self, info_l):
        sql_insert_detection = """
            insert ignore into `douban_top250`.`movies_info` (`电影名称`,`导演`,`主演信息`,`类型`,`上映日期`,`剧情简介`,`排名`,`片长`)
            values ("{l[0]}","{l[1]}","{l[2]}","{l[3]}","{l[4]}","{l[5]}","{l[6]}","{l[7]}");
        """.format(l=info_l)
        try:
            self.cursor.execute(sql_insert_detection)
            self.mysql_conn.commit()
        except Exception as e:
            self.mysql_conn.rollback()
            print("导入数据到数据库时报错:{}".format(e))
            exit(0)
        else:
            print("{}获取信息成功...\n".format(info_l[0]))

    def transferContent(self, content):
        # 将同时含有'和"的字符串都转义
        if content is None:
            return None
        else:
            string = ""
            for c in content:
                if c == '"':
                    string += '\\\"'
                elif c == "'":
                    string += "\\\'"
                elif c == "\\":
                    string += "\\\\"
                else:
                    string += c
            return string


if __name__ == "__main__":
    Top250()

演示gif如下:

参考文章:

最后编辑于: 2019 年 09 月 02 日