2020年8月

Posted on 2020年8月30日2020年8月29日2020年8月30日2020年8月29日

【Python】爬虫(Xpath):批量爬取站长免费简历

批量爬取站长免费简历, 可以自行选择爬取页数.

from lxml import etree
import requests
import os


# 封装解析下载函数
def cv_down(tree, headers):
    cv_href = tree.xpath('//div[@class="sc_warp  mt20"]/div/div/div/a/@href')
    for href in cv_href:
        act_response = requests.get(url=href, headers=headers).text
        act_tree = etree.HTML(act_response)
        cv_title = act_tree.xpath('//div[@class="ppt_tit clearfix"]/h1/text()')
        cv_title = cv_title[0].encode('ISO-8859-1').decode('utf-8') + '.rar'
        dow_url = act_tree.xpath('//div[@class="clearfix mt20 downlist"]/ul/li[1]/a/@href')[0]
        doc = requests.get(url=dow_url, headers=headers).content
        cv_path = './免费简历/' + cv_title
        with open(cv_path, 'wb') as fp:
            fp.write(doc)
            print(cv_title, '下载完成！！！')


# 检查文件夹是否存在,并创建文件夹
if not os.path.exists('./免费简历'):
    os.mkdir('./免费简历')

first_page_url = 'http://sc.chinaz.com/jianli/free.html'
headers = {
    "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36'
}

tree = etree.HTML(requests.get(url=first_page_url, headers=headers).text)
page_limit = tree.xpath('//div[@class="pagination fr clearfix clear"]/a[8]/b/text()')[0]

while True:
    print(page_limit + " pages at most")
    page = input("Please enter how many page you want: ")

    if page.isdigit() and 1 <= int(page) <= int(page_limit):
        for i in range(1, int(page) + 1):
            if i == 1:
                cv_down(tree, headers)
            else:
                other_page_url = 'http://sc.chinaz.com/jianli/free_' + str(i) + '.html'  # 一页以上的特殊性需要重新制定链接
                response = requests.get(url=other_page_url, headers=headers).text
                tree = etree.HTML(response)
                cv_down(tree, headers)
        break
    else:
        print("You can only enter right numbers!!!")
        continue

from lxml import etree

import requests

import os

# 封装解析下载函数

def cv_down(tree, headers):

cv_href = tree.xpath('//div[@class="sc_warp mt20"]/div/div/div/a/@href')

for href in cv_href:

act_response = requests.get(url=href, headers=headers).text

act_tree = etree.HTML(act_response)

cv_title = act_tree.xpath('//div[@class="ppt_tit clearfix"]/h1/text()')

cv_title = cv_title[0].encode('ISO-8859-1').decode('utf-8') + '.rar'

dow_url = act_tree.xpath('//div[@class="clearfix mt20 downlist"]/ul/li[1]/a/@href')[0]

doc = requests.get(url=dow_url, headers=headers).content

cv_path = './免费简历/' + cv_title

with open(cv_path, 'wb') as fp:

fp.write(doc)

print(cv_title, '下载完成！！！')

# 检查文件夹是否存在,并创建文件夹

if not os.path.exists('./免费简历'):

os.mkdir('./免费简历')

first_page_url = 'http://sc.chinaz.com/jianli/free.html'

headers = {

"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36'

}

tree = etree.HTML(requests.get(url=first_page_url, headers=headers).text)

page_limit = tree.xpath('//div[@class="pagination fr clearfix clear"]/a[8]/b/text()')[0]

while True:

print(page_limit + " pages at most")

page = input("Please enter how many page you want: ")

if page.isdigit() and 1 <= int(page) <= int(page_limit):

for i in range(1, int(page) + 1):

if i == 1:

cv_down(tree, headers)

else:

other_page_url = 'http://sc.chinaz.com/jianli/free_' + str(i) + '.html' # 一页以上的特殊性需要重新制定链接

response = requests.get(url=other_page_url, headers=headers).text

tree = etree.HTML(response)

cv_down(tree, headers)

break

else:

print("You can only enter right numbers!!!")

continue

Posted in 随便说Tagged Python, Xpath, 爬虫Leave a Comment

by Zeno

Posted on 2020年8月29日2020年8月29日

【Python】爬虫(Xpath):批量爬取站长免费PPT模版

本来是打算写一个异步爬取的,但是暂时还没有发现是什么问题好像没有实现异步

import requests
from lxml import etree
import os
import asyncio
import aiohttp


def get_ppt_list(page_need, headers):
    for i in range(1, int(page_need)+1):
        if i == 1:
            url = 'http://sc.chinaz.com/ppt/free.html'
            list_page_text = requests.get(url=url, headers=headers).text
            list_page_tree = etree.HTML(list_page_text)
            list_ppt_url = list_page_tree.xpath('//*[@id="main"]/div/div/a/@href')
            tasks = []
            for url in list_ppt_url:
                c = get_ppt(url, headers)
                task = asyncio.ensure_future(c)
                tasks.append(task)
            loop = asyncio.get_event_loop()
            loop.run_until_complete(asyncio.wait(tasks))

        else:
            url = 'http://sc.chinaz.com/ppt/free_{}.html'.format(i)
            list_page_text = requests.get(url=url, headers=headers).text
            list_page_tree = etree.HTML(list_page_text)
            list_ppt_url = list_page_tree.xpath('//*[@id="main"]/div/div/a/@href')
            tasks = []
            for url in list_ppt_url:
                c = get_ppt(url, headers)
                task = asyncio.ensure_future(c)
                tasks.append(task)
            loop = asyncio.get_event_loop()
            loop.run_until_complete(asyncio.wait(tasks))


async def get_ppt(url, headers):
    async with aiohttp.ClientSession() as session:
        async with await session.get(url=url, headers=headers) as ppt_page:
            ppt_page_text = await ppt_page.text()
            ppt_page_tree = etree.HTML(ppt_page_text)
            title = ppt_page_tree.xpath('//div[@class="ppt_tit clearfix"]/h1/text()')[0]
            title = title + '.rar'
            ppt_path = './站长PPT/' + title
            down_url = ppt_page_tree.xpath('//*[@id="down"]/div[2]/ul/li[3]/a/@href')[0]
            async with await session.get(url=down_url, headers=headers) as ppt:
                ppt = await ppt.read()
                with open(ppt_path, 'wb') as fp:
                    fp.write(ppt)
                print("{} 下载成功!!!".format(title))


url = 'http://sc.chinaz.com/ppt/free.html'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36',
}

list_page_text = requests.get(url=url, headers=headers).text
list_page_tree = etree.HTML(list_page_text)
page_limit = list_page_tree.xpath('//div[@class="pagination fr clearfix clear"]/a[8]/b/text()')[0]


while True:
    print("该模板页数上限是{}, 请输入您所需要的总页数".format(page_limit))
    page_need = input("<<<")
    if page_need.isdigit() and 1 <= int(page_need) <= int(page_limit):
        if not os.path.exists('./站长PPT'):
            os.mkdir('./站长PPT')
        get_ppt_list(page_need, headers)
        break
    else:
        print("请重新输入正确的页数")
        continue

import requests

from lxml import etree

import os

import asyncio

import aiohttp

def get_ppt_list(page_need, headers):

for i in range(1, int(page_need)+1):

if i == 1:

url = 'http://sc.chinaz.com/ppt/free.html'

list_page_text = requests.get(url=url, headers=headers).text

list_page_tree = etree.HTML(list_page_text)

list_ppt_url = list_page_tree.xpath('//*[@id="main"]/div/div/a/@href')

tasks = []

for url in list_ppt_url:

c = get_ppt(url, headers)

task = asyncio.ensure_future(c)

tasks.append(task)

loop = asyncio.get_event_loop()

loop.run_until_complete(asyncio.wait(tasks))

else:

url = 'http://sc.chinaz.com/ppt/free_{}.html'.format(i)

list_page_text = requests.get(url=url, headers=headers).text

list_page_tree = etree.HTML(list_page_text)

list_ppt_url = list_page_tree.xpath('//*[@id="main"]/div/div/a/@href')

tasks = []

for url in list_ppt_url:

c = get_ppt(url, headers)

task = asyncio.ensure_future(c)

tasks.append(task)

loop = asyncio.get_event_loop()

loop.run_until_complete(asyncio.wait(tasks))

async def get_ppt(url, headers):

async with aiohttp.ClientSession() as session:

async with await session.get(url=url, headers=headers) as ppt_page:

ppt_page_text = await ppt_page.text()

ppt_page_tree = etree.HTML(ppt_page_text)

title = ppt_page_tree.xpath('//div[@class="ppt_tit clearfix"]/h1/text()')[0]

title = title + '.rar'

ppt_path = './站长PPT/' + title

down_url = ppt_page_tree.xpath('//*[@id="down"]/div[2]/ul/li[3]/a/@href')[0]

async with await session.get(url=down_url, headers=headers) as ppt:

ppt = await ppt.read()

with open(ppt_path, 'wb') as fp:

fp.write(ppt)

print("{} 下载成功!!!".format(title))

url = 'http://sc.chinaz.com/ppt/free.html'

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36',

}

list_page_text = requests.get(url=url, headers=headers).text

list_page_tree = etree.HTML(list_page_text)

page_limit = list_page_tree.xpath('//div[@class="pagination fr clearfix clear"]/a[8]/b/text()')[0]

while True:

print("该模板页数上限是{}, 请输入您所需要的总页数".format(page_limit))

page_need = input("<<<")

if page_need.isdigit() and 1 <= int(page_need) <= int(page_limit):

if not os.path.exists('./站长PPT'):

os.mkdir('./站长PPT')

get_ppt_list(page_need, headers)

break

else:

print("请重新输入正确的页数")

continue

Posted in 随便说Tagged aiohttp, asyncio, Python, Xpath, 异步, 爬虫Leave a Comment

by Zeno

Posted on 2020年8月16日2020年8月16日

【Python】爬虫(Xpath):批量爬取彼岸图网4K图(非真正4K)

因为真正的4K图需要登录,并且登录后一天也就只能下载一张,所以就只解析爬取了内容页的高清图片(1200*633左右)

代码实现了我目前所能想到的用户骚操作的判定,例如选择的类目不在这个网站内, 或者这个类目没有那么多页之类的废话不多说直接上代码吧

# By.Zeno


import requests
from lxml import etree
import os


# 封装从栏目页进入内容页连接解析函数
def name(url, type_name, headers):
    url = url + page_list_url[int(type_name)]
    doc_name = './' + page_list_name[int(type_name)].encode('ISO-8859-1').decode('gbk')
    page_list_text = requests.get(url=url, headers=headers).text
    page_list_tree = etree.HTML(page_list_text)
    page_list_limit = page_list_tree.xpath('//*[@id="main"]/div[4]/a[7]/text()')[0]
    while True:
        print("{}上限页数为{}页".format(page_list_name[int(type_name)].encode('ISO-8859-1').decode('gbk'), page_list_limit))
        page_need = input("请输入您要爬取{}的页数: ".format(page_list_name[int(type_name)].encode('ISO-8859-1').decode('gbk')))

        if page_need.isdigit() and 1 <= int(page_need) <= int(page_list_limit):  # 判断是否填写有误(包括页数判断)
            if not os.path.exists(doc_name):
                os.mkdir(doc_name)
            for i in range(1, int(page_need) + 1):
                if i == 1:  # 第1页爬取
                    crawler(page_list_tree, doc_name)
                else:  # 第2页及以上爬取
                    page_url = url + 'index_' + str(i) + '.html'
                    page_list_text = requests.get(url=page_url, headers=headers).text
                    page_list_tree = etree.HTML(page_list_text)
                    crawler(page_list_tree, doc_name)
            break
        else:
            print("请重新输入正确的数字")


# 封装内容页图片连接解析函数
def crawler(page_list_tree, doc_name):
    img_list_url = page_list_tree.xpath('//*[@id="main"]/div[3]/ul/li/a/@href')
    img_name = page_list_tree.xpath('//*[@id="main"]/div[3]/ul/li/a/b/text()')
    for i in range(len(img_name)):
        img_url = 'http://pic.netbian.com' + img_list_url[i]
        name = img_name[i].encode('ISO-8859-1').decode('gbk') + '.jpg'
        img_page = requests.get(url=img_url, headers=headers).text
        img_page_tree = etree.HTML(img_page)
        page_img_src = img_page_tree.xpath('//*[@id="img"]/img/@src')[0]
        page_img_src = 'http://pic.netbian.com' + page_img_src
        img = requests.get(url=page_img_src, headers=headers).content
        img_path = doc_name + '/' + name
        with open(img_path, 'wb') as fp:
            fp.write(img)
            print(name + '下载成功!!!')


url = 'http://pic.netbian.com'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3775.400 QQBrowser/10.6.4208.400'
}
response = requests.get(url=url, headers=headers).text
tree = etree.HTML(response)
page_list_url = tree.xpath('//*[@id="main"]/div[2]/a/@href')
page_list_name = tree.xpath('//*[@id="main"]/div[2]/a/text()')
while True:
    print("0.风景 1.美女 2.游戏 3.动漫 4.影视 5.明星 6.汽车 7.动物 8.人物 9.美食 10.宗教 11.背景")
    type_name = input("请输入对应数字: ")
    if type_name.isdigit() and 0 <= int(type_name) <= 11: #判断是否数字且是否超出可爬取范围
        name(url, type_name, headers)
        break
    else:
        print("请重新输入正确的数字")
        continue

# By.Zeno

import requests

from lxml import etree

import os

# 封装从栏目页进入内容页连接解析函数

def name(url, type_name, headers):

url = url + page_list_url[int(type_name)]

doc_name = './' + page_list_name[int(type_name)].encode('ISO-8859-1').decode('gbk')

page_list_text = requests.get(url=url, headers=headers).text

page_list_tree = etree.HTML(page_list_text)

page_list_limit = page_list_tree.xpath('//*[@id="main"]/div[4]/a[7]/text()')[0]

while True:

print("{}上限页数为{}页".format(page_list_name[int(type_name)].encode('ISO-8859-1').decode('gbk'), page_list_limit))

page_need = input("请输入您要爬取{}的页数: ".format(page_list_name[int(type_name)].encode('ISO-8859-1').decode('gbk')))

if page_need.isdigit() and 1 <= int(page_need) <= int(page_list_limit): # 判断是否填写有误(包括页数判断)

if not os.path.exists(doc_name):

os.mkdir(doc_name)

for i in range(1, int(page_need) + 1):

if i == 1: # 第1页爬取

crawler(page_list_tree, doc_name)

else: # 第2页及以上爬取

page_url = url + 'index_' + str(i) + '.html'

page_list_text = requests.get(url=page_url, headers=headers).text

page_list_tree = etree.HTML(page_list_text)

crawler(page_list_tree, doc_name)

break

else:

print("请重新输入正确的数字")

# 封装内容页图片连接解析函数

def crawler(page_list_tree, doc_name):

img_list_url = page_list_tree.xpath('//*[@id="main"]/div[3]/ul/li/a/@href')

img_name = page_list_tree.xpath('//*[@id="main"]/div[3]/ul/li/a/b/text()')

for i in range(len(img_name)):

img_url = 'http://pic.netbian.com' + img_list_url[i]

name = img_name[i].encode('ISO-8859-1').decode('gbk') + '.jpg'

img_page = requests.get(url=img_url, headers=headers).text

img_page_tree = etree.HTML(img_page)

page_img_src = img_page_tree.xpath('//*[@id="img"]/img/@src')[0]

page_img_src = 'http://pic.netbian.com' + page_img_src

img = requests.get(url=page_img_src, headers=headers).content

img_path = doc_name + '/' + name

with open(img_path, 'wb') as fp:

fp.write(img)

print(name + '下载成功!!!')

url = 'http://pic.netbian.com'

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3775.400 QQBrowser/10.6.4208.400'

}

response = requests.get(url=url, headers=headers).text

tree = etree.HTML(response)

page_list_url = tree.xpath('//*[@id="main"]/div[2]/a/@href')

page_list_name = tree.xpath('//*[@id="main"]/div[2]/a/text()')

while True:

print("0.风景 1.美女 2.游戏 3.动漫 4.影视 5.明星 6.汽车 7.动物 8.人物 9.美食 10.宗教 11.背景")

type_name = input("请输入对应数字: ")

if type_name.isdigit() and 0 <= int(type_name) <= 11: #判断是否数字且是否超出可爬取范围

name(url, type_name, headers)

break

else:

print("请重新输入正确的数字")

continue

Posted in 随便说Tagged Python, Xpath, 爬虫Leave a Comment

by Zeno

Posted on 2020年8月10日2020年8月10日

【Python】爬虫图片爬取-Zeno

还是那个视频的教程,代码都是自己敲得,也是自己思路,除了乱码问题以外

#需求: 解析下载图片数据 http://pic.netbian.com/4kdongwu/
from lxml import etree
import requests
import os

if not os.path.exists('./4k动物'):
    os.mkdir('./4k动物')

url = 'http://pic.netbian.com/4kdongwu/index.html'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36'
}




page_text = requests.get(url=url, headers=headers).text
tree = etree.HTML(page_text)

li_list = tree.xpath('//div[@class="slist"]/ul/li')

print(li_list)

for i in li_list:
    src = i.xpath('./a/@href')
    list_src = 'http://pic.netbian.com' + src[0]
    img_page = requests.get(url=list_src, headers=headers).text
    img_tree = etree.HTML(img_page)
    img_title = img_tree.xpath('//div[@class="photo-pic"]/a/img/@alt')
    img_title = img_title[0].encode('ISO-8859-1').decode('gbk') + '.jpg'
    img_src = 'http://pic.netbian.com/' + img_tree.xpath('//div[@class="photo-pic"]/a/img/@src')[0]
    img = requests.get(img_src, headers=headers).content
    img_path = './4k动物/' + img_title
    with open(img_path, 'wb') as fp:
        fp.write(img)
        print(img_title, "---------", "下载成功!!!")

#需求: 解析下载图片数据 http://pic.netbian.com/4kdongwu/

from lxml import etree

import requests

import os

if not os.path.exists('./4k动物'):

os.mkdir('./4k动物')

url = 'http://pic.netbian.com/4kdongwu/index.html'

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36'

}

page_text = requests.get(url=url, headers=headers).text

tree = etree.HTML(page_text)

li_list = tree.xpath('//div[@class="slist"]/ul/li')

print(li_list)

for i in li_list:

src = i.xpath('./a/@href')

list_src = 'http://pic.netbian.com' + src[0]

img_page = requests.get(url=list_src, headers=headers).text

img_tree = etree.HTML(img_page)

img_title = img_tree.xpath('//div[@class="photo-pic"]/a/img/@alt')

img_title = img_title[0].encode('ISO-8859-1').decode('gbk') + '.jpg'

img_src = 'http://pic.netbian.com/' + img_tree.xpath('//div[@class="photo-pic"]/a/img/@src')[0]

img = requests.get(img_src, headers=headers).content

img_path = './4k动物/' + img_title

with open(img_path, 'wb') as fp:

fp.write(img)

print(img_title, "---------", "下载成功!!!")

Posted in 随便说Tagged Python, 爬虫Leave a Comment

by Zeno

Posted on 2020年8月8日2020年8月10日2020年8月8日2020年8月10日

【Python】爬虫初次实战-Zeno

一直都说想学爬虫,前两天就开始认真在B站看了一下爬虫,其实之前也看过一点,但是之前看的那个是用urllib的用起来就比较麻烦.然后这次在B站找到的一个还不错,直接奉上链接吧:https://www.bilibili.com/video/BV1Yh411o7Sz

到目前为止看到了bs4的实战(其实还没有开始看这一集), 因为看了前面的requests 正则 bs4的简单的基础内容之后,我觉得我可以完成这一集的实战内容,就利用中午休息的时间完成了这一集的内容(可能效率会代码慢了点)总的来说还算是成功的.走了走了继续看爬虫了

以下就是爬取三国演义所有内容的代码:

# 需求: 爬取三国演义小说所有的章节标题和章节内容
import requests
from bs4 import BeautifulSoup
import os

if not os.path.exists('./三国演义'):
    os.mkdir('./三国演义')

url = 'https://www.shicimingju.com/book/sanguoyanyi.html'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36'
}
# a标签链接储存列表
list_link = []

# 页面提取 -> obj
list_page = requests.get(url=url, headers=headers).text

soup = BeautifulSoup(list_page, 'lxml')

# 提取所有a标签
list_a = soup.select('.book-mulu a')

# 遍历a标签并提取链接储存
for i in range(len(list_a)):
    article_page = requests.get(url='https://www.shicimingju.com' + soup.select('.book-mulu a')[i]['href'],
                                headers=headers).text  # 获取目录得到的内容页链接
    article_soup = BeautifulSoup(article_page, 'lxml')
    title = article_soup.find('h1').get_text()  # 定位标题并提取文本
    title = title + '.txt'  # 创建txt文件名
    article = article_soup.find('div', class_='chapter_content').get_text()  # 定位内容并提取文本
    article_path = './三国演义/' + title  # 设定保存路径

    # 持久化存储
    with open(article_path, 'w', encoding='utf-8') as fp:
        fp.write(article)
        print(title, '下载完成!!!')

print('Over!!!')

# 需求: 爬取三国演义小说所有的章节标题和章节内容

import requests

from bs4 import BeautifulSoup

import os

if not os.path.exists('./三国演义'):

os.mkdir('./三国演义')

url = 'https://www.shicimingju.com/book/sanguoyanyi.html'

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36'

}

# a标签链接储存列表

list_link = []

# 页面提取 -> obj

list_page = requests.get(url=url, headers=headers).text

soup = BeautifulSoup(list_page, 'lxml')

# 提取所有a标签

list_a = soup.select('.book-mulu a')

# 遍历a标签并提取链接储存

for i in range(len(list_a)):

article_page = requests.get(url='https://www.shicimingju.com' + soup.select('.book-mulu a')[i]['href'],

headers=headers).text # 获取目录得到的内容页链接

article_soup = BeautifulSoup(article_page, 'lxml')

title = article_soup.find('h1').get_text() # 定位标题并提取文本

title = title + '.txt' # 创建txt文件名

article = article_soup.find('div', class_='chapter_content').get_text() # 定位内容并提取文本