有MJJ说python的爬取效率低,我不太赞同
下面简单测试下python的异步爬取速度,其实也不低
环境:win7+python3.6
使用的库:aiohttp asyncio csv bs4
import aiohttp
import asyncio
from bs4 import BeautifulSoup
import csv
import time
async def write_csv():
urllist=['https://www.aitaotu.com/guonei/list_{}.html'.format(x) for x in range(605)]\
+['https://www.aitaotu.com/rihan/list_{}.html'.format(x) for x in range(202)]\
+['https://www.aitaotu.com/gangtai/list_{}.html'.format(x) for x in range(33)]\
+['https://www.aitaotu.com/meinv/list_{}.html'.format(x) for x in range(89)]\
headers={
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate',
'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4',
'Host':'www.aitaotu.com',
'Referer':'https://www.aitaotu.com/guonei/',
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
}
with open('aitaotu.csv', 'a+', newline='', encoding='utf-8') as f:
f_csv = csv.writer(f)
for url in urllist:
async with aiohttp.ClientSession() as session:
async with session.get(url, verify_ssl=False, headers=headers) as resp:
print(resp.status)
if resp.status==200:
respdata=await resp.text()
page = BeautifulSoup(respdata,'lxml')
hrefs = page.select('#infinite_scroll > div > div.item_t > div > a')
img_urls = page.select('#infinite_scroll > div > div.item_t > div > a > img')
titles = page.select('#infinite_scroll > div > div.item_b.clearfix > div.title > span > a')
for href1, img_url1, title1 in zip(hrefs, img_urls, titles):
href=href1.get('href')
img_url=img_url1.get('data-original')
title=title1.get_text()
data={'链接':'https://www.aitaotu.com/'+href,
'图片':img_url,
'标题':title
}
print(data)
f_csv.writerow(data.values())
start=time.time()
loop = asyncio.get_event_loop()
loop.run_until_complete(write_csv())
print('总共耗时 %s s' % (time.time()-start) )







![[python] 遍历目录并对文件进行分析处理-微慑信息网-VulSee.com](https://vulsee.com/wp-content/uploads/2022/02/93e4d3f736e48d49179e03aecb4bd333.png)
![[八卦] 伊莱克斯总经理助理石靖和她的姐妹们[组图]-微慑信息网-VulSee.com](http://w.886.cn/4S9i/36565723/36565723_13518.jpg)

![[匿名者攻击] 台湾银行企业工会疑遭拖库-微慑信息网-VulSee.com](http://hackernews.cc/wp-content/uploads/2017/08/WechatIMG3051.jpeg)
![[记录] 2024观安杯部分writeup-微慑信息网-VulSee.com](https://vulsee.com/wp-content/uploads/2024/11/429e3b2028f8b9283937c25c1f696b85.png)
![CVE-2019-0708 scanner tools[POC]-微慑信息网-VulSee.com](https://github.com/zerosum0x0/CVE-2019-0708/raw/master/screenshot.png)


![[八卦] 王婷婷—揭秘一个大三女生的性爱录像-微慑信息网-VulSee.com](http://free.86hy.com/crack/pic/1.jpg)
![[随笔]今天国际警察节-微慑信息网-VulSee.com](http://photo.sohu.com/20041017/Img222528326.jpg)

青云网
