Comments (5)
from bs4 import BeautifulSoup
import xlwt
import multiprocessing
import time
def request_douban(url):
try:
response = requests.get(url,headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'})
if response.status_code == 200:
return response.text
except requests.RequestException:
return None
book = xlwt.Workbook(encoding='utf-8', style_compression=0)
sheet = book.add_sheet('豆瓣电影Top250-test', cell_overwrite_ok=True)
sheet.write(0, 0, '名称')
sheet.write(0, 1, '图片')
sheet.write(0, 2, '排名')
sheet.write(0, 3, '评分')
sheet.write(0, 4, '作者')
sheet.write(0, 5, '简介')
n = 1
def save_content(soup):
list = soup.find(class_='grid_view').find_all('li')
for item in list:
item_name = item.find(class_='title').string
item_img = item.find('a').find('img').get('src')
item_index = item.find(class_='').string
item_score = item.find(class_='rating_num').string
item_author = item.find('p').text
if (item.find(class_='inq') != None):
item_intr = item.find(class_='inq').string
# print('爬取电影:' + item_index + ' | ' + item_name +' | ' + item_img +' | ' + item_score +' | ' + item_author +' | ' + item_intr )
print('爬取电影:' + item_index + ' | ' + item_name + ' | ' + item_score + ' | ' + item_intr)
global n
sheet.write(n, 0, item_name)
sheet.write(n, 1, item_img)
sheet.write(n, 2, item_index)
sheet.write(n, 3, item_score)
sheet.write(n, 4, item_author)
sheet.write(n, 5, item_intr)
n = n + 1
def main(url):
html = request_douban(url)
soup = BeautifulSoup(html, 'lxml')
save_content(soup)
if __name__ == '__main__':
start = time.time()
urls = []
pool = multiprocessing.Pool(multiprocessing.cpu_count())
for i in range(0, 10):
url = 'https://movie.douban.com/top250?start=' + str(i * 25) + '&filter='
urls.append(url)
pool.map(main, urls)
pool.close()
pool.join()
book.save(u'豆瓣最受欢迎的250部电影-test.xls')
from learn_python3_spider.
在此贴上代码,各位大佬运行看看
from learn_python3_spider.
我也碰到了一样的问题,我觉得这个应该是Excel写的时候的问题,我用了两种方法试了一下
第一种是定义一个全局变量open('文件名','w')假如为f然后用f去写没有问题(虽然会乱序)
第二种是在main函数中对于爬取的每一页都写一个Excel文件,这样Excel的worksheet就没有全局共享,而是生成了多个Excel文件,最后把这些Excel文件合在一起就好了
from learn_python3_spider.
@HqmJoker 可以提交个 PR 么?
from learn_python3_spider.
@wistbean 已提交pr
from learn_python3_spider.
Related Issues (20)
- 怎么无法访问了
- 好
- (please help) having problem in my code
- error HOT 3
- 博主老司机无疑了
- 基础
- 爬取 20w 表情包 HOT 1
- **获取到搜索的input框后需要先.click()然后再.send_keys()** HOT 2
- 大佬POST数据拿不到这个怎么搞要 HOT 1
- 爬虫学习
- 1 HOT 2
- 真诚求问 大佬有没有 兴趣接个广子啊?
- GitHub合作 HOT 5
- 爬虫11中爬取蔡徐坤教程soup中返回的一堆数字是不是bilibili的反爬手段?还是说是我操作问题
- 教程这里有一些小错误,让新手很困惑 HOT 2
- 使用Google浏览器运行自动化测试程序时该浏览器内存占用会不断提升,超过6G后网页就会奔溃,这个问题有什么解决方法吗? HOT 1
- 网页禁F12 HOT 1
- 请求商务推广合作 HOT 5
- genymotion 中 app 闪退 HOT 2
- 关于爬取豆瓣网第一步获取请求失败的原因(我认为的)及解决方法 HOT 2
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from learn_python3_spider.