创建数据库
mysql> create database taobao;
创建表
create table goods (
id
title varchar(100),
price varchar(100),
link varchar(100),
comment varchar(100)
)
创建爬虫项目
scrapy startproject taobao
cd taobao
# 创建爬虫文件
scrapy genspider -t basic app taobao.com
修改items.py
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class TaobaoItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title = scrapy.Field()
link = scrapy.Field()
price = scrapy.Field()
comment = scrapy.Field()
修改settings.py
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'taobao.pipelines.TaobaoPipeline': 300,
}
