在Scrapy中使用MySQL插入数据

Question

这是我的演示，我认为没有错误，但是此代码无法向mysql插入数据！

setting.py

这是设置代码：

BOT_NAME = 'Scan'
SPIDER_MODULES = ['scan.spiders']
#NEWSPIDER_MODULE = 'scan.spiders'
ITEM_PIPELINES = ['scan.pipelines.MySQLStorePipeline']

这是管道代码，我认为它不包含任何错误：

pipelines.py

from scrapy import log
from twisted.enterprise import adbapi
from scrapy.http import Request
from scrapy.exceptions import DropItem
from scrapy.contrib.pipeline.images import ImagesPipeline
import datetime
import MySQLdb
import MySQLdb.cursors


class MySQLStorePipeline(object):

    def __init__(self):
        self.db = adbapi.ConnectionPool('MySQLdb',
            db = 'spider',
            host='localhost',
            user = 'root',
            passwd = '123456',
            cursorclass = MySQLdb.cursors.DictCursor,
            charset = 'utf8',
            use_unicode = True
        )

    def process_item(self, item, spider):
        query = self.db.runInteraction(self._conditional_insert, item)
        query.addErrback(self.handle_error)
        return item

    def _conditional_insert(self, tx, item):
        if item.get('url'):
            tx.execute(\
                "insert into spider (url) "
                "values (%s)",(item['link'])
            )
            #log.msg("Item stored in db: %s" % item, level=log.DEBUG)
    def handle_error(self, e):
        log.err(e)

这是蜘蛛模块，我认为它不包含任何错误！

spider.py

# coding=utf-8
from urlparse import urljoin
import simplejson

from scrapy.http import Request
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import HtmlXPathSelector

from scan.items import ScanItem

class ScanSpider(CrawlSpider):
    name = 'Scan'
    allowed_domains = ["a.com"]
    start_urls = [
        "http://www.a.com",
    ]
    rules = (
        Rule(SgmlLinkExtractor(allow=(r'http://(.*?)'),deny_domains=(r'qq.com'))
        ),
        Rule(SgmlLinkExtractor(allow=(r'http://www.a.com')), callback="parse_item"),
        )

    def parse_item(self, response):
        hxs = HtmlXPathSelector(response)
        item = ScanItem()
        items = []
        #item['title'] = hxs.select('//title/text()').extract()
        item['url'] = hxs.select('//a[@href]').re('(\"http://(.*?)\")').extract()
        items.append(item)
        return items
SPIDER = ScanSpider()

Answer 1

您非常需要阅读http://doc.scrapy.org ，但缺少基本知识。

def parse_item(self, response):
    hxs = HtmlXPathSelector(response)
    item = ScanItem()
    items = []
    #item['title'] = hxs.select('//title/text()').extract()
    item['url'] = hxs.select('//a[@href]').re('(\"http://(.*?)\")').extract()
    items.append(item)
    return items

在您的parse_items ，无需在列表项中添加项目，您只需返回以下item即可

    def parse_item(self, response):
       hxs = HtmlXPathSelector(response)
       item = ScanItem()
       #item['title'] = hxs.select('//title/text()').extract()
       item['url'] = hxs.select('//a[@href]').re('(\"http://(.*?)\")').extract()
       return item

请记住item['url']包含urls列表

在您的MySQLStorePipeline

def _conditional_insert(self, tx, item):
    if item.get('url'):
        tx.execute(\
            "insert into spider (url) "
            "values (%s)",(item['link'])
        )

您试图在数据库中插入item['link'] ，而从未填充item['link']但仅填充item['url'] 。

Answer 2

目前，无需进一步澄清。

我认为，由于刮的回报预期是一个Item （其子类，或者甚至只是一个dict -我承认我没有检查的代码完全看不出这应该 / 不工作），则：

item = ScanItem()
items = []
...
return items

看起来有点sus。

您需要多做一些分析，才能获得有意义/正确的答案。