[英]scrapy throwing an error when uploading to db
我得到的錯誤消息是:exceptions.TypeError:'NoneType'對象沒有屬性' getitem '。 這是我的代碼:
# -*- coding: utf-8 -*-
import scrapy,MySQLdb,codecs
from scrapy import Request
class MyItems(scrapy.Item):
topLinks = scrapy.Field() #links at top of home page to grab artist page
artists = scrapy.Field() # list of all the artists
artists_urls = scrapy.Field() #url of the page for the artist + songs
song_name = scrapy.Field() #name of each song
song_duration = scrapy.Field() #duration of the song, duh
song_dl = scrapy.Field() #dl link for the proxy site for songs
rd = scrapy.Field()
class UpdaterSpider(scrapy.Spider):
name = "updater"
allowed_domains = [
'myfreemp3.cc', 'unref.eu', 'safeurl.eu'
]
start_urls = (
'http://www.example.com/',
)
def __init__(self, *a, **kw):
super(UpdaterSpider, self).__init__(*a, **kw)
self.item = MyItems()
self.db = MySQLdb.connect( #setup my SQL database info
"127.0.0.1", #host
"root", #user
"uconn3", #password
"Music" #database
)
self.cursor = self.db.cursor() #create a cursor to execute SQL statements
self.item = MyItems()
self.y = 0
def parse(self, response):
sql = "SELECT * FROM Songs"
self.cursor.execute(sql)
result = self.cursor.fetchone()
while result is not None:
yield Request(url=result[3], callback= lambda r: self.parse_dl(r, result[1], result[2]))
result = self.cursor.fetchone()
def parse_dl(self, response, songname, duration):
self.item['rd'] = response.xpath("//head/script/text()").extract()
for x in range(len(self.item['rd'])):
self.item['rd'][x] = self.item['rd'][x].split('"')[1]
sql = "INSERT INTO Songs (" \
"Artist, songName, Duration, URL, Genre) " \
"VALUES ('Placeholder', '%s', '%s', '%s', 'Placeholder')" % \
(songname.decode('utf-8'), duration.decode('utf-8'), self.item['rd'][x])
self.cursor.execute(sql)
self.db.commit()
當我刪除sql語句行時,該錯誤已修復,但我無法弄清楚為什么這會弄亂事情。
edit1:追溯
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/twisted/internet/base.py", line 1201, in mainLoop
self.runUntilCurrent()
File "/usr/lib/python2.7/dist-packages/twisted/internet/base.py", line 824, in runUntilCurrent
call.func(*call.args, **call.kw)
File "/usr/lib/python2.7/dist-packages/twisted/internet/defer.py", line 382, in callback
self._startRunCallbacks(result)
File "/usr/lib/python2.7/dist-packages/twisted/internet/defer.py", line 490, in _startRunCallbacks
self._runCallbacks()
--- <exception caught here> ---
File "/usr/lib/python2.7/dist-packages/twisted/internet/defer.py", line 577, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/user/PycharmProjects/untitled/mp3_scraper/mp3_scraper/spiders/updater.py", line 48, in <lambda>
yield Request(url=result[3], callback= lambda r: self.parse_dl(r, result[1], result[2]))
exceptions.TypeError: 'NoneType' object has no attribute '__getitem__'
看起來像
self.item = MyItems()
沒有方法大括號[]調用的方法getitem我認為-問題不在數據庫中,請提供完整的追溯
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.