I cant quite figure out what's wrong with this code. I would like to scrape the first page, and then, for each link on that page, go to the second page to extract the item description. When i run the code below, i get: exception.TypeError: url must be str or unicode, got list. here is my code:
from scrapy.spider import Spider
from scrapy.selector import Selector
from scrapy.http import Request
from scrapy.item import Item, Field
from scrapy.contrib.loader import ItemLoader
from scrapy.contrib.loader.processor import MapCompose, Join
from scrapy.contrib.loader import XPathItemLoader
from my.items import myItem
class mySpider(Spider):
name = "my"
allowed_domains = ["my.com"]
start_urls = ['http://sjg.my.com/cf_jy.cfm']
def parse(self, response):
s = Selector(response)
rows = s.xpath('//table[@class="table-order"]//tr')
for row in rows:
l = XPathItemLoader(item=myItem(), selector=row)
l.default_input_processor = MapCompose(unicode.strip)
l.default_output_processor = Join()
l.add_xpath('title', './/a[contains(@href,"cf_jy.cfm?hu_pg")]/text()')
l.add_xpath('url1', './/a/@href')
l.add_xpath('dates', './/td[4]/text()')
l.add_xpath('rev', './/td[@align="right"]/text()')
l.add_xpath('typ', './/td[3]/text()')
l.add_value('name', u'gsf')
request = Request(l.get_xpath('.//a/@href'), callback=self.parse_link,meta={'l':l})
yield request
def parse_link(self, response):
l = response.meta["l"]
s = Selector(response)
q = s.xpath("//div[@class='content-main']/td[@class='text']/p/text()").extract()
l.add_value('description',q)
yield l.load_item()
Thanks in advance.
print l.get_xpath('.//a/@href')
to see what you get.