スクラップを使用したさまざまな製品情報の盗用

Question

以下は、製品情報をスクラップするためのコードです。ページには多くの商品があります。私はそれらをすべて擦り取り、次のページに移動します。問題は、ページ上のすべての製品を反復処理するのではなく、ページ上の最初の製品のみを選択していることです。どこが間違っていますか？

import re 
import time 
import sys 
from scrapy.spider import BaseSpider 
from scrapy.selector import Selector 
from scrapy.http import Request 
import parsedatetime 
from datetime import datetime 
from airline_sentiment.items import * 
from airline_sentiment.spiders.crawlerhelper import * 

class TripAdvisorRestaurantBaseSpider(BaseSpider): 
    name = "shoebuy" 

    allowed_domains = ["shoebuy.com"] 
    base_uri = "http://www.shoebuy.com" 
    start_urls = [ 
       base_uri + "/womens-leather-boots/category_2493?cm_sp=cat-_-d_womensboots_tiles_b1_leather-_-092216" 
       ] 


    def parse(self, response): 

     sel = Selector(response) 

     snode_airline = sel.xpath('//*[starts-with(@class, "pt_grid")]/div[starts-with(@class, "pt_product\")]') 

     for snode_restaurant in snode_airline: 
      tripadvisor_item = AirlineSentimentItem() 

      tripadvisor_item['url'] = self.base_uri + clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "pt_info")]/a/@href')) 

      tripadvisor_item['name'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "pt_info")]/a/span[@class="pt_title"]/text()')) 
      tripadvisor_item['price'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "pt_prices")]/span[@class="pt_price"]/text()')) 
      tripadvisor_item['discount'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "pt_prices")]/div[@class="pt_discount"]/span[@class="pt_percent_off"]/text()')) 
      tripadvisor_item['orig_price'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "pt_prices")]/div[@class="pt_discount"]/span[@class="pt_price_orig"]/text()')) 
      tripadvisor_item['stars'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//*[@class="bv-rating-ratio"]/span/span[3]/text()')) 
      tripadvisor_item['reviews'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "bv-inline-rating-container")]/dl/dd[2]/span/text()')) 

      yield Request(url=tripadvisor_item['url'], meta={'tripadvisor_item': tripadvisor_item}, callback=self.parse_fetch_review) 


     next_page_url = clean_parsed_string(get_parsed_string(sel, '//div[@class="paging"]/a[@class="next"]/@href')) 
     if next_page_url and len(next_page_url) > 0: 
      yield Request(url=self.base_uri + next_page_url, meta={'tripadvisor_item': tripadvisor_item}, callback=self.parse_next_page) 

    def parse_next_page(self, response): 
     sel = Selector(response) 

     snode_airline = sel.xpath('//*[starts-with(@class, "pt_grid")]/div[starts-with(@class, "pt_product")]') 

     for snode_restaurant in snode_airline: 

      tripadvisor_item = AirlineSentimentItem() 

      tripadvisor_item['url'] = self.base_uri + clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "pt_info")]/a/@href')) 
      tripadvisor_item['name'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "pt_info")]/a/span[@class="pt_title"]/text()')) 
      tripadvisor_item['price'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "pt_prices")]/span[@class="pt_price"]/text()')) 
      tripadvisor_item['discount'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "pt_prices")]/div[@class="pt_discount"]/span[@class="pt_percent_off"]/text()')) 
      tripadvisor_item['orig_price'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "pt_prices")]/div[@class="pt_discount"]/span[@class="pt_price_orig"]/text()')) 
      tripadvisor_item['stars'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//*[@class="bv-rating-ratio"]/span/span[3]/text()')) 
      tripadvisor_item['reviews'] = clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "bv-inline-rating-container")]/dl/dd[2]/span/text()')) 

      yield Request(url=tripadvisor_item['url'], meta={'tripadvisor_item': tripadvisor_item}, callback=self.parse_fetch_review) 

     next_page_url = clean_parsed_string(get_parsed_string(sel, '//div[@class="paging"]/a[@class="next"]/@href')) 
     if next_page_url and len(next_page_url) > 0: 
      yield Request(url=self.base_uri + next_page_url, meta={'tripadvisor_item': tripadvisor_item}, callback=self.parse_next_page) 

    def parse_fetch_review(self, response): 

     tripadvisor_item = response.meta['tripadvisor_item'] 
     sel = Selector(response) 

     snode_reviews = sel.xpath('//*[starts-with(@class, "product_info_wrapper")]') 

     for snode_review in snode_reviews: 

      tripadvisor_item['img'] = self.base_uri + clean_parsed_string(get_parsed_string(snode_review, '//div[starts-with(@class,"large_thumb")]/img/@src')) 

      tripadvisor_item['desc'] = clean_parsed_string(get_parsed_string(snode_review, '//*[starts-with(@class,"product_information")]/div[1]/span/text()')) 

      tripadvisor_item['brand'] = clean_parsed_string(get_parsed_string(snode_review, '//div[starts-with(@class,"seo_module")]/h3/text()')) 

     yield tripadvisor_item 

Answer 1

これは、障害のあるラインです：あなたは、XPathの相対をしなかったので

'.//div[starts-with(@class, "pt_info")]/a/@href' 

：

 tripadvisor_item['url'] = self.base_uri + clean_parsed_string(get_parsed_string(snode_restaurant, '//div[starts-with(@class, "pt_info")]/a/@href')) 

XPathは相対的なノードを示すために.//divよう.で始まる必要がありますあなたのノード（ '。'記法付き）には、ページ上のすべてのアイテムのURLとして常に最初の商品リンクがあります。今すぐscrapyに自動重複URLフィルタがあるので、レビューを取得するためのすべてのリクエストが後で除外され、最初の項目を取得するだけです。

Tl; dr：あなたの//の前に.を相対xpathに追加するだけです。