小売業者のウェブサイト上のすべての製品を掻き集める

Question

私たちは、Forever 21のウェブサイト上のすべてのカテゴリのすべての製品を掻き取ろうとしています。製品ページがあれば、必要な情報を抽出する方法を知っており、カテゴリを指定すると、すべての製品を抽出できます。ただし、すべての製品カテゴリをクロールする方法はわかりません。ここでは、特定のカテゴリの我々のコードであり、すべての製品を取得：

import requests 
from bs4 import BeautifulSoup 
import json 
#import re 

params = {"action": "getcategory", 
      "br": "f21", 
      #"category": re.compile('\S+'), 
      "category": "dress", 
      "pageno": 1, 
      "pagesize": "", 
      "sort": "", 
      "fsize": "", 
      "fcolor": "", 
      "fprice": "", 
      "fattr": ""} 

url = "http://www.forever21.com/Ajax/Ajax_Category.aspx" 
js = requests.get(url, params=params).json() 
soup = BeautifulSoup(js[u'CategoryHTML'], "html.parser") 
i = 0 
j = 0 

while len(soup.select("div.item_pic a")) != 0: 
    for a in soup.select("div.item_pic a"): 
     #print a["href"] 
     i = i + 1 

    params["pageno"] = params["pageno"] + 1 
    j = j + 1 
    js = requests.get(url, params=params).json() 
    soup = BeautifulSoup(js[u'CategoryHTML'], "html.parser") 

print i 
print j 

あなたがコメントで見ることができるように、我々はカテゴリの正規表現を使用しようとしましたが、まったく成功していません。 iとjは単なる製品とページカウンタです。すべての製品カテゴリを取得するために、このコードを変更/追加する方法に関する提案はありますか？

Answer 1

あなたは、カテゴリページをこすりし、ナビゲーションメニューからすべてのサブカテゴリを取得することができます。

import requests 
from bs4 import BeautifulSoup 


url = "http://www.forever21.com/Product/Category.aspx?br=f21&category=app-main" 
response = requests.get(url, headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36"}) 

soup = BeautifulSoup(response.content, "html.parser") 
menues = [li["class"][0] for li in soup.select("#has_sub .white nav ul > li")] 
print(menues) 

プリント：

[u'women-new-arrivals', u'want_list', u'dress', u'top_blouses', u'outerwear_coats-and-jackets', u'bottoms', u'intimates_loungewear', u'activewear', u'swimwear_all', u'acc', u'shoes', u'branded-shop-women-clothing', u'sale_women|women', u'women-new-arrivals-clothing-dresses', u'women-new-arrivals-clothing-tops', u'women-new-arrivals-clothing-outerwear', u'women-new-arrivals-clothing-bottoms', u'women-new-arrivals-clothing-intimates-loungewear', u'women-new-arrivals-clothing-swimwear', u'women-new-arrivals-clothing-activewear', u'women-new-arrivals-accessories|women-new-arrivals', u'women-new-arrivals-shoes|women-new-arrivals', u'promo-web-exclusives', u'promo-best-sellers-app', u'backinstock-women', u'promo-shop-by-outfit-women', u'occasion-shop-wedding', u'contemporary-main', u'promo-basics', u'21_items', u'promo-summer-forever', u'promo-coming-soon', u'dress_casual', u'dress_romper', u'dress_maxi', u'dress_midi', u'dress_mini', u'occasion-shop-dress', u'top_blouses-off-shoulder', u'top_blouses-lace-up', u'top_bodysuits-bustiers', u'top_graphic-tops', u'top_blouses-crop-top', u'top_t-shirts', u'sweater', u'top_blouses-sweatshirts-hoodies', u'top_blouses-shirts', u'top_plaids', u'outerwear_bomber-jackets', u'outerwear_blazers', u'outerwear_leather-suede', u'outerwear_jean-jackets', u'outerwear_lightweight', u'outerwear_utility-jackets', u'outerwear_trench-coats', u'outerwear_faux-fur', u'promo-jeans-refresh|bottoms', u'bottoms_pants', u'bottoms_skirt', u'bottoms_shorts', u'bottoms_shorts-active', u'bottoms_leggings', u'bottoms_sweatpants', u'bottom_jeans|', u'intimates_loungewear-bras', u'intimates_loungewear-panties', u'intimates_loungewear-bodysuits-slips', u'intimates_loungewear-seamless', u'intimates_loungewear-accessories', u'intimates_loungewear-sets', u'activewear_top', u'activewear_sports-bra', u'activewear_bottoms', u'activewear_accessories', u'swimwear_tops', u'swimwear_bottoms', u'swimwear_one-piece', u'swimwear_cover-ups', u'acc_features', u'acc_jewelry', u'acc_handbags', u'acc_glasses', u'acc_hat', u'acc_hair', u'acc_legwear', u'acc_scarf-gloves', u'acc_home-and-gift-items', u'shoes_features', u'shoes_boots', u'shoes_high-heels', u'shoes_sandalsflipflops', u'shoes_wedges', u'shoes_flats', u'shoes_oxfords-loafers', u'shoes_sneakers', u'Shoes_slippers', u'branded-shop-new-arrivals-women', u'branded-shop-women-clothing-dresses', u'branded-shop-women-clothing-tops', u'branded-shop-women-clothing-outerwear', u'branded-shop-women-clothing-bottoms', u'branded-shop-women-clothing-intimates', u'branded-shop-women-accessories|branded-shop-women-clothing', u'branded-shop-women-accessories-jewelry|', u'branded-shop-shoes-women|branded-shop-women-clothing', u'branded-shop-sale-women', u'/brandedshop/brandlist.aspx', u'promo-branded-boho-me', u'promo-branded-rare-london', u'promo-branded-selfie-leslie', u'sale-newly-added', u'sale_dresses', u'sale_tops', u'sale_outerwear', u'sale_sweaters', u'sale_bottoms', u'sale_intimates', u'sale_swimwear', u'sale_activewear', u'sale_acc', u'sale_shoes', u'the-outlet', u'sale-under-5', u'sale-under-10', u'sale-under-15'] 

注brとcategory GETパラメータの値を。 f21は「女性」カテゴリ、app-mainはカテゴリのメインページです。