ウェブサイトからデータを取得するためのウェブスクラブ

Question

私は、各ページに10のプロパティリストを持つ、ウェブサイトをスクラップしようとするpython &を学んでいます。各ページの各リスティングから情報を抽出したいと思います。 次のように最初の5ページのための私のコードは次のとおりです。 - 上記のコードは私のために働いていない

import requests 
from bs4 import BeautifulSoup 

urls = [] 
for i in range(1,5): 
    pages = "http://www.realcommercial.com.au/sold/property-offices-retail-showrooms+bulky+goods-land+development-hotel+leisure-medical+consulting-other-in-vic/list-{0}?includePropertiesWithin=includesurrounding&activeSort=list-date&autoSuggest=true".format(i) 
    urls.append(pages) 
    for info in urls: 
     page = requests.get(info) 
     soup = BeautifulSoup(page.content, 'html.parser') 
     links = soup.find_all('a', attrs ={'class' :'details-panel'}) 
     hrefs = [link['href'] for link in links] 
     Data = [] 
     for urls in hrefs: 
      pages = requests.get(urls) 
      soup_2 =BeautifulSoup(pages.content, 'html.parser') 
      Address_1 = soup_2.find_all('p', attrs={'class' :'full-address'}) 
      Address = [Address.text.strip() for Address in Address_1] 
      Date = soup_2.find_all('li', attrs ={'class' :'sold-date'}) 
      Sold_Date = [Sold_Date.text.strip() for Sold_Date in Date] 
      Area_1 =soup_2.find_all('ul', attrs={'class' :'summaryList'}) 
      Area = [Area.text.strip() for Area in Area_1] 
      Agency_1=soup_2.find_all('div', attrs={'class' :'agencyName ellipsis'}) 
      Agency_Name=[Agency_Name.text.strip() for Agency_Name in Agency_1] 
      Agent_1=soup_2.find_all('div', attrs={'class' :'agentName ellipsis'}) 
      Agent_Name=[Agent_Name.text.strip() for Agent_Name in Agent_1] 
      Data.append(Sold_Date+Address+Area+Agency_Name+Agent_Name) 

。目的を達成するための正しいコーディングを教えてください。

Answer 1

あなたのコードには、変数 "urls"を2回宣言したという問題が1つあります。あなたは、以下のようなコードを更新する必要があります。代わりに.formatのコードと使用文字列連結で

import requests 
from bs4 import BeautifulSoup 

urls = [] 
for i in range(1,6): 
    pages = "http://www.realcommercial.com.au/sold/property-offices-retail-showrooms+bulky+goods-land+development-hotel+leisure-medical+consulting-other-in-vic/list-{0}?includePropertiesWithin=includesurrounding&activeSort=list-date&autoSuggest=true".format(i) 
    urls.append(pages) 

Data = [] 
for info in urls: 
    page = requests.get(info) 
    soup = BeautifulSoup(page.content, 'html.parser') 
    links = soup.find_all('a', attrs ={'class' :'details-panel'}) 
    hrefs = [link['href'] for link in links] 

    for href in hrefs: 
     pages = requests.get(href) 
     soup_2 =BeautifulSoup(pages.content, 'html.parser') 
     Address_1 = soup_2.find_all('p', attrs={'class' :'full-address'}) 
     Address = [Address.text.strip() for Address in Address_1] 
     Date = soup_2.find_all('li', attrs ={'class' :'sold-date'}) 
     Sold_Date = [Sold_Date.text.strip() for Sold_Date in Date] 
     Area_1 =soup_2.find_all('ul', attrs={'class' :'summaryList'}) 
     Area = [Area.text.strip() for Area in Area_1] 
     Agency_1=soup_2.find_all('div', attrs={'class' :'agencyName ellipsis'}) 
     Agency_Name=[Agency_Name.text.strip() for Agency_Name in Agency_1] 
     Agent_1=soup_2.find_all('div', attrs={'class' :'agentName ellipsis'}) 
     Agent_Name=[Agent_Name.text.strip() for Agent_Name in Agent_1] 
     Data.append(Sold_Date+Address+Area+Agency_Name+Agent_Name) 

print Data 

Answer 2

利用ヘッダ（I）

コードがすることができます。この

import requests 
from bs4 import BeautifulSoup 

urls = [] 
for i in range(1,6): 
    pages = 'http://www.realcommercial.com.au/sold/property-offices-retail-showrooms+bulky+goods-land+development-hotel+leisure-medical+consulting-other-in-vic/list-'i+'?includePropertiesWithin=includesurrounding&activeSort=list-date&autoSuggest=true' 
    urls.append(pages) 

Data = [] 
for info in urls: 
    headers = {'User-agent':'Mozilla/5.0'} 
    page = requests.get(info,headers=headers) 
    soup = BeautifulSoup(page.content, 'html.parser') 
    links = soup.find_all('a', attrs ={'class' :'details-panel'}) 
    hrefs = [link['href'] for link in links] 

for href in hrefs: 
    pages = requests.get(href) 
    soup_2 =BeautifulSoup(pages.content, 'html.parser') 
    Address_1 = soup_2.find_all('p', attrs={'class' :'full-address'}) 
    Address = [Address.text.strip() for Address in Address_1] 
    Date = soup_2.find_all('li', attrs ={'class' :'sold-date'}) 
    Sold_Date = [Sold_Date.text.strip() for Sold_Date in Date] 
    Area_1 =soup_2.find_all('ul', attrs={'class' :'summaryList'}) 
    Area = [Area.text.strip() for Area in Area_1] 
    Agency_1=soup_2.find_all('div', attrs={'class' :'agencyName ellipsis'}) 
    Agency_Name=[Agency_Name.text.strip() for Agency_Name in Agency_1] 
    Agent_1=soup_2.find_all('div', attrs={'class' :'agentName ellipsis'}) 
    Agent_Name=[Agent_Name.text.strip() for Agent_Name in Agent_1] 
    Data.append(Sold_Date+Address+Area+Agency_Name+Agent_Name) 

print Data 

Answer 3

のように見えますあなたのコードをより安全にするために、BeautifulSoupにhrefを含むリンクを与えるように指示してください。また、ページ番号を含めるようにURLを変更するのではなく、下部にnext >リンクを抽出することもできます。

import requests 
from bs4 import BeautifulSoup 

base_url = r"http://www.realcommercial.com.au" 
url = base_url + "/sold/property-offices-retail-showrooms+bulky+goods-land+development-hotel+leisure-medical+consulting-other-in-vic/list-1?includePropertiesWithin=includesurrounding&activeSort=list-date&autoSuggest=true" 
data = [] 

for _ in range(10): 
    print(url) 
    page = requests.get(url) 
    soup = BeautifulSoup(page.content, 'html.parser') 
    hrefs = [link['href'] for link in soup.find_all('a', attrs={'class' : 'details-panel'}, href=True)] 

    for href in hrefs: 
     pages = requests.get(href) 
     soup_2 = BeautifulSoup(pages.content, 'html.parser') 
     Address_1 = soup_2.find_all('p', attrs={'class' :'full-address'}) 
     Address = [Address.text.strip() for Address in Address_1] 
     Date = soup_2.find_all('li', attrs ={'class' :'sold-date'}) 
     Sold_Date = [Sold_Date.text.strip() for Sold_Date in Date] 
     Area_1 = soup_2.find_all('ul', attrs={'class' :'summaryList'}) 
     Area = [Area.text.strip() for Area in Area_1] 
     Agency_1 = soup_2.find_all('div', attrs={'class' :'agencyName ellipsis'}) 
     Agency_Name = [Agency_Name.text.strip() for Agency_Name in Agency_1] 
     Agent_1 = soup_2.find_all('div', attrs={'class' :'agentName ellipsis'}) 
     Agent_Name = [Agent_Name.text.strip() for Agent_Name in Agent_1] 

     data.append(Sold_Date+Address+Area+Agency_Name+Agent_Name) 

    # Find next page (if any) 
    next_button = soup.find('li', class_='rui-pagination-next') 

    if next_button: 
     url = base_url + next_button.parent['href'] 
    else: 
     break 


for entry in data: 
    print(entry) 
    print("---------")