BeautifulSoup find_all（ "img"）がすべてのサイトで動作しない

Question

私はどのWebサイトからでも画像をダウンロードするためのPythonスクリプトを作成しようとしています。それは働いていますが、一貫していません。特に、find_all（ "img"）は2番目のURLに対してそうしていません。スクリプトは次のとおりです。

# works for http://proof.nationalgeographic.com/2016/02/02/photo-of-the-day-best-of-january-3/ 
# but not http://www.nationalgeographic.com/photography/proof/2017/05/lake-chad-desertification/ 
import requests 
from PIL import Image 
from io import BytesIO 
from bs4 import BeautifulSoup 

def url_to_image(url, filename): 
    # get HTTP response, open as bytes, save the image 
    # http://docs.python-requests.org/en/master/user/quickstart/#binary-response-content 
    req = requests.get(url) 
    i = Image.open(BytesIO(req.content)) 
    i.save(filename) 

# open page, get HTML request and parse with BeautifulSoup 
html = requests.get("http://proof.nationalgeographic.com/2016/02/02/photo-of-the-day-best-of-january-3/") 
soup = BeautifulSoup(html.text, "html.parser") 

# find all JPEGS in our soup and write their "src" attribute to array 
urls = [] 
for img in soup.find_all("img"): 
    if img["src"].endswith("jpg"): 
     print("endswith jpg") 
     urls.append(str(img["src"])) 
    print(str(img)) 

jpeg_no = 00 
for url in urls: 
    url_to_image(url, filename="NatGeoPix/" + str(jpeg_no) + ".jpg") 
    jpeg_no += 1 

Answer 1

イメージは、失敗したページでJavaScriptでレンダリングされます。

例えばを（あなたがWeb-scraping JavaScript page with Pythonを見るdryscrape使用しない場合） まずdryscrape

でページをレンダリング

import requests 
from PIL import Image 
from io import BytesIO 
from bs4 import BeautifulSoup 
import dryscrape 

def url_to_image(url, filename): 
    # get HTTP response, open as bytes, save the image 
    # http://docs.python-requests.org/en/master/user/quickstart/#binary-response-content 
    req = requests.get(url) 
    i = Image.open(BytesIO(req.content)) 
    i.save(filename) 

# open page, get HTML request and parse with BeautifulSoup 

session = dryscrape.Session() 
session.visit("http://www.nationalgeographic.com/photography/proof/2017/05/lake-chad-desertification/") 
response = session.body() 
soup = BeautifulSoup(response, "html.parser") 

# find all JPEGS in our soup and write their "src" attribute to array 
urls = [] 
for img in soup.find_all("img"): 
    if img["src"].endswith("jpg"): 
     print("endswith jpg") 
     urls.append(str(img["src"])) 
     print(str(img)) 

jpeg_no = 00 
for url in urls: 
    url_to_image(url, filename="NatGeoPix/" + str(jpeg_no) + ".jpg") 
    jpeg_no += 1 

しかし、私はまた、あなたが絶対URLではない相対的な1持っていることを確認します：

import requests 
from PIL import Image 
from io import BytesIO 
from bs4 import BeautifulSoup 
import dryscrape 
from urllib.parse import urljoin 


def url_to_image(url, filename): 
    # get HTTP response, open as bytes, save the image 
    # http://docs.python-requests.org/en/master/user/quickstart/#binary-response-content 
    req = requests.get(url) 
    i = Image.open(BytesIO(req.content)) 
    i.save(filename) 

# open page, get HTML request and parse with BeautifulSoup 
base = "http://www.nationalgeographic.com/photography/proof/2017/05/lake-chad-desertification/" 
session = dryscrape.Session() 
session.visit(base) 
response = session.body() 
soup = BeautifulSoup(response, "html.parser") 

# find all JPEGS in our soup and write their "src" attribute to array 
urls = [] 
for img in soup.find_all("img"): 
    if img["src"].endswith("jpg"): 
     print("endswith jpg") 
     urls.append(str(img["src"])) 
     print(str(img)) 

jpeg_no = 00 
for url in urls: 
    if url.startswith('http'): 
     absoute = url 
    else: 
     absoute = urljoin(base, url) 
    print (absoute) 
    url_to_image(absoute, filename="NatGeoPix/" + str(jpeg_no) + ".jpg") 
    jpeg_no += 1