Pythonで擦っている間に複数のページをループする

Question

IDMbで複数のページを解析しようとしています。パーサは、1ページから情報を収集することに固執しています。私はこれを無駄に解決するために多くのフォーラムを試みました。私はそれが私の組み込みループを正しく設定しなかったり、私の最初の要求を設定しないことと関係があると考えています。助けてください。ありがとう。

このスクリプトの問題：1ページのループ。

#Basic libraries 

from requests import get 
from bs4 import BeautifulSoup 
import pandas as pd 
from random import randint 

#More advanced libraries 
from time import sleep 
from time import time 
from IPython.core.display import clear_output 
from warnings import warn 

base_url = 'http://www.imdb.com/search/title?release_date=2000,2017&sort=num_votes,desc&page=' 
response = get(base_url) 

soup = BeautifulSoup(response.text, 'lxml') 

#data lists to append 

names   = [] 
years   = [] 
imdb_ratings = [] 
metascores  = [] 
votes   = [] 

#preparing the monitoring loop 

pages = str(range(1,5)) 

start_time = time() 
requests = 0 

#for every page in the interval 1-4 
for page in pages: 

#make a get request 
    response = get(base_url + page) 

#pause the loop 
    sleep(randint(8,15)) 

#Monitor the requests 
    requests += 1 
    elapsed_time = time() - start_time 
    if requests > 4: 
     warn:('Number of requests was greater than expected.') 
     break 

    elif response.status_code != 200: 
     warn('Request: {}; Frequency: {} requests/s'.format(requests, response.status_code)) 

    else: 
     print('Request: {}; Frequency: {} requests/s'.format(requests, requests/elapsed_time)) 
     clear_output(wait = True) 

    page_html = BeautifulSoup(response.text, 'lxml') 

#root 
    movie_containers = soup.find_all('div', class_= 'lister-item mode-advanced') 

#looping through containers 
for container in movie_containers: 
    if container.find('div', class_ = 'ratings-metascore') is not None: 

     #The name 
     name = container.h3.a.text 
     #print(name) 
     names.append(name) 

     #The Year 
     year = container.find('span', class_ = 'lister-item-year').text 
     #print(year) 
     years.append(year) 

     #IDMb rating    
     imdb = container.strong.text 
     #print(imdb) 
     imdb_ratings.append(imdb) 

     #Metascore    
     metascore = container.find('span', class_= 'metascore').text 
     #print(metascore) 
     metascores.append(int(metascore)) 

     #Number of Votes 
     vote = container.find('span', attrs = {'name':'nv'})['data-value'] 
     #print(vote) 
     votes.append(int(vote)) 


#keeping track of data   
test_df= pd.DataFrame({'Movie': names, 
         'Year': years, 
         'IMDb': imdb_ratings, 
         'Metascore': metascores, 
         'Votes': votes}) 
print(test_df.info()) 
test_df 

`

Answer 1

解決方法1：あなたは何ができるか は、あなたがそのページ上のデータを収集し、作業が完了し、いつでもある、+へのURL page値を変更することにより、次のページに移動1。

http://www.imdb.com/search/title? 
release_date=2000,2017&sort=num_votes,desc&page=2&ref_=adv_nxt 

解決策2：ページの下部にnext URLをクリックすることで同じ動作を得ることができます。これを行うには、ページの一番下にスクロールする必要があります。

はここで終わり、私はそれを把握test.csv

#Basic libraries 

from requests import get 
from bs4 import BeautifulSoup 
import pandas as pd 
from random import randint 

#More advanced libraries 
from time import sleep 
from time import time 
from IPython.core.display import clear_output 
from warnings import warn 

base_url = 'http://www.imdb.com/search/title?release_date=2000,2017&sort=num_votes,desc&page=' 
response = get(base_url) 

soup = BeautifulSoup(response.text, 'lxml') 

#data lists to append 

names   = [] 
years   = [] 
imdb_ratings = [] 
metascores  = [] 
votes   = [] 

#preparing the monitoring loop 

pages = str(range(1,5)) 

start_time = time() 
requests = 0 

#for every page in the interval 1-4 
urls = [base_url+str(x) for x in range(0,10)] 
for url in urls: 

#make a get request 
    response = get(url) 

#pause the loop 
    sleep(randint(2,3)) 

#Monitor the requests 
    requests += 1 
    elapsed_time = time() - start_time 
    if requests > 4: 
     warn('Number of requests was greater than expected.') 
     break 

    elif response.status_code != 200: 
     warn('Request: {}; Frequency: {} requests/s'.format(requests, response.status_code)) 

    else: 
     print('Request: {}; Frequency: {} requests/s'.format(requests, requests/elapsed_time)) 
     clear_output(wait = True) 

    page_html = BeautifulSoup(response.text, 'lxml') 

#root 
    movie_containers = soup.find_all('div', class_= 'lister-item mode-advanced') 

    #looping through containers 
    for container in movie_containers: 
     if container.find('div', class_ = 'ratings-metascore') is not None: 

      #The name 
      name = container.h3.a.text 
      #print(name) 
      names.append(name) 

      #The Year 
      year = container.find('span', class_ = 'lister-item-year').text 
      #print(year) 
      years.append(year) 

      #IDMb rating    
      imdb = container.strong.text 
      #print(imdb) 
      imdb_ratings.append(imdb) 

      #Metascore    
      metascore = container.find('span', class_= 'metascore').text 
      #print(metascore) 
      metascores.append(int(metascore)) 

      #Number of Votes 
      vote = container.find('span', attrs = {'name':'nv'})['data-value'] 
      #print(vote) 
      votes.append(int(vote)) 


#keeping track of data   
test_df= pd.DataFrame({'Movie': names, 
         'Year': years, 
         'IMDb': imdb_ratings, 
         'Metascore': metascores, 
         'Votes': votes}) 
print(test_df.info()) 
test_df.to_csv("test.csv", sep=",", encoding="utf-8") 

Answer 2

で出力し、あなたの修正コード、です。ループの最後にページ+ = 1を追加し、各データエントリの末尾にコンマを追加するだけです。

pages = 1 
ranger = range(1,4) 
requests = 0 

for n in ranger: 

#make a get request 

    response = get(base_url + str(pages)) 

    soup = BeautifulSoup(response.text, 'lxml') 

#pause the loop 
    sleep(randint(2,3)) 

#Monitor the requests 
    requests += 1 
    elapsed_time = time() - start_time 
    if requests > 4: 
     warn('Number of requests was greater than expected.') 
     break 

    if response.status_code != 200: 
     warn('Request: {}; Frequency: {} requests/s'.format(requests, response.status_code)) 

    else: 
     print('Request: {}; Frequency: {} requests/s'.format(requests, requests/elapsed_time)) 
     clear_output(wait = True) 

#root 
    movie_containers = soup.find_all('div', class_= 'lister-item mode-advanced') 

#looping through containers 
    for container in movie_containers: 
     if container.find('div', class_ = 'ratings-metascore') is not None: 

     #The name 
      name = container.h3.a.text 
     #print(name) 
      names.append(name + ',') 

     #The Year 
      year = container.find('span', class_ = 'lister-item-year').text 
      years.append(year + ',') 

     #IDMb rating    
      imdb = container.strong.text 
     #print(imdb) 
      imdb_ratings.append(imdb + ',') 

     #Metascore    
      metascore = container.find('span', class_= 'metascore').text 
      metascores.append(metascore + ',') 

     #Number of Votes 
      vote = container.find('span', attrs = {'name':'nv'})['data-value'] 
      votes.append(vote + ',') 

    pages += 1