ウィキペディアのテーブルからURLを抽出する

Question

私はhereから取られた以下のコードを使用して、ウィキペディアのテーブルを削ります。テスト情報に加えて、私は各引用/国ごとに各ハイパーリンクを訪問し、それらのページのそれぞれからテキストをコピーしたいと思います。 BeautifulSoupでこれを行う方法はありますか？

# -*- coding: utf-8 -*- 
""" 
Scrape a table from wikipedia using python. Allows for cells spanning multiple rows and/or columns. Outputs csv files for 
each table 
""" 

from bs4 import BeautifulSoup 
import urllib.request 
import os 
import codecs 

# wiki = "https://en.wikipedia.org/wiki/International_Phonetic_Alphabet_chart_for_English_dialects" 
wiki = 'https://en.wikipedia.org/wiki/List_of_national_capitals_in_alphabetical_order' 
header = {'User-Agent': 'Mozilla/5.0'} # Needed to prevent 403 error on Wikipedia 
req = urllib.request.Request(wiki, headers=header) 
page = urllib.request.urlopen(req) 
soup = BeautifulSoup(page, "html.parser") 

tables = soup.findAll("table", {"class": "wikitable"}) 

# show tables 
# for table in tables: 
#  print("###############") 
#  print(table)#.text)#[:100]) 

for tn in range(len(tables)): 
    table = tables[tn] 

    # preinit list of lists 
    rows = table.findAll("tr") 
    row_lengths = [len(r.findAll(['th', 'td'])) for r in rows] 
    ncols = max(row_lengths) 
    nrows = len(rows) 
    data = [] 
    for i in range(nrows): 
     rowD = [] 
     for j in range(ncols): 
      rowD.append('') 
     data.append(rowD) 

    # process html 
    for i in range(len(rows)): 
     row = rows[i] 
     rowD = [] 
     cells = row.findAll(["td", "th"]) 
     for j in range(len(cells)): 
      cell = cells[j] 

      # lots of cells span cols and rows so lets deal with that 
      cspan = int(cell.get('colspan', 1)) 
      rspan = int(cell.get('rowspan', 1)) 
      for k in range(rspan): 
       for l in range(cspan): 
        data[i + k][j + l] += cell.text 

     data.append(rowD) 

     # write data out 

     page = os.path.split(wiki)[1] 
    fname = 'output_{}_t{}.csv'.format(page, tn) 
    f = codecs.open(fname, 'w') # ,encoding='utf-8') 
    for i in range(nrows): 
     rowStr = ','.join(data[i]) 
     rowStr = rowStr.replace('\n', '') 
     # print(rowStr) 
     rowStr = rowStr # .encode('unicode_escape') 
     f.write(rowStr + '\n') 

    f.close() 

Answer 1

from bs4 import BeautifulSoup 
import requests 
wiki_url = 'https://en.wikipedia.org/wiki/List_of_national_capitals_in_alphabetical_order' 
print('Fetching main wiki article: %s' % wiki_url) 
page = requests.get(wiki_url).text 
print('Done. Extracting table links..') 
html = BeautifulSoup(page) 
table = html.find('table', 'wikitable') 
links = table.findAll('a') 
links_content = {} 
print('Done extracting links. About to fetch: %s links..' % len(links)) 
for link in links: 
    print('Fetching: %s' % link) 
    links_content[link] = requests.get(link).text