BeautifulSoup 4をPythonで掻き集める - 初心者

Question

最後の質問（here）に感謝してくれてありがとう。しかし、私は現在、私の最終的なデータフレームを準備しています。私は元のテーブルからすべてのデータを抽出してそれを表示することができましたが、現在はホームチームとアウェイチームをdfに追加したいが、それを把握することはできないようだ。ここには私が現在持っているものがあり、here is the site私は掻きたいです。

from urllib.request import urlopen # import the library 
from bs4 import BeautifulSoup # Import BS 
from bs4 import SoupStrainer # Import Soup Strainer 
import pandas as pd # import pandas as a package 

basescrape = 'http://www.footywire.com/afl/footy/ft_match_statistics?mid=' 
matchid = '6172' 

scrapeweb1 = basescrape+matchid 

page = urlopen(scrapeweb1) # access the website 
only_tables = SoupStrainer('table', attrs={"width" : "583"}) # parse only table elements when parsing 
soup = BeautifulSoup(page, 'html.parser', parse_only=only_tables) # parse the html 

only_teams = SoupStrainer('table', attrs={"width" : "376"}) # parse only team qtr score elements when parsing 
soup2 = BeautifulSoup(page, 'html.parser', parse_only=only_teams) # parse the html 


# only valid rows with player data in 

table = soup.find_all("tr", attrs={"onmouseover" : "this.bgColor='#cbcdd0';"}) 


# create variables to keep the data in 

hometeam = [] 
awayteam = [] 

player = [] 
kicks = [] 
handballs = [] 
disposals = [] 
marks = [] 
goals = [] 
behinds = [] 
tackles = [] 
hitouts = [] 
inside50s = [] 
freesfor = [] 
freesagainst = [] 
fantasy = [] 
supercoach = [] 

# Find all the <tr> tag pairs, skip the first one, then for each. 
for row in soup.find_all("tr", attrs={"onmouseover" : "this.bgColor='#cbcdd0';"}): 
    # Create a variable of all the <td> tag pairs in each <tr> tag pair, 
    col = row.find_all('td') 

    # Create a variable of the string inside 1st <td> tag pair, 
    column_1 = col[0].string.strip() 
    # and append it to player variable 
    player.append(column_1) 

    # Create a variable of the string inside 2nd <td> tag pair, 
    column_2 = col[1].string.strip() 
    # and append it to kicks variable 
    kicks.append(column_2) 

    # Create a variable of the string inside 3rd <td> tag pair, 
    column_3 = col[2].string.strip() 
    # and append it to handballs variable 
    handballs.append(column_3) 

    # Create a variable of the string inside 4th <td> tag pair, 
    column_4 = col[3].string.strip() 
    # and append it to disposals variable 
    disposals.append(column_4) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_5 = col[4].string.strip() 
    # and append it to marks variable 
    marks.append(column_5) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_6 = col[5].string.strip() 
    # and append it to goals variable 
    goals.append(column_6) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_7 = col[6].string.strip() 
    # and append it to behinds variable 
    behinds.append(column_7) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_8 = col[7].string.strip() 
    # and append it to tackles variable 
    tackles.append(column_8) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_9 = col[8].string.strip() 
    # and append it to hitouts variable 
    hitouts.append(column_9) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_10 = col[9].string.strip() 
    # and append it to inside50s variable 
    inside50s.append(column_10) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_11 = col[10].string.strip() 
    # and append it to freesfo variable 
    freesfor.append(column_11) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_12 = col[11].string.strip() 
    # and append it to freesagainst variable 
    freesagainst.append(column_12) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_13 = col[12].string.strip() 
    # and append it to fantasy variable 
    fantasy.append(column_13) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_14 = col[13].string.strip() 
    # and append it to supercoach variable 
    supercoach.append(column_14) 

# Find all the <tr> tag pairs, then for each. 
for row in soup2.find_all("tr", class_= "leftbold"): 
    # Create a variable of all the <td> tag pairs in each <tr> tag pair, 
    col2 = row.find_all('td') 

    # Create a variable of the string inside 1st <td> tag pair, 
    hometeam = col2[0].string.strip() 
    # and append it to player variable 
    # hometeam.append(column2_1) 

    # Create a variable of the string inside 2nd <td> tag pair, 
    awayteam = col2[1].string.strip() 
    # and append it to kicks variable 
    # awayteam.append(column2_2) 


# Create a variable of the value of the columns 
columns = {'match_id': matchid, 'home_team': hometeam, 'away_team': awayteam, 'player': player, 'kicks': kicks, 'handballs': handballs, 'disposals': disposals, 'marks': marks, 'goals': goals, 'behinds': behinds, 'tackles': tackles, 'hitouts': hitouts, 'inside_50s': inside50s, 'frees_for': freesfor, 'frees_against': freesagainst, 'fantasy': fantasy, 'supercoach': supercoach} 

# Create a dataframe from the columns variable - n 
df = pd.DataFrame(columns, columns = ['match_id', 'home_team', 'away_team', 'player', 'kicks', 'handballs', 'disposals', 'marks', 'goals', 'behinds', 'tackles', 'hitouts', 'inside_50s', 'frees_for', 'frees_against', 'fantasy', 'supercoach']) 

print(df) 

# print(soup.prettify()) 

# print(table) 

明らかに、配列はすべて同じ長さではないため、データフレームは機能しません。ホームと離れたチームを擦って、それを変数に保存して、マッチドイドと同じように動作させるにはどうすればいいですか？

「hometeam」変数を最初の22行に表示させ、23-44行目の「離れた場所」に表示させる方法がありますか？そうすれば、プレーヤーは1つのチームに帰属しますか？

は、私は私が間違ってここにこのセクションをやっている感じ：あなたの助けを

# Find all the <tr> tag pairs, then for each. 
for row in soup2.find_all("tr", class_= "leftbold"): 
    # Create a variable of all the <td> tag pairs in each <tr> tag pair, 
    col2 = row.find_all('td') 

    # Create a variable of the string inside 1st <td> tag pair, 
    hometeam = col2[0].string.strip() 
    # and append it to player variable 
    # hometeam.append(column2_1) 

    # Create a variable of the string inside 2nd <td> tag pair, 
    awayteam = col2[1].string.strip() 
    # and append it to kicks variable 
    # awayteam.append(column2_2) 

どうもありがとうございました。

（また、 "+"を文字列に使用しているので、 ".join"演算子を使用してscrapeweb1を動作させることはできませんでした。それは "ではなく「/ TD」を使用しています

<table border="0" cellspacing="0" cellpadding="0" width="376" id="matchscoretable"> 
<tr> 
<th class="leftbold" height="23" width="100">Team</td> 

...ので、私は、ソースを検査し、そのテーブルの一部の間違ったHTMLがあります表示されます。失敗した、

scrapeweb1 = "".join(basescrape, matchid) 

編集）以下であります/ th "と表示され、美しいスープでパースされるとテーブルタグが閉じます。

[<table border="0" cellpadding="0" cellspacing="0" id="matchscoretable" width="376"> 
<tr> 
<th class="leftbold" height="23" width="100">Team</th></tr></table>] 

私はこの問題を解決するために管理

Answer 1

ここでは、一つの方法ですそれを行うことができます：これを行うために時間を割いて非常に

from urllib.request import urlopen # import the library 
from bs4 import BeautifulSoup # Import BS 
from bs4 import SoupStrainer # Import Soup Strainer 
import pandas as pd # import pandas as a package 

basescrape = 'http://www.footywire.com/afl/footy/ft_match_statistics?mid=' 
matchid = '6172' 
url = ''.join([basescrape,matchid]) 

# changed the table width to 585 to get first row with team name 
only_tables = SoupStrainer('table', attrs={"width" : "585"}) # parse only table elements when parsing 
soup = BeautifulSoup(urlopen(url), 'html.parser', parse_only=only_tables) # parse the html 
# use the table titles as anchor points 
teams = soup.find_all('td', attrs={'class':'innertbtitle', 'align':'left'}) 
# create an empty list for the players 
player_list = [] 
# iterate through anchor points 
for team in teams: 
    # extract team name from the table title 
    team_name = team.text.strip().split(' ', maxsplit=1)[0] 
    # get the rows from the next table relative to anchor point 
    trs = team.find_next('table', attrs={'width':583}).find_all('tr') 
    # create list of labels using first row in table 
    labels = [td.text for td in trs.pop(0).find_all('td')] 
    # iterate through the remaining rows 
    for row in trs: 
     # build dictionary using label as key and text of each cell as value 
     player_dict = {label:value.text for label,value in 
         zip(labels, row.find_all('td'))} 
     # add team name to dictionary 
     player_dict['team'] = team_name 
     # append dictionary to the list 
     player_list.append(player_dict) 

# create the dataframe 
df = pd.DataFrame(player_list) 
print(df) 

Answer 2

を家を取得する別の方法を見て、離れたチームの名前する必要があり、ここに完成したコードは、今だ...

from urllib.request import urlopen # import the library 
from bs4 import BeautifulSoup # Import BS 
from bs4 import SoupStrainer # Import Soup Strainer 
import pandas as pd # import pandas as a package 

basescrape = 'http://www.footywire.com/afl/footy/ft_match_statistics?mid=' 
matchid = '6172' 

scrapeweb1 = basescrape+matchid 

page = urlopen(scrapeweb1) # access the website 
page2 = urlopen(scrapeweb1) # access the website 

only_tables = SoupStrainer('table', attrs={"width" : "583"}) # parse only table elements when parsing 

soup = BeautifulSoup(page, 'html.parser', parse_only=only_tables) # parse the html 
soup2 = BeautifulSoup(page2, 'html.parser') # parse the html 


# only valid rows with player data in 

table = soup.find_all("tr", attrs={"onmouseover" : "this.bgColor='#cbcdd0';"}) 


# create variables to keep the data in 

Table1 = soup2.find_all('table', attrs={'width':"375"})[1] 

hometeam = Table1.find_all('td', attrs={'width':"124"})[0].string.strip() 
awayteam = Table1.find_all('td', attrs={'width':"124"})[1].string.strip() 


player = [] 
kicks = [] 
handballs = [] 
disposals = [] 
marks = [] 
goals = [] 
behinds = [] 
tackles = [] 
hitouts = [] 
inside50s = [] 
freesfor = [] 
freesagainst = [] 
fantasy = [] 
supercoach = [] 

# Find all the <tr> tag pairs, skip the first one, then for each. 
for row in soup.find_all("tr", attrs={"onmouseover" : "this.bgColor='#cbcdd0';"}): 
    # Create a variable of all the <td> tag pairs in each <tr> tag pair, 
    col = row.find_all('td') 

    # Create a variable of the string inside 1st <td> tag pair, 
    column_1 = col[0].string.strip() 
    # and append it to player variable 
    player.append(column_1) 

    # Create a variable of the string inside 2nd <td> tag pair, 
    column_2 = col[1].string.strip() 
    # and append it to kicks variable 
    kicks.append(column_2) 

    # Create a variable of the string inside 3rd <td> tag pair, 
    column_3 = col[2].string.strip() 
    # and append it to handballs variable 
    handballs.append(column_3) 

    # Create a variable of the string inside 4th <td> tag pair, 
    column_4 = col[3].string.strip() 
    # and append it to disposals variable 
    disposals.append(column_4) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_5 = col[4].string.strip() 
    # and append it to marks variable 
    marks.append(column_5) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_6 = col[5].string.strip() 
    # and append it to goals variable 
    goals.append(column_6) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_7 = col[6].string.strip() 
    # and append it to behinds variable 
    behinds.append(column_7) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_8 = col[7].string.strip() 
    # and append it to tackles variable 
    tackles.append(column_8) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_9 = col[8].string.strip() 
    # and append it to hitouts variable 
    hitouts.append(column_9) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_10 = col[9].string.strip() 
    # and append it to inside50s variable 
    inside50s.append(column_10) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_11 = col[10].string.strip() 
    # and append it to freesfo variable 
    freesfor.append(column_11) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_12 = col[11].string.strip() 
    # and append it to freesagainst variable 
    freesagainst.append(column_12) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_13 = col[12].string.strip() 
    # and append it to fantasy variable 
    fantasy.append(column_13) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_14 = col[13].string.strip() 
    # and append it to supercoach variable 
    supercoach.append(column_14) 




# Create a variable of the value of the columns 
columns = {'match_id': matchid, 'home_team': hometeam, 'away_team': awayteam, 'player': player, 'kicks': kicks, 'handballs': handballs, 'disposals': disposals, 'marks': marks, 'goals': goals, 'behinds': behinds, 'tackles': tackles, 'hitouts': hitouts, 'inside_50s': inside50s, 'frees_for': freesfor, 'frees_against': freesagainst, 'fantasy': fantasy, 'supercoach': supercoach} 

# Create a dataframe from the columns variable - n 
df = pd.DataFrame(columns, columns = ['match_id', 'home_team', 'away_team', 'player', 'kicks', 'handballs', 'disposals', 'marks', 'goals', 'behinds', 'tackles', 'hitouts', 'inside_50s', 'frees_for', 'frees_against', 'fantasy', 'supercoach']) 

print(df)