forループを使用しない2つのデータセットの行のファジーマッチング

Question

2つのデータセットAとBがそれぞれ8つのcoloumnsを持っています。データセットAには942個の行があり、データセットBには5079個の行があります。データセットAとデータセットBを比較し、ファジーマッチングを行う必要があります。データセットBに一致する行がある場合は、データセットAの「一致」を追加列にマークする必要があります。

私はRには比較的新しいので、forloopの代わりにlapply、mapply、またはsapplyでrコードを最適化する方法がわからない。あなたは音声マッチングを実行することを可能にするパッケージRecordLinkage、見たいかもしれませんそれぞれcmd_newleads_selcolsとc4cleads_selcolsのための私のコードです

############################## 
# Install Necessary Packages # 
############################## 


#install.packages("openxlsx") 
#install.packages("stringdist") 
#install.packages("XLConnect") 


############################## 
#  Load Packages  # 
############################## 


library(openxlsx) 
library(stringdist) 
library(XLConnect) 


cmd_newleads <- read.xlsx("Src/CMD - New Leads to Load.xlsx", sheet = "Top Leads Full Data", startRow = 1, colNames = TRUE) 
cmd_newleads[c("Lead_Match","Opportunity_Match")] <- "" 
c4c_leads <- read.xlsx("Src/C4C - Leads.xlsx", sheet = "Leads", startRow = 1, colNames = TRUE) 
#c4c_opportunities <- read.xlsx("Src/C4C - Opportunities Data 6-24-16.xlsx", sheet = "Export 06-24-2016 04.55.46 PM", startRow = 1, colNames = TRUE) 


cmd_newleads_selcols <- cmd_newleads[,c("project_name","project_address","project_city","project_state_province_region_code","project_postalcode","project_country","project_sector","project_type")] 
cmd_newleads_selcols[is.na(cmd_newleads_selcols)] <- "" 
#rownames(cmd_newleads_selcols) 

c4cleads_selcols <- c4c_leads[,c("Lead","Address1.(Lead)","City.(Lead)","Region.(Lead)","Postal.Code.(Lead)","Country.(Lead)","Sector.(Lead)","Type.(Lead)")] 
c4cleads_selcols[is.na(c4cleads_selcols)] <- "" 
#cmd_c4copportunities_selcols <- c4c_opportunities[,c("project_name","project_address","project_city","project_state_province_region_code","project_postalcode","project_country","project_sector","project_type")] 

rcount_cmdnewleads <- nrow(cmd_newleads) 
rcount_c4cleads <- nrow(c4c_leads) 
#rcount_c4copportunities <- nrow(c4c_opportunities) 




for(i in 1:rcount_cmdnewleads) 
{ 

    cmd_project_name <- cmd_newleads_selcols[i,1] 
    cmd_project_address <- cmd_newleads_selcols[i,2] 
    cmd_project_city <- cmd_newleads_selcols[i,3] 
    cmd_project_region_code <- cmd_newleads_selcols[i,4] 
    cmd_project_postalcode <- cmd_newleads_selcols[i,5] 
    cmd_project_country <- cmd_newleads_selcols[i,6] 
    cmd_project_sector <- cmd_newleads_selcols[i,7] 
    cmd_project_type <- cmd_newleads_selcols[i,8] 


    for(j in 1:rcount_c4cleads) 
    { 

     c4cleads_project_name <- c4cleads_selcols[j,1] 
     c4cleads_project_address <- c4cleads_selcols[j,2] 
     c4cleads_project_city <- c4cleads_selcols[j,3] 
     c4cleads_project_region_code <- c4cleads_selcols[j,4] 
     c4cleads_project_postalcode <- c4cleads_selcols[j,5] 
     c4cleads_project_country <- c4cleads_selcols[j,6] 
     c4cleads_project_sector <- c4cleads_selcols[j,7] 
     c4cleads_project_type <- c4cleads_selcols[j,8] 

     project_percent <- stringsim(cmd_project_name,c4cleads_project_name, method="dl", p=0.1) 
     address_percent <- stringsim(cmd_project_address,c4cleads_project_address, method="dl", p=0.1) 
     city_percent <- stringsim(cmd_project_city,c4cleads_project_city, method="dl", p=0.1) 
     region_percent <- stringsim(cmd_project_region_code,c4cleads_project_region_code, method="dl", p=0.1) 
     postalcode_percent <- stringsim(cmd_project_postalcode,c4cleads_project_postalcode, method="dl", p=0.1) 
     country_percent <- stringsim(cmd_project_country,c4cleads_project_country, method="dl", p=0.1) 
     sector_percent <- stringsim(cmd_project_sector,c4cleads_project_sector, method="dl", p=0.1) 
     type_percent <- stringsim(cmd_project_type,c4cleads_project_type, method="dl", p=0.1) 

     if(project_percent > 0.833 && address_percent > 0.833 && city_percent > 0.833 && region_percent > 0.833 && postalcode_percent > 0.833 && country_percent > 0.833 && sector_percent > 0.833 && type_percent > 0.833) 
     { 
     cmd_newleads[i,51] <- c4cleads[j,c4cleads$Lead.ID] 
     } 
     else 
     { 
     cmd_newleads[i,51] <- "New Lead" 
     } 

    } 

} 

サンプル・データ

project_name project_address project_city 
1  Wynn Mystic Casino & Hotel 22 Chemical Ln  Everett 
2 Northpoint Complex Development  East Street Cambridge 
3 Northpoint Complex Development  East Street Cambridge 
4 Northpoint Complex Development  East Street Cambridge 
5 Northpoint Complex Development  East Street Cambridge 
6 Northpoint Complex Development  East Street Cambridge 
    project_state_province_region_code project_postalcode 
1         MA    02149 
2         MA    02138 
3         MA    02138 
4         MA    02138 
5         MA    02138 
6         MA    02138 
      project_country project_sector project_type 
1 United States of America Hospitality New Building 
2 United States of America  Apartments New Building 
3 United States of America  Apartments New Building 
4 United States of America  Apartments New Building 
5 United States of America  Apartments New Building 
6 United States of America  Apartments New Building 







Lead   Address1.(Lead) City.(Lead) Region.(Lead) Postal.Code.(Lead) Country.(Lead) 
1 1 Hotel Brooklyn Bridge Park Old Fulton St & Furman St Brooklyn  New York    11201 United States 
2  10 Trinity Square Hotel   10 Trinity Square  London    #    EC3P United Kingdom 
3     100 Stewart   1900 1st Avenue  Seattle Washington    98101 United States 
4    1136 S Wabash       #   #    #     # Not assigned 
5   115-129 37th Street  115-129 37th Street Union CIty New Jersey     # United States 
6    1418 W Addison   1418 w Addison  Chicago    #    60613 Not assigned 
      Sector.(Lead)  Type.(Lead) 
1   Hospitality  New Building 
2   Hospitality Brand Conversion 
3   Hospitality  New Building 
4 High Rise Residential  New Building 
5    Developer  New Building 
6 High Rise Residential  New Building 

Answer 1

、確率レコードリンケージ後

機械学習のアプローチが含まれます。

Answer 2

効率の問題が発生している場合は、forループを使用しているためではありません。主な問題は、の2つのデータセットにある可能なすべての組み合わせの行を大量に処理していることです。より効率的な言語機能を使用すると、少し速くなるかもしれませんが、多くの不必要な計算をしているという事実は変わりません。

データマッチングの問題を効率的に改善する方法の1つは、は、不必要な計算を減らすために明らかな不一致を避けることです。たとえば、内部ループを変更して、最初にいくつかの重要な条件をチェックすることができます。スコアが低い（つまり、明らかに不一致である）場合、残りの属性について類似度スコアを計算する必要はありません。例えば

：

for(i in 1:rcount_cmdnewleads) 
{ 

    cmd_project_name <- cmd_newleads_selcols[i,1] 
    ... 

for(j in 1:rcount_c4cleads) 
    { 

    c4cleads_project_name <- c4cleads_selcols[j,1] 
    project_percent <- stringsim(cmd_project_name,c4cleads_project_name, method="dl", p=0.1) 
    if (project_percent < .83) { 

     # you already know that this is a non-match, so go to the next one 
     next 

    } else { 

     # check the rest of the values! 
     ... 

    } 
    } 
} 

私はR RecordLinkageパッケージに慣れていないんだけど、Pythonのrecordlinkageパッケージには、効率を高めるためにプロセスの初期段階で明らかに非マッチを除外するためのツールがあります。明白な不一致を除外することによって、レコードリンケージのスピードアップについての詳細は、this tutorialを参照してください。