エラスティックは断片全体で異なるクエリーノルムを検索します

Question

私はむしろESの方が新しく、検索結果の品質を向上させるためにESでスコアリングを勉強しています。私はqueryNorm関数がシャード間で非常に異なる（5倍の大きさの）状況に遭遇しました。私は、クエリ内の用語に対するidfの依存関係を見ることができます。これは、断片間で異なる場合があります。しかし、私の場合、私は単一の検索用語+断片間のidf測定値はお互いに近い（確かにX 5倍の差を生じさせるには不十分である）。私は、私の質問とExplainエンドポイントからの結果を含む私のセットアップを簡単に説明します。

セットアップ 5つのシャードに分散している〜6500個のドキュメントのインデックスがあります。私は、以下のクエリに現れるフィールドにはインデックス時間がないことを言及します。私のセットアップでは、ES 2.4で "query_then_fetch"を使用しています。マイクエリ：

{ 
    "query" : { 
    "bool" : { 
     "must" : [ { 
     "bool" : { 
      "must" : [ ], 
      "must_not" : [ ], 
      "should" : [ { 
       "multi_match" : { 
        "query" : "pds", 
        "fields" : [ "field1" ], 
        "lenient" : true, 
        "fuzziness" : "0" 
       } 
      }, { 
       "multi_match" : { 
        "query" : "pds", 
        "fields" : [ "field2" ], 
        "lenient" : true, 
        "fuzziness" : "0", 
        "boost" : 1000.0 
       } 
      }, { 
       "multi_match" : { 
        "query" : "pds", 
        "fields" : [ "field3" ], 
        "lenient" : true, 
        "fuzziness" : "0", 
        "boost" : 500.0 
       } 
      }, { 
       "multi_match" : { 
        "query" : "pds", 
        "fields" : [ "field4" ], 
        "lenient" : true, 
        "fuzziness" : "0", 
        "boost": 100.0 
       } 
      } ], 
     "must_not" : [ ], 
     "should" : [ ], 
     "filter" : [ ] 
    } 
    }, 
    "size" : 1000, 
    "min_score" : 0.0 
} 

は（他の1ほどの大きクエリノルム5X時間を有する1）文書の2のための出力を説明する：

{ 
    "_shard" : 4, 
    "_explanation" : { 
    "value" : 2.046937, 
    "description" : "product of:", 
    "details" : [ { 
     "value" : 4.093874, 
     "description" : "sum of:", 
     "details" : [ { 
     "value" : 0.112607226, 
     "description" : "weight(field1:pds in 93) [PerFieldSimilarity], result of:", 
     "details" : [ { 
      "value" : 0.112607226, 
      "description" : "score(doc=93,freq=1.0), product of:", 
      "details" : [ { 
      "value" : 0.019996, 
      "description" : "queryWeight, product of:", 
      "details" : [ { 
       "value" : 2.0, 
       "description" : "boost", 
       "details" : [ ] 
      }, { 
       "value" : 5.6314874, 
       "description" : "idf(docFreq=11, maxDocs=1232)", 
       "details" : [ ] 
      }, { 
       "value" : 0.0017753748, 
       "description" : "queryNorm", 
       "details" : [ ] 
      } ] 
      }, { 
      "value" : 5.6314874, 
      "description" : "fieldWeight in 93, product of:", 
      "details" : [ { 
       "value" : 1.0, 
       "description" : "tf(freq=1.0), with freq of:", 
       "details" : [ { 
       "value" : 1.0, 
       "description" : "termFreq=1.0", 
       "details" : [ ] 
       } ] 
      }, { 
       "value" : 5.6314874, 
       "description" : "idf(docFreq=11, maxDocs=1232)", 
       "details" : [ ] 
      }, { 
       "value" : 1.0, 
       "description" : "fieldNorm(doc=93)", 
       "details" : [ ] 
      } ] 
      } ] 
     } ] 
     }, { 
     "value" : 3.9812667, 
     "description" : "weight(field4:pds in 93) [PerFieldSimilarity], result of:", 
     "details" : [ { 
      "value" : 3.9812667, 
      "description" : "score(doc=93,freq=2.0), product of:", 
      "details" : [ { 
      "value" : 0.9998001, 
      "description" : "queryWeight, product of:", 
      "details" : [ { 
       "value" : 100.0, 
       "description" : "boost", 
       "details" : [ ] 
      }, { 
       "value" : 5.6314874, 
       "description" : "idf(docFreq=11, maxDocs=1232)", 
       "details" : [ ] 
      }, { 
       "value" : 0.0017753748, 
       "description" : "queryNorm", 
       "details" : [ ] 
      } ] 
      }, { 
      "value" : 3.9820628, 
      "description" : "fieldWeight in 93, product of:", 
      "details" : [ { 
       "value" : 1.4142135, 
       "description" : "tf(freq=2.0), with freq of:", 
       "details" : [ { 
       "value" : 2.0, 
       "description" : "termFreq=2.0", 
       "details" : [ ] 
       } ] 
      }, { 
       "value" : 5.6314874, 
       "description" : "idf(docFreq=11, maxDocs=1232)", 
       "details" : [ ] 
      }, { 
       "value" : 0.5, 
       "description" : "fieldNorm(doc=93)", 
       "details" : [ ] 
      } ] 
      } ] 
     } ] 
     } ] 
    }, { 
     "value" : 0.5, 
     "description" : "coord(2/4)", 
     "details" : [ ] 
    } ] 
    } 
}, 
{ 
    "_shard" : 2, 
    "_explanation" : { 
    "value" : 0.4143453, 
    "description" : "product of:", 
    "details" : [ { 
     "value" : 0.8286906, 
     "description" : "sum of:", 
     "details" : [ { 
     "value" : 0.018336227, 
     "description" : "weight(field1:pds in 58) [PerFieldSimilarity], result of:", 
     "details" : [ { 
      "value" : 0.018336227, 
      "description" : "score(doc=58,freq=1.0), product of:", 
      "details" : [ { 
      "value" : 0.0030464241, 
      "description" : "queryWeight, product of:", 
      "details" : [ { 
       "value" : 2.0, 
       "description" : "boost", 
       "details" : [ ] 
      }, { 
       "value" : 6.0189342, 
       "description" : "idf(docFreq=11, maxDocs=1815)", 
       "details" : [ ] 
      }, { 
       "value" : 2.5307006E-4, 
       "description" : "queryNorm", 
       "details" : [ ] 
      } ] 
      }, { 
      "value" : 6.0189342, 
      "description" : "fieldWeight in 58, product of:", 
      "details" : [ { 
       "value" : 1.0, 
       "description" : "tf(freq=1.0), with freq of:", 
       "details" : [ { 
       "value" : 1.0, 
       "description" : "termFreq=1.0", 
       "details" : [ ] 
       } ] 
      }, { 
       "value" : 6.0189342, 
       "description" : "idf(docFreq=11, maxDocs=1815)", 
       "details" : [ ] 
      }, { 
       "value" : 1.0, 
       "description" : "fieldNorm(doc=58)", 
       "details" : [ ] 
      } ] 
      } ] 
     } ] 
     }, { 
     "value" : 0.81035435, 
     "description" : "weight(field4:pds in 58) [PerFieldSimilarity], result of:", 
     "details" : [ { 
      "value" : 0.81035435, 
      "description" : "score(doc=58,freq=2.0), product of:", 
      "details" : [ { 
      "value" : 0.1523212, 
      "description" : "queryWeight, product of:", 
      "details" : [ { 
       "value" : 100.0, 
       "description" : "boost", 
       "details" : [ ] 
      }, { 
       "value" : 6.0189342, 
       "description" : "idf(docFreq=11, maxDocs=1815)", 
       "details" : [ ] 
      }, { 
       "value" : 2.5307006E-4, 
       "description" : "queryNorm", 
       "details" : [ ] 
      } ] 
      }, { 
      "value" : 5.3200364, 
      "description" : "fieldWeight in 58, product of:", 
      "details" : [ { 
       "value" : 1.4142135, 
       "description" : "tf(freq=2.0), with freq of:", 
       "details" : [ { 
       "value" : 2.0, 
       "description" : "termFreq=2.0", 
       "details" : [ ] 
       } ] 
      }, { 
       "value" : 6.0189342, 
       "description" : "idf(docFreq=11, maxDocs=1815)", 
       "details" : [ ] 
      }, { 
       "value" : 0.625, 
       "description" : "fieldNorm(doc=58)", 
       "details" : [ ] 
      } ] 
      } ] 
     } ] 
     } ] 
    }, { 
     "value" : 0.5, 
     "description" : "coord(2/4)", 
     "details" : [ ] 
    } ] 
    } 
} 

お知らせどのシャードでの文書からfield1にqueryNorm 4は "0.0017753748"（idf 5.6314874）であり、シャード2のdocの同じフィールドのqueryNormは "0.0002.5307006"（idf 6.0189342）です。私はhttp://lucene.apache.org/core/4_0_0/core/org/apache/lucene/search/similarities/TFIDFSimilarity.htmlの数式を使用してqueryNormの計算を手作業で行いましたが、同じ回答を達成することはできませんでした。

queryNormについては、あまりにも多くのスレッドや投稿を見たことがありません。私が有用だと分かったのはhttp://www.openjems.com/tag/querynorm/です（これは実際Solrですが、クエリーは "query_then_fetch"なのでLuceneの計算だけが重要なので、同様に動作するはずです）。しかし、私は同じ方法を使用して右のqueryNormの値を導き出すことができませんでした（私が理解すると速く、私の場合はt.getBoost（）はインデックス時間フィールドのブースト+特別なフィールドブーストなし）。

ここで何が起こっているのか、誰にも示唆はありますか？

Answer 1

あなたはdfs_query_then_fetch等しくなるようにsearch_typeを設定することができます。

この場合

{ 
    "search_type": "dfs_query_then_fetch", 
    "query": { 
     "bool": { 
      "must": [ 
       { 
        "bool": { 
         "must": [], 
         "must_not": [], 
         "should": [ 
          { 
           "multi_match": { 
            "query": "pds", 
            "fields": [ 
             "field1" 
            ], 
            "lenient": true, 
            "fuzziness": "0" 
           } 
          }, 
          { 
           "multi_match": { 
            "query": "pds", 
            "fields": [ 
             "field2" 
            ], 
            "lenient": true, 
            "fuzziness": "0", 
            "boost": 1000.0 
           } 
          } 
         ] 
        } 
       }, 
       { 
        "multi_match": { 
         "query": "pds", 
         "fields": [ 
          "field3" 
         ], 
         "lenient": true, 
         "fuzziness": "0", 
         "boost": 500.0 
        } 
       }, 
       { 
        "multi_match": { 
         "query": "pds", 
         "fields": [ 
          "field4" 
         ], 
         "lenient": true, 
         "fuzziness": "0", 
         "boost": 100.0 
        } 
       } 
      ], 
      "must_not": [], 
      "should": [], 
      "filter": [] 
     } 
    }, 
    "size": 1000, 
    "min_score": 0.0 
} 

すべての標準値は、グローバルになります。ただし、クエリのパフォーマンスに影響を与える可能性があります。インデックスが小さい場合は、単一のシャードでインデックスを作成することもできます。しかし、さらに多くの文書がある場合、これらの値は異なるものでなければなりません。