解析ログ

Question

私はGROKといくつかのログを解析しようとしているが、ログの行は時々同じように見ていないときにそれをやって、いくつかの問題を抱えています

...

マイログファイルは、このようなルックスを言うことができます：

[2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager) 
[2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager) 
[2017-02-03 19:26:20,605] INFO Rolled new log segment for \'omega-replica-sync-dev-8\' in 21 ms. (kafka.log.Log) 
[2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log) 
[2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log) 

私の現在のノードのコードは次のようになります。

'use strict'; 

var nodegrok = require('node-grok'); 
var Regex = require("regex"); 
var zlib = require('zlib'); 

var msg = '[2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:26:20,605] INFO Rolled new log segment for \'omega-replica-sync-dev-8\' in 21 ms. (kafka.log.Log)\n[2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log)\n[2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log)' 

console.log('message: ', msg); 

var p2 = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} \\[%{DATA:message1}\\]: %{GREEDYDATA:message2}' 

var lines = msg.toString().split('\n'); 

for(var i = 0;i < lines.length;i++){ 

    console.log('line [i]:', lines[i]) 
    var str = lines[i] 

    var patterns = require('node-grok').loadDefaultSync(); 
    var pattern = patterns.createPattern(p2) 
    console.log('pattern:', pattern.parseSync(lines[i])); 

} 

が、最後の2が出力nullに思える...そのパターンに第三の部分が欠落しているので。

line [i]: [2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager) 
pattern: { timestamp: '2017-02-03 19:15:51,112', 
    level: 'INFO', 
    message1: 'Group Metadata Manager on Broker 1', 
    message2: 'Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)' } 
line [i]: [2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager) 
pattern: { timestamp: '2017-02-03 19:25:51,112', 
    level: 'INFO', 
    message1: 'Group Metadata Manager on Broker 1', 
    message2: 'Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)' } 
line [i]: [2017-02-03 19:26:20,605] INFO Rolled new log segment for 'omega-replica-sync-dev-8' in 21 ms. (kafka.log.Log) 
pattern: null 
line [i]: [2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log) 
pattern: null 
line [i]: [2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log) 
pattern: null 

さまざまな形式の行をgrokでフォーマットするにはどうすればよいですか？

Answer 1

ここでは、私が作業しなければならないことの1つの方法です...本質的に、パターンがif文と一致するかどうかを調べて、評価しますが、ログの6つの可能なフォーマットがある場合はどうなりますか？それで入れ子になっているステートメントがあれば、私はそれを正しく6なければならないのですか？私にとって効率的な方法のように聞こえる...良い方法がありますか？

'use strict'; 

var nodegrok = require('node-grok'); 
var Regex = require("regex"); 
var zlib = require('zlib'); 

var msg = '[2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:26:20,605] INFO Rolled new log segment for \'omega-replica-sync-dev-8\' in 21 ms. (kafka.log.Log)\n[2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log)\n[2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log)' 

console.log('message: ', msg); 

var p2 = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} \\[%{DATA:message1}\\]: %{GREEDYDATA:message2}' 

var lines = msg.toString().split('\n'); 

for(var i = 0;i < lines.length;i++){ 

    console.log('line [i]:', lines[i]) 
    var str = lines[i] 
    var p = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} \\[%{DATA:message1}\\]: %{GREEDYDATA:message2}' 
    var p2 = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} %{GREEDYDATA:message2}' 

    var patterns = require('node-grok').loadDefaultSync(); 
    var pattern = patterns.createPattern(p) 

    if (pattern.parseSync(lines[i]) == null) { 
     var pattern = patterns.createPattern(p2) 
     console.log('patternf:', pattern.parseSync(lines[i])); 

    } else { 
     console.log('pattern:', pattern.parseSync(lines[i])); 
    } 

}