0
logstash 1.5.3をlogstash 2.3.4にアップグレードしようとしています。 1.5.3ためLogstash 2.3.4 elasticsearchにテンプレートをインストールしようとしたときに動かなくなった
logstashの設定は以下のようになります。 - ：私はインデックスjabong-*のカスタムテンプレートを使用していますelasticsearchで
input {
kafka {
zk_connect => "kafka:2181"
group_id => "logstash"
topic_id => "logstash_logs2"
reset_beginning => false
consumer_threads => 3
}
}
filter {
if [app] == "walle_slowquery" or [app] == "walle_slowindex" {
ruby {
code => "event['timestamp'] = event['@timestamp']"
}
}
grok {
match => [
"timestamp", "^(?<app_log_time>%{YEAR}-%{MONTHNUM}-%{MONTHDAY})"
]
}
mutate {
rename => {
"app_log_time" => "[@metadata][app_log_time]"
}
}
}
output {
if [env] == "prod" or [env] == "common" {
elasticsearch_java {
#For daily index creation used the time notation, Remove if not required.
index => "jabong-%{env}-%{app}-%{iver}-%{[@metadata][app_log_time]}"
cluster => "elasticsearch"
network_host => "172.16.84.230"
hosts => ["172.16.84.230:9300"]
protocol => "transport"
}
file {
path => "/var/log/shop/%{env}/%{app}/%{app}_%{host}_%{[@metadata][app_log_time]}.log"
}
stdout { codec => rubydebug }
}
}
- ：
input {
kafka {
zk_connect => "kafka:2181"
group_id => "logstash"
topic_id => "logstash_logs2"
reset_beginning => false
consumer_threads => 3
}
}
filter {
if [app] == "walle_slowquery" or [app] == "walle_slowindex" {
ruby {
code => "event['timestamp'] = event['@timestamp']"
}
}
grok {
match => [
"timestamp", "^(?<app_log_time>%{YEAR}-%{MONTHNUM}-%{MONTHDAY})"
]
}
mutate {
rename => {
"app_log_time" => "[@metadata][app_log_time]"
}
}
}
output {
if [env] == "prod" or [env] == "common" {
elasticsearch {
index => "jabong-%{env}-%{app}-%{iver}-%{[@metadata][app_log_time]}"
cluster => "elasticsearch"
host => ["172.16.84.230:9300"]
protocol => "transport"
}
file {
path => "/var/log/shop/%{env}/%{app}/%{app}_%{host}_%{[@metadata][app_log_time]}.log"
}
stdout { codec => rubydebug }
}
}
私は2.3.4のために、以下のようlogstashの設定をmodifed -
{
"order": 0,
"template": "jabong-*",
"settings": {
"index.refresh_interval": "5s"
},
"mappings": {
"_default_": {
"dynamic_templates": [
{
"string_fields": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"type": "string",
"fields": {
"raw": {
"ignore_above": 256,
"index": "not_analyzed",
"type": "string"
}
}
},
"match_mapping_type": "string",
"match": "*"
}
}
],
"_all": {
"omit_norms": true,
"enabled": true
},
"properties": {
"geoip": {
"dynamic": true,
"type": "object",
"properties": {
"location": {
"type": "geo_point"
}
}
},
"@version": {
"index": "not_analyzed",
"type": "string"
}
}
}
},
"aliases": {}
}
Kafka 0.8とlogstash-input-kafkaプラグインを使用しているのは2.0.8で、Elasticsearchはバージョン1.7.1です。
私は冗長モードでlogstashを起動した場合、logstashは以下のメッセージで立ち往生しているように見える： -
[email protected]:/opt/logstash-2.3.4# bin/logstash -f conf/logstash_indexer.conf --debug
Reading config file {:config_file=>"/opt/logstash-2.3.4/conf/logstash_indexer.conf", :level=>:debug, :file=>"logstash/config/loader.rb", :line=>"69", :method=>"local_config"}
Plugin not defined in namespace, checking for plugin file {:type=>"input", :name=>"kafka", :path=>"logstash/inputs/kafka", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
Plugin not defined in namespace, checking for plugin file {:type=>"codec", :name=>"json", :path=>"logstash/codecs/json", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
config LogStash::Codecs::JSON/@charset = "UTF-8" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@zk_connect = "kafka:2181" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@group_id = "logstash" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@topic_id = "logstash_logs2" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@reset_beginning = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@consumer_threads = 3 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@codec = <LogStash::Codecs::JSON charset=>"UTF-8"> {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@white_list = nil {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@black_list = nil {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@auto_offset_reset = "largest" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@auto_commit_interval_ms = 1000 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@queue_size = 20 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@rebalance_max_retries = 4 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@rebalance_backoff_ms = 2000 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@consumer_timeout_ms = -1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@consumer_restart_on_error = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@consumer_restart_sleep_ms = 0 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@decorate_events = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@consumer_id = nil {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@fetch_message_max_bytes = 1048576 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@decoder_class = "kafka.serializer.DefaultDecoder" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@key_decoder_class = "kafka.serializer.DefaultDecoder" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
Plugin not defined in namespace, checking for plugin file {:type=>"filter", :name=>"ruby", :path=>"logstash/filters/ruby", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
config LogStash::Filters::Ruby/@code = "event['timestamp'] = event['@timestamp']" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Ruby/@add_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Ruby/@remove_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Ruby/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Ruby/@remove_field = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Ruby/@periodic_flush = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
Plugin not defined in namespace, checking for plugin file {:type=>"filter", :name=>"grok", :path=>"logstash/filters/grok", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
config LogStash::Filters::Grok/@match = {"timestamp"=>"^(?<app_log_time>%{YEAR}-%{MONTHNUM}-%{MONTHDAY})"} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@add_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@remove_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@remove_field = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@periodic_flush = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@patterns_dir = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@patterns_files_glob = "*" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@break_on_match = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@named_captures_only = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@keep_empty_captures = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@singles = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@tag_on_failure = ["_grokparsefailure"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@overwrite = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
Plugin not defined in namespace, checking for plugin file {:type=>"filter", :name=>"mutate", :path=>"logstash/filters/mutate", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
config LogStash::Filters::Mutate/@rename = {"app_log_time"=>"[@metadata][app_log_time]"} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Mutate/@add_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Mutate/@remove_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Mutate/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Mutate/@remove_field = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Mutate/@periodic_flush = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
Plugin not defined in namespace, checking for plugin file {:type=>"output", :name=>"elasticsearch_java", :path=>"logstash/outputs/elasticsearch_java", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
Plugin not defined in namespace, checking for plugin file {:type=>"output", :name=>"file", :path=>"logstash/outputs/file", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
Plugin not defined in namespace, checking for plugin file {:type=>"output", :name=>"stdout", :path=>"logstash/outputs/stdout", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
starting agent {:level=>:info, :file=>"logstash/agent.rb", :line=>"207", :method=>"execute"}
starting pipeline {:id=>"main", :level=>:info, :file=>"logstash/agent.rb", :line=>"469", :method=>"start_pipeline"}
Settings: Default pipeline workers: 12
log4j java properties setup {:log4j_level=>"DEBUG", :level=>:debug, :file=>"logstash/logging.rb", :line=>"89", :method=>"setup_log4j"}
Registering kafka {:group_id=>"logstash", :topic_id=>"logstash_logs2", :zk_connect=>"kafka:2181", :level=>:info, :file=>"logstash/inputs/kafka.rb", :line=>"133", :method=>"register"}
Running kafka {:group_id=>"logstash", :topic_id=>"logstash_logs2", :zk_connect=>"kafka:2181", :level=>:info, :file=>"logstash/inputs/kafka.rb", :line=>"140", :method=>"run"}
Plugin not defined in namespace, checking for plugin file {:type=>"codec", :name=>"plain", :path=>"logstash/codecs/plain", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
config LogStash::Codecs::Plain/@charset = "UTF-8" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@index = "jabong-%{env}-%{app}-%{iver}-%{[@metadata][app_log_time]}" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@cluster = "elasticsearch" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@network_host = "172.16.84.230" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@hosts = ["172.16.84.230:9300"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@protocol = "transport" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@codec = <LogStash::Codecs::Plain charset=>"UTF-8"> {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@workers = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@manage_template = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@template_name = "logstash" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@template_overwrite = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@parent = nil {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@flush_size = 500 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@idle_flush_time = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@upsert = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@doc_as_upsert = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@max_retries = 3 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@script = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@script_type = "inline" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@script_lang = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@script_var_name = "event" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@scripted_upsert = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@retry_max_interval = 2 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@retry_max_items = 500 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@retry_on_conflict = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@pipeline = nil {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@action = "index" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@max_inflight_requests = 50 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@sniffing = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
Using mapping template from {:path=>nil, :level=>:info, :file=>"logstash/outputs/elasticsearch/template_manager.rb", :line=>"6", :method=>"install_template"}
Attempting to install template {:manage_template=>{"template"=>"logstash-*", "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true, "omit_norms"=>true}, "dynamic_templates"=>[{"message_field"=>{"match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"string", "index"=>"analyzed", "omit_norms"=>true, "fielddata"=>{"format"=>"disabled"}}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"string", "index"=>"analyzed", "omit_norms"=>true, "fielddata"=>{"format"=>"disabled"}, "fields"=>{"raw"=>{"type"=>"string", "index"=>"not_analyzed", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"string", "index"=>"not_analyzed"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"float"}, "longitude"=>{"type"=>"float"}}}}}}}, :level=>:info, :file=>"logstash/outputs/elasticsearch/template_manager.rb", :line=>"8", :method=>"install_template"}
誰かが私が間違って何が起こっているかを知らせることができますか？