ELK+syslog+nginx访问日志收集+分词处理
一、nginx访问日志配置:
1、日志格式配置：
log_format json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"size":$body_bytes_sent,'
'"xff":"$http_x_forwarded_for",'
'"upstreamhost":"$upstream_addr",'
'"status":"$status",'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"http_host":"$host",'
'"url":"$uri"}';
2、访问日志配置：
access_log syslog:server=xx.xx.xx.xx:5140 json;
二、logstash配置
input {
syslog{
port => "5140"
}
}
filter {
json {
source =>"message"
remove_field => ["message"]
}
}
output {
elasticsearch { hosts => ['xx.xx.xx.xx:9200','xx.xx.xx.xx:9200']
index => 'nginx_rfd-%{+YYYY.MM.dd.HH}'
template => "/usr/local/logstash/nginx.json"
template_name => "nginx_*"
template_overwrite => true
}
}
注意index名称要包含到template_name里
三、自定义mappings
cat /usr/local/logstash/nginx.json
{
"template": "nginx_*",
"order":1,
"settings": { "index.refresh_interval" : "60s" },
"mappings": {
"_default_": {
"_all" : { "enabled" : false },
"properties": {
"@timestamp" : { "type" : "date" },
"@version" : { "type" : "integer", "index" : "not_analyzed" },
"url": {
"type": "string",
"index": "not_analyzed"
},
"host": {
"type": "ip",
"index": "not_analyzed"
},
"clientip": {
"type": "ip",
"index": "not_analyzed"
},
"size": {
"type": "integer"
},
"xff": {
"type": "string",
"index": "not_analyzed"
},
"upstreamhost": {
"type": "string",
"index": "not_analyzed"
},
"http_host": {
"type": "string",
"index": "not_analyzed"
},
"status": {
"type": "integer"
},
"responseTime": {
"type": "string",
"index": "not_analyzed"
},
"upstreamtime": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
四、生成统计图形
五、参考文档
https://elasticsearch.cn/article/154
http://blog.csdn.net/choelea/article/details/53320140
http://www.cnblogs.com/hanyifeng/p/5860731.html
