基于Docker应用容器日志文件收集

系统

Centos7.0 X86_64 以上

FROM centos

MAINTAINER jaymarco@shsnc.com

#Install system library

RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && rpm -ivh http://rpms.remirepo.net/enterprise/remi-release-7.rpm && \

yum install -y php70-php-gd.x86_64 php70-php-mcrypt.x86_64 php70-php-fpm.x86_64 php70-php-pecl-redis.x86_64 python-setuptools \

php70-php-mbstring.x86_64 php70-php-snmp.x86_64 php70-php-pecl-zip.x86_64 php70-php-xml.x86_64 \

php70-php-mysqlnd.x86_64 php70-php-pecl-mysql.x86_64 gcc gcc-c++ automake libtool make cmake openssl openssl-devel pcre-devel && \

yum clean all

#Make install nginx

RUN rpm -ivh http://nginx.org/packages/centos/7/x86_64/RPMS/nginx-1.10.3-1.el7.ngx.x86_64.rpm

COPY nginx.conf /etc/nginx/nginx.conf

#set php www.conf config

RUN sed -e 's/127.0.0.1:9000/9000/' \

-e '/allowed_clients/d' \

-e '/catch_workers_output/s/^;//' \

-e '/error_log/d' \

-e 's/;listen.backlog = 511/listen.backlog = 1024/' \

-e 's/pm.max_children = 50/pm.max_children = 300/' \

-e 's/pm.start_servers = 5/pm.start_servers = 30/' \

-e 's/pm.min_spare_servers = 5/pm.min_spare_servers = 30/' \

-e 's/pm.max_spare_servers = 35/pm.max_spare_servers = 60/' \

-e 's/;pm.max_requests = 500/pm.max_requests = 10240/' \

-e 's/;request_slowlog_timeout = 0/request_slowlog_timeout = 2/' \

-e 's/;request_terminate_timeout = 0/request_terminate_timeout = 20/' \

-e 's/;rlimit_files = 1024/rlimit_files = 65535/' \

-i /etc/opt/remi/php70/php-fpm.d/www.conf && \

sed -e 's/max_execution_time = 30/max_execution_time = 150/' \

-e 's/max_input_time = 60/max_input_time = 300/' \

-i /etc/opt/remi/php70/php.ini && \

sed -e 's/daemonize = yes/daemonize = no/' \

-e 's/;rlimit_files = 1024/rlimit_files = 65535/' -i /etc/opt/remi/php70/php-fpm.conf && \

cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime &&\

echo 'Asia/Shanghai' >/etc/timezone

RUN easy_install supervisor && \

mkdir -p /var/log/supervisor && \

mkdir -p /var/run/sshd && \

mkdir -p /var/run/supervisord

#Add supervisord conf

ADD supervisord.conf /etc/supervisord.conf

#copy start script

ADD startserv.sh /startserv.sh

RUN chmod +x /startserv.sh

#Set port

EXPOSE 9000

# For collecting logs, install filebeat plugin

RUN mkdir /logs

RUN ln -s /var/log/nginx /logs/

RUN ln -s /var/opt/remi/php70/log/php-fpm /logs

ADD filebeat-6.0.0-linux-x86_64.tar.gz /var/log/

RUN chmod +x /var/log/filebeat/filebeat

#Start web server

#ENTRYPOINT ["/var/log/filebeat/init.sh"]

CMD ["/startserv.sh"]

user nginx;

worker_processes 2;

worker_cpu_affinity auto;

error_log /var/log/nginx/error.log error;

worker_rlimit_nofile 10240;

worker_priority -2;

events {

use epoll;

accept_mutex on;

worker_connections 10240;

}

http {

include mime.types;

default_type application/octet-stream;

log_format main '$remote_addr - $remote_user [$time_local] "$request" '

'$status $body_bytes_sent "$http_referer" '

'"$http_user_agent" "$http_x_forwarded_for"'

'upstream_addr:"$upstream_addr" '

'upstream_cache_status:"$upstream_cache_status" '

'upstream_status:"$upstream_status" ';

access_log /var/log/nginx/access.log main;

sendfile on;

sendfile_max_chunk 512k;

aio threads;

directio 4m;

keepalive_timeout 65;

open_log_file_cache max=1000 inactive=20s valid=1m min_uses=2;

gzip on;

gzip_comp_level 4;

gzip_disable "MSIE [1-6].";

gzip_min_length 10k;

gzip_http_version 1.0;

gzip_types text/plain text/css text/xml text/javascript application/xml application/x-javascript application/xml+rss application/javascript application/json;

gzip_vary on;

client_max_body_size 2m;

include /etc/nginx/conf.d/*.conf;

}

[unix_http_server]

file=/tmp/supervisor.sock ; (the path to the socket file)

[supervisord]

logfile=/tmp/supervisord.log ; (main log file;default $CWD/supervisord.log)

logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB)

logfile_backups=10 ; (num of main logfile rotation backups;default 10)

loglevel=info ; (log level;default info; others: debug,warn,trace)

pidfile=/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid)

nodaemon=true ; (start in foreground if true;default false)

minfds=1024 ; (min. avail startup file descriptors;default 1024)

minprocs=200 ; (min. avail process descriptors;default 200)

user=root ;

; the below section must remain in the config file for RPC

; (supervisorctl/web interface) to work, additional interfaces may be

; added by defining them in separate rpcinterface: sections

[rpcinterface:supervisor]

supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisorctl]

serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket

[program:php-fpm]

command=/opt/remi/php70/root/usr/sbin/php-fpm -F

[program:nginx]

command=/usr/sbin/nginx -c /etc/nginx/nginx.conf

[program:filebeat]

command=/var/log/filebeat/filebeat -c /var/log/filebeat/filebeat.yml

#!/bin/sh

ip=`ip a|grep -w inet|grep -v -w lo|awk '{print $2}'|awk -F'/' '{print $1}'`

LOGS="/logs/"

#FILE=`ls -l $LOGS |awk '/^d/ {print $NF}'`

FILE=`ls $LOGS`

HOME="/var/log/filebeat"

BAK="$HOME/bak"

CONF="$HOME/filebeat.yml"

HOST_NAME=`hostname`

cp $BAK $CONF

for name in $FILE

do

sed -i "/paths/a\ - $LOGS$name/*.log" $CONF

done

sed -i "s/#APP_ID#/$APP_ID/g" $CONF

sed -i "s/#ip#/$ip/g" $CONF

sed -i "s/#SERVICE_ID#/$SERVICE_ID/g" $CONF

sed -i "s/#HOST_NAME#/$HOST_NAME/g" $CONF

sed -i "s/#LOGSTASH_HOST#/$LOGSTASH_HOST/g" $CONF

/usr/bin/supervisord -n -c /etc/supervisord.conf

filebeat:

spool_size: 10240

idle_timeout: "10s"

prospectors:

-

paths:

- /logs/php-fpm/*.log

- /logs/nginx/*.log

fields:

app_id: "6db116df"

service_id: "_6db116df_64a00233"

host_name: "139b3e343614"

fields_under_root: true

tail_files: true

document_type: "172.17.0.2"

processors:

- drop_fields:

fields: ["input_type", "beat", "offset"]

output.logstash:

hosts: ["XX.XX.XX.XX:5044"]

worker: 2