一、docker部署
考虑到docker的优势,如一致的运行环境,轻松的迁移等,我们采用docker部署filebeat。下面是docker-compose部署的例子
3.1、配置docker-compose文件
version: "3"
services:
elasticsearch:
image: elasticsearch:7.11.1
container_name: elasticsearch
hostname: elasticsearch1
environment:
- discovery.type=single-node
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ports:
- 9200:9200
- 9300:9300
networks:
- "elk-net"
kibana:
image: docker.elastic.co/kibana/kibana:7.1.1
environment:
- SERVER_NAME=kibana
- ELASTICSEARCH_URL=http://elasticsearch1:9200
- XPACK_MONITORING_ENABLED=true
ports:
- "5601:5601"
networks:
- "elk-net"
depends_on:
- "elasticsearch"
filebeat:
image: docker.elastic.co/beats/filebeat:7.1.1
volumes:
- ./filebeat/logs/nginx:/var/log/nginx/
- ./filebeat/logs/biz:/var/log/biz/
- ./filebeat/logs/log4j:/var/log/log4j/
- ./filebeat/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
networks:
- "elk-net"
depends_on:
- "elasticsearch"
- "kibana"
networks:
elk-net:
这里将采集log4j,nginx和业务日志到es,然后展示到kibana
配置filebeat
# 详情如下:
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/*.log
scan_frequency: 10s
tail_files: true
fields:
index_name: "nginx_log"
- type: log
enabled: true
#不以[开头的行都合并到上一行的末尾
multiline.type: pattern
multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^Caused by:'
multiline.negate: false
multiline.match: after
paths:
- /var/log/log4j/*.log
fields:
index_name: "log4j_log"
- type: log
enabled: true
multiline.type: pattern
multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^Caused by:'
multiline.negate: false
multiline.match: after
fields:
index_name: "biz_log"
scan_frequency: 10s
pipeline: "extract-traceid-pipeline"
paths:
- /var/log/biz/*.log
# # # 7.x的版本中需要禁用此索引生命周期,否则在指定es索引名字的时候会有问题
setup.ilm.enabled: false
setup.template.name: "my-log"
setup.template.pattern: "my-*"
setup.template.enabled: true
setup.template.overwrite: false
# 输出到es
output.elasticsearch:
#worker: 1
#bulk_max_size: 1500
hosts: ["elasticsearch1:9200"]
index: "pb-%{[fields.index_name]}-*"
indices:
- index: "pb-nginx-%{+yyyy.MM.dd}"
when.equals:
fields.index_name: "nginx_log"
- index: "pb-log4j-%{+yyyy.MM.dd}"
when.equals:
fields.index_name: "log4j_log"
- index: "pb-biz-%{+yyyy.MM.dd}"
when.equals:
fields.index_name: "biz_log"