feat: 新增elasticsearch、filebeat、kibana组合的日志系统
This commit is contained in:
@@ -0,0 +1,59 @@
|
|||||||
|
version: '3'
|
||||||
|
|
||||||
|
# 网桥 -> 方便相互通讯
|
||||||
|
networks:
|
||||||
|
efk:
|
||||||
|
|
||||||
|
services:
|
||||||
|
elasticsearch:
|
||||||
|
image: registry.cn-shanghai.aliyuncs.com/odboy/elasticsearch:7.14.1 # 原镜像`elasticsearch:7.14.1`
|
||||||
|
container_name: efk_elasticsearch # 容器名为'efk_elasticsearch'
|
||||||
|
restart: unless-stopped # 指定容器退出后的重启策略为始终重启，但是不考虑在Docker守护进程启动时就已经停止了的容器
|
||||||
|
volumes: # 数据卷挂载路径设置,将本机目录映射到容器目录
|
||||||
|
- "./elasticsearch/data:/usr/share/elasticsearch/data"
|
||||||
|
- "./elasticsearch/logs:/usr/share/elasticsearch/logs"
|
||||||
|
- "./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml"
|
||||||
|
# - "./elasticsearch/config/jvm.options:/usr/share/elasticsearch/config/jvm.options"
|
||||||
|
environment: # 设置环境变量,相当于docker run命令中的-e
|
||||||
|
TZ: Asia/Shanghai
|
||||||
|
LANG: en_US.UTF-8
|
||||||
|
TAKE_FILE_OWNERSHIP: "true" # 权限
|
||||||
|
discovery.type: single-node
|
||||||
|
ES_JAVA_OPTS: "-Xmx512m -Xms512m"
|
||||||
|
ELASTIC_PASSWORD: "123456" # elastic账号密码
|
||||||
|
ports:
|
||||||
|
- "9200:9200"
|
||||||
|
- "9300:9300"
|
||||||
|
networks:
|
||||||
|
- efk
|
||||||
|
|
||||||
|
kibana:
|
||||||
|
image: registry.cn-shanghai.aliyuncs.com/odboy/kibana:7.14 # 原镜像`kibana:7.14.1`
|
||||||
|
container_name: efk_kibana
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- "./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml"
|
||||||
|
ports:
|
||||||
|
- "5601:5601"
|
||||||
|
depends_on:
|
||||||
|
- elasticsearch
|
||||||
|
links:
|
||||||
|
- elasticsearch
|
||||||
|
networks:
|
||||||
|
- efk
|
||||||
|
|
||||||
|
filebeat:
|
||||||
|
image: registry.cn-shanghai.aliyuncs.com/odboy/elastic-filebeat:7.14.1 # 原镜像`elastic/filebeat:7.14.1`
|
||||||
|
container_name: efk_filebeat
|
||||||
|
restart: unless-stopped # 指定容器退出后的重启策略为始终重启，但是不考虑在Docker守护进程启动时就已经停止了的容器
|
||||||
|
volumes: # 数据卷挂载路径设置,将本机目录映射到容器目录
|
||||||
|
- "./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml"
|
||||||
|
- "./filebeat/logs:/usr/share/filebeat/logs"
|
||||||
|
- "./filebeat/my-log:/usr/share/filebeat/my-log"
|
||||||
|
environment: # 设置环境变量,相当于docker run命令中的-e
|
||||||
|
TZ: Asia/Shanghai
|
||||||
|
LANG: en_US.UTF-8
|
||||||
|
depends_on:
|
||||||
|
- elasticsearch
|
||||||
|
networks:
|
||||||
|
- efk
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
cluster.name: "docker-cluster"
|
||||||
|
network.host: 0.0.0.0
|
||||||
|
http.port: 9200
|
||||||
|
# 开启es跨域
|
||||||
|
http.cors.enabled: true
|
||||||
|
http.cors.allow-origin: "*"
|
||||||
|
http.cors.allow-headers: Authorization
|
||||||
|
# 开启安全控制
|
||||||
|
xpack.security.enabled: true
|
||||||
|
xpack.security.transport.ssl.enabled: true
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
## JVM configuration
|
||||||
|
|
||||||
|
################################################################
|
||||||
|
## IMPORTANT: JVM heap size
|
||||||
|
################################################################
|
||||||
|
##
|
||||||
|
## You should always set the min and max JVM heap
|
||||||
|
## size to the same value. For example, to set
|
||||||
|
## the heap to 4 GB, set:
|
||||||
|
##
|
||||||
|
## -Xms4g
|
||||||
|
## -Xmx4g
|
||||||
|
##
|
||||||
|
## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
|
||||||
|
## for more information
|
||||||
|
##
|
||||||
|
################################################################
|
||||||
|
|
||||||
|
# Xms represents the initial size of total heap space
|
||||||
|
# Xmx represents the maximum size of total heap space
|
||||||
|
|
||||||
|
-Xms1g
|
||||||
|
-Xmx1g
|
||||||
|
|
||||||
|
################################################################
|
||||||
|
## Expert settings
|
||||||
|
################################################################
|
||||||
|
##
|
||||||
|
## All settings below this section are considered
|
||||||
|
## expert settings. Don't tamper with them unless
|
||||||
|
## you understand what you are doing
|
||||||
|
##
|
||||||
|
################################################################
|
||||||
|
|
||||||
|
## GC configuration
|
||||||
|
8-13:-XX:+UseConcMarkSweepGC
|
||||||
|
8-13:-XX:CMSInitiatingOccupancyFraction=75
|
||||||
|
8-13:-XX:+UseCMSInitiatingOccupancyOnly
|
||||||
|
|
||||||
|
## G1GC Configuration
|
||||||
|
# NOTE: G1 GC is only supported on JDK version 10 or later
|
||||||
|
# to use G1GC, uncomment the next two lines and update the version on the
|
||||||
|
# following three lines to your version of the JDK
|
||||||
|
# 10-13:-XX:-UseConcMarkSweepGC
|
||||||
|
# 10-13:-XX:-UseCMSInitiatingOccupancyOnly
|
||||||
|
14-:-XX:+UseG1GC
|
||||||
|
14-:-XX:G1ReservePercent=25
|
||||||
|
14-:-XX:InitiatingHeapOccupancyPercent=30
|
||||||
|
|
||||||
|
## JVM temporary directory
|
||||||
|
-Djava.io.tmpdir=${ES_TMPDIR}
|
||||||
|
|
||||||
|
## heap dumps
|
||||||
|
|
||||||
|
# generate a heap dump when an allocation from the Java heap fails
|
||||||
|
# heap dumps are created in the working directory of the JVM
|
||||||
|
-XX:+HeapDumpOnOutOfMemoryError
|
||||||
|
|
||||||
|
# specify an alternative path for heap dumps; ensure the directory exists and
|
||||||
|
# has sufficient space
|
||||||
|
-XX:HeapDumpPath=data
|
||||||
|
|
||||||
|
# specify an alternative path for JVM fatal error logs
|
||||||
|
-XX:ErrorFile=logs/hs_err_pid%p.log
|
||||||
|
|
||||||
|
## JDK 8 GC logging
|
||||||
|
8:-XX:+PrintGCDetails
|
||||||
|
8:-XX:+PrintGCDateStamps
|
||||||
|
8:-XX:+PrintTenuringDistribution
|
||||||
|
8:-XX:+PrintGCApplicationStoppedTime
|
||||||
|
8:-Xloggc:logs/gc.log
|
||||||
|
8:-XX:+UseGCLogFileRotation
|
||||||
|
8:-XX:NumberOfGCLogFiles=32
|
||||||
|
8:-XX:GCLogFileSize=64m
|
||||||
|
|
||||||
|
# JDK 9+ GC logging
|
||||||
|
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# 可参考 https://www.elastic.co/guide/en/beats/filebeat/7.14/filebeat-reference-yml.html
|
||||||
|
|
||||||
|
# 收集系统日志
|
||||||
|
filebeat:
|
||||||
|
inputs:
|
||||||
|
- type: log
|
||||||
|
enabled: true
|
||||||
|
paths:
|
||||||
|
- /usr/share/filebeat/my-log/demo*.log
|
||||||
|
tags: [ "demo" ]
|
||||||
|
# java多行日志合并
|
||||||
|
multiline:
|
||||||
|
pattern: '^\d{4}\-\d{2}\-\d{2}\s\d{2}:\d{2}:\d{2}' # 匹配的正则 不是以 2021-10-01 12:00:00 格式开头的将合并到上一行
|
||||||
|
negate: true # 是否反向匹配
|
||||||
|
match: after # 取值为 after 则放到上一行之后，取值为 before 则放到下一行之前
|
||||||
|
timeout: 3s
|
||||||
|
|
||||||
|
|
||||||
|
# 输出到elasticsearch
|
||||||
|
output.elasticsearch:
|
||||||
|
hosts: [ "elasticsearch:9200" ] # ES地址，可以有多个，用逗号","隔开
|
||||||
|
username: "elastic" # ES用户名
|
||||||
|
password: "123456" # ES密码
|
||||||
|
indices:
|
||||||
|
- index: "demo-%{+yyyy.MM.dd}" # 以日期为后缀，每天新建一个索引
|
||||||
|
when.contains:
|
||||||
|
tags: "demo"
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
docker-compose -f docker-compose.yml -p efk up -d
|
||||||
|
# 当前目录下所有文件赋予权限(读、写、执行) -- 解决es启动报错问题...
|
||||||
|
chmod -R 777 ./elasticsearch
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
#
|
||||||
|
# ** THIS IS AN AUTO-GENERATED FILE **
|
||||||
|
#
|
||||||
|
|
||||||
|
# Default Kibana configuration for docker target
|
||||||
|
|
||||||
|
server.name: kibana
|
||||||
|
server.host: "0.0.0.0"
|
||||||
|
server.publicBaseUrl: "http://kibana:5601" # 这里地址改为你访问kibana的地址，不能以 / 结尾
|
||||||
|
elasticsearch.hosts: [ "http://elasticsearch:9200" ] # http://www.zhengqingya.com:9200 TODO 修改为自己的ip
|
||||||
|
xpack.monitoring.ui.container.elasticsearch.enabled: true
|
||||||
|
elasticsearch.username: "elastic" # es账号
|
||||||
|
elasticsearch.password: "123456" # es密码
|
||||||
|
i18n.locale: zh-CN # 中文
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# EFK
|
||||||
|
|
||||||
|
`Elasticsearch` + `Filebeat` + `Kibana` 搭建日志监控系统
|
||||||
|
|
||||||
|
1. `Filebeat` 采集日志，相比Logstash更加轻量级和易部署，对系统资源开销更小，如果对于日志不需要进行过滤分析的，可以直接使用filebeat
|
||||||
|
2. `Elasticsearch` 日志搜索
|
||||||
|
3. `Kibana` 日志展示
|
||||||
|
|
||||||
|
### 安装部署
|
||||||
|
```shell
|
||||||
|
# 运行
|
||||||
|
sh install.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
1. ES访问地址：[`ip地址:9200`](http://192.168.235.104:9200)
|
||||||
|
默认账号密码：`elastic/123456`
|
||||||
|
2. kibana访问地址：[`ip地址:5601`](http://192.168.235.104:5601)
|
||||||
|
默认账号密码：`elastic/123456`
|
||||||
|
|
||||||
|
### 示例图
|
||||||
|
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
docker-compose -f docker-compose.yml -p efk down
|
||||||
Reference in New Issue
Block a user