From eb9c7bc10c585c700f3df49a4650ebb355830648 Mon Sep 17 00:00:00 2001
From: Odboy <tianjun@odboy.cn>
Date: Fri, 21 Feb 2025 19:24:40 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9Eelasticsearch?=
 =?UTF-8?q?=E3=80=81filebeat=E3=80=81kibana=E7=BB=84=E5=90=88=E7=9A=84?=
 =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=B3=BB=E7=BB=9F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 Log/efk/docker-compose.yml                    | 59 ++++++++++++++
 .../elasticsearch/config/elasticsearch.yml    | 10 +++
 Log/efk/elasticsearch/config/jvm.options      | 77 +++++++++++++++++++
 Log/efk/filebeat/filebeat.yml                 | 27 +++++++
 Log/efk/install.sh                            |  3 +
 Log/efk/kibana/config/kibana.yml              | 14 ++++
 Log/efk/run.md                                | 21 +++++
 Log/efk/shutdown.sh                           |  1 +
 8 files changed, 212 insertions(+)
 create mode 100644 Log/efk/docker-compose.yml
 create mode 100644 Log/efk/elasticsearch/config/elasticsearch.yml
 create mode 100644 Log/efk/elasticsearch/config/jvm.options
 create mode 100644 Log/efk/filebeat/filebeat.yml
 create mode 100644 Log/efk/install.sh
 create mode 100644 Log/efk/kibana/config/kibana.yml
 create mode 100644 Log/efk/run.md
 create mode 100644 Log/efk/shutdown.sh

diff --git a/Log/efk/docker-compose.yml b/Log/efk/docker-compose.yml
new file mode 100644
index 0000000..cd73a82
--- /dev/null
+++ b/Log/efk/docker-compose.yml
@@ -0,0 +1,59 @@
+version: '3'
+
+# 网桥 -> 方便相互通讯
+networks:
+  efk:
+
+services:
+  elasticsearch:
+    image: registry.cn-shanghai.aliyuncs.com/odboy/elasticsearch:7.14.1      # 原镜像`elasticsearch:7.14.1`
+    container_name: efk_elasticsearch         # 容器名为'efk_elasticsearch'
+    restart: unless-stopped                   # 指定容器退出后的重启策略为始终重启，但是不考虑在Docker守护进程启动时就已经停止了的容器
+    volumes:                                  # 数据卷挂载路径设置,将本机目录映射到容器目录
+      - "./elasticsearch/data:/usr/share/elasticsearch/data"
+      - "./elasticsearch/logs:/usr/share/elasticsearch/logs"
+      - "./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml"
+#      - "./elasticsearch/config/jvm.options:/usr/share/elasticsearch/config/jvm.options"
+    environment:                              # 设置环境变量,相当于docker run命令中的-e
+      TZ: Asia/Shanghai
+      LANG: en_US.UTF-8
+      TAKE_FILE_OWNERSHIP: "true"  # 权限
+      discovery.type: single-node
+      ES_JAVA_OPTS: "-Xmx512m -Xms512m"
+      ELASTIC_PASSWORD: "123456" # elastic账号密码
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    networks:
+      - efk
+
+  kibana:
+    image: registry.cn-shanghai.aliyuncs.com/odboy/kibana:7.14       # 原镜像`kibana:7.14.1`
+    container_name: efk_kibana
+    restart: unless-stopped
+    volumes:
+      - "./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml"
+    ports:
+      - "5601:5601"
+    depends_on:
+      - elasticsearch
+    links:
+      - elasticsearch
+    networks:
+      - efk
+
+  filebeat:
+    image: registry.cn-shanghai.aliyuncs.com/odboy/elastic-filebeat:7.14.1      # 原镜像`elastic/filebeat:7.14.1`
+    container_name: efk_filebeat
+    restart: unless-stopped                   # 指定容器退出后的重启策略为始终重启，但是不考虑在Docker守护进程启动时就已经停止了的容器
+    volumes:                                  # 数据卷挂载路径设置,将本机目录映射到容器目录
+      - "./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml"
+      - "./filebeat/logs:/usr/share/filebeat/logs"
+      - "./filebeat/my-log:/usr/share/filebeat/my-log"
+    environment:          # 设置环境变量,相当于docker run命令中的-e
+      TZ: Asia/Shanghai
+      LANG: en_US.UTF-8
+    depends_on:
+      - elasticsearch
+    networks:
+      - efk
\ No newline at end of file
diff --git a/Log/efk/elasticsearch/config/elasticsearch.yml b/Log/efk/elasticsearch/config/elasticsearch.yml
new file mode 100644
index 0000000..cb5076f
--- /dev/null
+++ b/Log/efk/elasticsearch/config/elasticsearch.yml
@@ -0,0 +1,10 @@
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0
+http.port: 9200
+# 开启es跨域
+http.cors.enabled: true
+http.cors.allow-origin: "*"
+http.cors.allow-headers: Authorization
+# 开启安全控制
+xpack.security.enabled: true
+xpack.security.transport.ssl.enabled: true
diff --git a/Log/efk/elasticsearch/config/jvm.options b/Log/efk/elasticsearch/config/jvm.options
new file mode 100644
index 0000000..8c013eb
--- /dev/null
+++ b/Log/efk/elasticsearch/config/jvm.options
@@ -0,0 +1,77 @@
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms1g
+-Xmx1g
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+8-13:-XX:+UseConcMarkSweepGC
+8-13:-XX:CMSInitiatingOccupancyFraction=75
+8-13:-XX:+UseCMSInitiatingOccupancyOnly
+
+## G1GC Configuration
+# NOTE: G1 GC is only supported on JDK version 10 or later
+# to use G1GC, uncomment the next two lines and update the version on the
+# following three lines to your version of the JDK
+# 10-13:-XX:-UseConcMarkSweepGC
+# 10-13:-XX:-UseCMSInitiatingOccupancyOnly
+14-:-XX:+UseG1GC
+14-:-XX:G1ReservePercent=25
+14-:-XX:InitiatingHeapOccupancyPercent=30
+
+## JVM temporary directory
+-Djava.io.tmpdir=${ES_TMPDIR}
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps; ensure the directory exists and
+# has sufficient space
+-XX:HeapDumpPath=data
+
+# specify an alternative path for JVM fatal error logs
+-XX:ErrorFile=logs/hs_err_pid%p.log
+
+## JDK 8 GC logging
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:logs/gc.log
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+# JDK 9+ GC logging
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
diff --git a/Log/efk/filebeat/filebeat.yml b/Log/efk/filebeat/filebeat.yml
new file mode 100644
index 0000000..63e996e
--- /dev/null
+++ b/Log/efk/filebeat/filebeat.yml
@@ -0,0 +1,27 @@
+# 可参考 https://www.elastic.co/guide/en/beats/filebeat/7.14/filebeat-reference-yml.html
+
+# 收集系统日志
+filebeat:
+  inputs:
+    - type: log
+      enabled: true
+      paths:
+        - /usr/share/filebeat/my-log/demo*.log
+      tags: [ "demo" ]
+      # java多行日志合并
+      multiline:
+        pattern: '^\d{4}\-\d{2}\-\d{2}\s\d{2}:\d{2}:\d{2}' # 匹配的正则 不是以 2021-10-01 12:00:00 格式开头的将合并到上一行
+        negate: true                                       # 是否反向匹配
+        match: after                                       # 取值为 after 则放到上一行之后，取值为 before 则放到下一行之前
+        timeout: 3s
+
+
+# 输出到elasticsearch
+output.elasticsearch:
+  hosts: [ "elasticsearch:9200" ] # ES地址，可以有多个，用逗号","隔开
+  username: "elastic"             # ES用户名
+  password: "123456"              # ES密码
+  indices:
+    - index: "demo-%{+yyyy.MM.dd}"     # 以日期为后缀，每天新建一个索引
+      when.contains:
+        tags: "demo"
diff --git a/Log/efk/install.sh b/Log/efk/install.sh
new file mode 100644
index 0000000..96e36bf
--- /dev/null
+++ b/Log/efk/install.sh
@@ -0,0 +1,3 @@
+docker-compose -f docker-compose.yml -p efk up -d
+# 当前目录下所有文件赋予权限(读、写、执行)  -- 解决es启动报错问题...
+chmod -R 777 ./elasticsearch
\ No newline at end of file
diff --git a/Log/efk/kibana/config/kibana.yml b/Log/efk/kibana/config/kibana.yml
new file mode 100644
index 0000000..eee7d31
--- /dev/null
+++ b/Log/efk/kibana/config/kibana.yml
@@ -0,0 +1,14 @@
+#
+# ** THIS IS AN AUTO-GENERATED FILE **
+#
+
+# Default Kibana configuration for docker target
+
+server.name: kibana
+server.host: "0.0.0.0"
+server.publicBaseUrl: "http://kibana:5601"  # 这里地址改为你访问kibana的地址，不能以 / 结尾
+elasticsearch.hosts: [ "http://elasticsearch:9200" ] # http://www.zhengqingya.com:9200 TODO 修改为自己的ip
+xpack.monitoring.ui.container.elasticsearch.enabled: true
+elasticsearch.username: "elastic"  # es账号
+elasticsearch.password: "123456"   # es密码
+i18n.locale: zh-CN # 中文
diff --git a/Log/efk/run.md b/Log/efk/run.md
new file mode 100644
index 0000000..e71e249
--- /dev/null
+++ b/Log/efk/run.md
@@ -0,0 +1,21 @@
+# EFK
+
+`Elasticsearch` + `Filebeat` + `Kibana` 搭建日志监控系统
+
+1. `Filebeat` 采集日志，相比Logstash更加轻量级和易部署，对系统资源开销更小，如果对于日志不需要进行过滤分析的，可以直接使用filebeat
+2. `Elasticsearch` 日志搜索
+3. `Kibana` 日志展示
+
+### 安装部署
+```shell
+# 运行
+sh install.sh
+```
+
+1. ES访问地址：[`ip地址:9200`](http://192.168.235.104:9200)
+   默认账号密码：`elastic/123456`
+2. kibana访问地址：[`ip地址:5601`](http://192.168.235.104:5601)
+   默认账号密码：`elastic/123456`
+
+### 示例图
+![demo](./demo.png)
\ No newline at end of file
diff --git a/Log/efk/shutdown.sh b/Log/efk/shutdown.sh
new file mode 100644
index 0000000..e79d5af
--- /dev/null
+++ b/Log/efk/shutdown.sh
@@ -0,0 +1 @@
+docker-compose -f docker-compose.yml -p efk down
\ No newline at end of file