feat: 扩充 Docker 命令并新增 diagnose 状态诊断
- Docker：新增只读、生命周期、清理、Compose 共 32 条命令，Linux 自动适配 podman - 危险命令 clearall/prunevolumes 未带 --yes 时仅提示并以退出码 2 结束 - 需要容器/镜像名的命令做必填参数校验，-- 之后参数计入位置参数 - diagnose：自动识别 Docker/Podman 容器或 K8s Pod，输出状态、根因、依据、建议并附日志与事件 - diagnose 中 -n 表示 namespace，其余命令 -n 仍为 --dry-run - 新增内置动作桥接与平台变体 helper，补充单元/集成/CLI 测试与 README
This commit is contained in:
@@ -0,0 +1,248 @@
|
||||
package diagnose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type dockerHealth struct {
|
||||
Status string `json:"Status"`
|
||||
}
|
||||
|
||||
type dockerContainer struct {
|
||||
ID string `json:"Id"`
|
||||
Name string `json:"Name"`
|
||||
RestartCount int `json:"RestartCount"`
|
||||
Config struct {
|
||||
Image string `json:"Image"`
|
||||
} `json:"Config"`
|
||||
State struct {
|
||||
Status string `json:"Status"`
|
||||
Running bool `json:"Running"`
|
||||
Paused bool `json:"Paused"`
|
||||
Restarting bool `json:"Restarting"`
|
||||
OOMKilled bool `json:"OOMKilled"`
|
||||
Dead bool `json:"Dead"`
|
||||
ExitCode int `json:"ExitCode"`
|
||||
Error string `json:"Error"`
|
||||
StartedAt string `json:"StartedAt"`
|
||||
FinishedAt string `json:"FinishedAt"`
|
||||
Health *dockerHealth `json:"Health"`
|
||||
} `json:"State"`
|
||||
}
|
||||
|
||||
func (s *Service) diagnoseDocker(ctx context.Context, src source, target string, opts Options) error {
|
||||
res, err := s.run(ctx, src.program, "inspect", "--type", "container", target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var containers []dockerContainer
|
||||
if err := json.Unmarshal(res.Stdout, &containers); err != nil {
|
||||
return fmt.Errorf("解析 %s inspect 输出失败: %w", src.program, err)
|
||||
}
|
||||
if len(containers) == 0 {
|
||||
return fmt.Errorf("%s 中没有名为 %q 的容器", src.program, target)
|
||||
}
|
||||
|
||||
c := containers[0]
|
||||
name := strings.TrimPrefix(c.Name, "/")
|
||||
if name == "" {
|
||||
name = target
|
||||
}
|
||||
|
||||
fmt.Fprintf(s.out, "目标: %s\n", name)
|
||||
fmt.Fprintf(s.out, "来源: Docker (%s)\n", src.program)
|
||||
fmt.Fprintf(s.out, "状态: %s ExitCode=%d OOMKilled=%t RestartCount=%d\n",
|
||||
c.State.Status, c.State.ExitCode, c.State.OOMKilled, c.RestartCount)
|
||||
if c.Config.Image != "" {
|
||||
fmt.Fprintf(s.out, "镜像: %s\n", c.Config.Image)
|
||||
}
|
||||
if c.State.Health != nil && c.State.Health.Status != "" {
|
||||
fmt.Fprintf(s.out, "健康: %s\n", c.State.Health.Status)
|
||||
}
|
||||
if c.State.StartedAt != "" && !strings.HasPrefix(c.State.StartedAt, "0001-") {
|
||||
fmt.Fprintf(s.out, "时间: %s", c.State.StartedAt)
|
||||
if c.State.FinishedAt != "" && !strings.HasPrefix(c.State.FinishedAt, "0001-") && c.State.FinishedAt != c.State.StartedAt {
|
||||
fmt.Fprintf(s.out, " → %s", c.State.FinishedAt)
|
||||
}
|
||||
fmt.Fprintln(s.out)
|
||||
}
|
||||
s.renderFinding("", judgeDocker(c))
|
||||
|
||||
if !opts.NoLogs {
|
||||
logRes, err := s.run(ctx, src.program, "logs", "--tail", strconv.Itoa(opts.Lines), "--timestamps", target)
|
||||
if err != nil {
|
||||
fmt.Fprintf(s.errOut, "警告: 获取日志失败: %v\n", err)
|
||||
} else {
|
||||
s.writeSection(fmt.Sprintf("最近日志 (%d 行)", opts.Lines), string(logRes.Stdout))
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.NoEvents {
|
||||
now := s.now()
|
||||
since := now.Add(-time.Hour).UTC().Format(time.RFC3339)
|
||||
until := now.UTC().Format(time.RFC3339)
|
||||
eventRes, err := s.run(ctx, src.program, "events",
|
||||
"--since", since, "--until", until, "--filter", "container="+name)
|
||||
if err != nil {
|
||||
fmt.Fprintf(s.errOut, "警告: 获取事件失败: %v\n", err)
|
||||
} else {
|
||||
s.writeSection("最近事件 (1h)", tailLines(string(eventRes.Stdout), 20))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// judgeDocker maps container state to a human-readable root cause.
|
||||
func judgeDocker(c dockerContainer) Finding {
|
||||
st := c.State
|
||||
evidence := "State.Status=" + st.Status
|
||||
if st.Status == "exited" || st.ExitCode != 0 {
|
||||
evidence += fmt.Sprintf(", ExitCode=%d", st.ExitCode)
|
||||
}
|
||||
if st.OOMKilled {
|
||||
evidence += ", OOMKilled=true"
|
||||
}
|
||||
if c.RestartCount > 0 {
|
||||
evidence += fmt.Sprintf(", RestartCount=%d", c.RestartCount)
|
||||
}
|
||||
if st.Error != "" {
|
||||
evidence += ", Error=" + truncate(st.Error, 200)
|
||||
}
|
||||
|
||||
switch {
|
||||
case st.Restarting || st.Status == "restarting":
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器处于重启循环，正在反复启动",
|
||||
Evidence: evidence,
|
||||
Advice: "查看最近日志与上一次退出码，确认应用是否启动即崩溃或依赖不可用",
|
||||
}
|
||||
|
||||
case st.Status == "exited":
|
||||
switch {
|
||||
case st.OOMKilled:
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器因内存超限被 OOM kill",
|
||||
Evidence: evidence,
|
||||
Advice: "提高容器的内存限制，或降低应用峰值内存；可用 kopsc docker stats 观察",
|
||||
}
|
||||
case st.ExitCode == 137:
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器收到 SIGKILL（137），可能被手动 kill 或 stop 超时后强杀",
|
||||
Evidence: evidence,
|
||||
Advice: "若并非人为操作，检查宿主机 OOM 日志与 stop 超时设置",
|
||||
}
|
||||
case st.ExitCode == 143:
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "容器收到 SIGTERM（143）后退出，通常是正常停止",
|
||||
Evidence: evidence,
|
||||
Advice: "若预期继续运行，检查是谁发送了停止信号或重启策略",
|
||||
}
|
||||
case st.ExitCode == 126:
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器内命令不可执行（126）：权限不足或可执行格式错误",
|
||||
Evidence: evidence,
|
||||
Advice: "检查 entrypoint/cmd 是否可执行、是否缺少可执行权限",
|
||||
}
|
||||
case st.ExitCode == 127:
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器内命令不存在（127）",
|
||||
Evidence: evidence,
|
||||
Advice: "检查 entrypoint/cmd 与镜像内实际的二进制路径",
|
||||
}
|
||||
case st.ExitCode == 0:
|
||||
if c.RestartCount > 0 {
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "容器正常退出，但已多次重启，重启策略可能导致循环",
|
||||
Evidence: evidence,
|
||||
Advice: "若是任务型容器，确认重启策略应为 no/on-failure 而不是 always",
|
||||
}
|
||||
}
|
||||
return Finding{
|
||||
Severity: "ok",
|
||||
Verdict: "容器已正常退出（ExitCode=0）",
|
||||
Evidence: evidence,
|
||||
}
|
||||
default:
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器以非零状态退出，应用自身报错",
|
||||
Evidence: evidence,
|
||||
Advice: "查看下方日志的末尾，定位应用错误；确认命令与配置参数",
|
||||
}
|
||||
}
|
||||
|
||||
case st.Status == "running":
|
||||
if st.Paused {
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "容器处于暂停（paused）状态",
|
||||
Evidence: evidence,
|
||||
Advice: "确认是否有人执行了 docker pause；唤醒使用 docker unpause",
|
||||
}
|
||||
}
|
||||
if st.Health != nil && strings.EqualFold(st.Health.Status, "unhealthy") {
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器健康检查失败（unhealthy）",
|
||||
Evidence: evidence + ", State.Health.Status=unhealthy",
|
||||
Advice: "查看日志与 healthcheck 命令；确认依赖服务可用、检查间隔与超时合理",
|
||||
}
|
||||
}
|
||||
return Finding{
|
||||
Severity: "ok",
|
||||
Verdict: "容器正在运行",
|
||||
Evidence: evidence,
|
||||
}
|
||||
|
||||
case st.Status == "created":
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "容器已创建但尚未启动",
|
||||
Evidence: evidence,
|
||||
Advice: "使用 kopsc docker start <容器> 启动；若启动失败查看 docker events",
|
||||
}
|
||||
|
||||
case st.Status == "paused":
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "容器已暂停",
|
||||
Evidence: evidence,
|
||||
}
|
||||
|
||||
case st.Status == "dead":
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器处于 dead 状态，运行时未能正常清理",
|
||||
Evidence: evidence,
|
||||
Advice: "通常需要删除后重新创建容器；检查存储驱动与宿主资源",
|
||||
}
|
||||
}
|
||||
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "容器状态未识别：" + st.Status,
|
||||
Evidence: evidence,
|
||||
}
|
||||
}
|
||||
|
||||
func truncate(s string, max int) string {
|
||||
s = strings.ReplaceAll(strings.TrimSpace(s), "\n", " ")
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max] + "…"
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package diagnose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestJudgeDocker(t *testing.T) {
|
||||
unhealthy := sampleDocker("running", 0, false, 0)
|
||||
unhealthy.State.Health = &dockerHealth{Status: "unhealthy"}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
container dockerContainer
|
||||
wantSeverity string
|
||||
wantContains string
|
||||
}{
|
||||
{"oom killed", sampleDocker("exited", 137, true, 3), "error", "OOM"},
|
||||
{"sigkill", sampleDocker("exited", 137, false, 0), "error", "SIGKILL"},
|
||||
{"sigterm", sampleDocker("exited", 143, false, 0), "warn", "SIGTERM"},
|
||||
{"normal exit", sampleDocker("exited", 0, false, 0), "ok", "正常退出"},
|
||||
{"normal exit with restarts", sampleDocker("exited", 0, false, 2), "warn", "重启"},
|
||||
{"application error", sampleDocker("exited", 1, false, 0), "error", "非零"},
|
||||
{"not executable", sampleDocker("exited", 126, false, 0), "error", "126"},
|
||||
{"missing command", sampleDocker("exited", 127, false, 0), "error", "127"},
|
||||
{"running", sampleDocker("running", 0, false, 0), "ok", "运行"},
|
||||
{"unhealthy", unhealthy, "error", "健康检查失败"},
|
||||
{"restarting", sampleDocker("restarting", 0, false, 5), "error", "重启循环"},
|
||||
{"created", sampleDocker("created", 0, false, 0), "warn", "未启动"},
|
||||
{"dead", sampleDocker("dead", 0, false, 0), "error", "dead"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := judgeDocker(tc.container)
|
||||
if f.Severity != tc.wantSeverity {
|
||||
t.Errorf("severity = %q, want %q (verdict %q)", f.Severity, tc.wantSeverity, f.Verdict)
|
||||
}
|
||||
if !strings.Contains(f.Verdict, tc.wantContains) {
|
||||
t.Errorf("verdict %q does not contain %q", f.Verdict, tc.wantContains)
|
||||
}
|
||||
if f.Evidence == "" {
|
||||
t.Error("evidence must not be empty")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnoseDockerOutput(t *testing.T) {
|
||||
inspectJSON := `[{"Id":"abc","Name":"/web","RestartCount":3,"Config":{"Image":"nginx:1.25"},"State":{"Status":"exited","ExitCode":137,"OOMKilled":true,"StartedAt":"2026-10-31T10:00:00Z","FinishedAt":"2026-10-31T10:05:00Z"}}]`
|
||||
|
||||
r := &fakeRunner{
|
||||
available: map[string]bool{"docker": true},
|
||||
respond: func(name string, args []string) (CommandResult, error) {
|
||||
switch {
|
||||
case len(args) > 0 && args[0] == "inspect":
|
||||
return CommandResult{Stdout: []byte(inspectJSON)}, nil
|
||||
case len(args) > 0 && args[0] == "logs":
|
||||
return CommandResult{Stdout: []byte("line1\nline2\n")}, nil
|
||||
case len(args) > 0 && args[0] == "events":
|
||||
return CommandResult{Stdout: []byte("die event\n")}, nil
|
||||
}
|
||||
return CommandResult{}, fmt.Errorf("unexpected call %s %v", name, args)
|
||||
},
|
||||
}
|
||||
|
||||
s, out, errOut := newTestService(r)
|
||||
if err := s.Run(context.Background(), []string{"web"}); err != nil {
|
||||
t.Fatalf("Run error: %v (stderr=%s)", err, errOut.String())
|
||||
}
|
||||
|
||||
text := out.String()
|
||||
for _, want := range []string{
|
||||
"目标: web",
|
||||
"来源: Docker (docker)",
|
||||
"ExitCode=137",
|
||||
"OOMKilled=true",
|
||||
"OOM",
|
||||
"依据:",
|
||||
"建议:",
|
||||
"最近日志 (50 行)",
|
||||
"line1",
|
||||
"最近事件 (1h)",
|
||||
"die event",
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("output missing %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnoseDockerHonoursNoLogsNoEvents(t *testing.T) {
|
||||
inspectJSON := `[{"Id":"abc","Name":"/web","State":{"Status":"running"}}]`
|
||||
r := &fakeRunner{
|
||||
available: map[string]bool{"docker": true},
|
||||
respond: func(name string, args []string) (CommandResult, error) {
|
||||
return CommandResult{Stdout: []byte(inspectJSON)}, nil
|
||||
},
|
||||
}
|
||||
|
||||
s, _, _ := newTestService(r)
|
||||
if err := s.Run(context.Background(), []string{"web", "--no-logs", "--no-events"}); err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if r.called("logs") {
|
||||
t.Error("logs should not be collected")
|
||||
}
|
||||
if r.called("events") {
|
||||
t.Error("events should not be collected")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package diagnose
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.odboy.cn/odboy/kenaito-kopsc/internal/platform"
|
||||
)
|
||||
|
||||
// fakeRunner records calls and returns canned output.
|
||||
type fakeRunner struct {
|
||||
available map[string]bool
|
||||
calls []string
|
||||
respond func(name string, args []string) (CommandResult, error)
|
||||
}
|
||||
|
||||
func (f *fakeRunner) LookPath(name string) (string, error) {
|
||||
if f.available[name] {
|
||||
return "/usr/bin/" + name, nil
|
||||
}
|
||||
return "", fmt.Errorf("%s not found", name)
|
||||
}
|
||||
|
||||
func (f *fakeRunner) Run(ctx context.Context, name string, args ...string) (CommandResult, error) {
|
||||
f.calls = append(f.calls, name+" "+strings.Join(args, " "))
|
||||
if f.respond == nil {
|
||||
return CommandResult{}, nil
|
||||
}
|
||||
return f.respond(name, args)
|
||||
}
|
||||
|
||||
func (f *fakeRunner) called(substr string) bool {
|
||||
for _, call := range f.calls {
|
||||
if strings.Contains(call, substr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func newTestService(r Runner) (*Service, *bytes.Buffer, *bytes.Buffer) {
|
||||
out, errOut := &bytes.Buffer{}, &bytes.Buffer{}
|
||||
s := New(platform.Info{OS: "linux", Arch: "amd64"}, r, out, errOut)
|
||||
s.now = func() time.Time { return time.Date(2026, 10, 31, 12, 0, 0, 0, time.UTC) }
|
||||
return s, out, errOut
|
||||
}
|
||||
|
||||
// sampleDocker builds an exited container.
|
||||
func sampleDocker(status string, exitCode int, oom bool, restarts int) dockerContainer {
|
||||
c := dockerContainer{ID: "abc", Name: "/web", RestartCount: restarts}
|
||||
c.Config.Image = "nginx:1.25"
|
||||
c.State.Status = status
|
||||
c.State.ExitCode = exitCode
|
||||
c.State.OOMKilled = oom
|
||||
c.State.Running = status == "running"
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
package diagnose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type pod struct {
|
||||
Metadata struct {
|
||||
Name string `json:"name"`
|
||||
Namespace string `json:"namespace"`
|
||||
} `json:"metadata"`
|
||||
Status struct {
|
||||
Phase string `json:"phase"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
Conditions []podCondition `json:"conditions"`
|
||||
ContainerStatuses []containerStatus `json:"containerStatuses"`
|
||||
} `json:"status"`
|
||||
}
|
||||
|
||||
type podCondition struct {
|
||||
Type string `json:"type"`
|
||||
Status string `json:"status"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type containerStatus struct {
|
||||
Name string `json:"name"`
|
||||
Ready bool `json:"ready"`
|
||||
RestartCount int `json:"restartCount"`
|
||||
State podState `json:"state"`
|
||||
LastState podState `json:"lastState"`
|
||||
}
|
||||
|
||||
type podState struct {
|
||||
Waiting *podWaiting `json:"waiting"`
|
||||
Running *podRunning `json:"running"`
|
||||
Terminated *podTerminated `json:"terminated"`
|
||||
}
|
||||
|
||||
type podWaiting struct {
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type podRunning struct {
|
||||
StartedAt string `json:"startedAt"`
|
||||
}
|
||||
|
||||
type podTerminated struct {
|
||||
Reason string `json:"reason"`
|
||||
ExitCode int `json:"exitCode"`
|
||||
Message string `json:"message"`
|
||||
StartedAt string `json:"startedAt"`
|
||||
FinishedAt string `json:"finishedAt"`
|
||||
}
|
||||
|
||||
func (s *Service) diagnoseK8s(ctx context.Context, src source, target string, opts Options) error {
|
||||
if _, err := s.runner.LookPath("kubectl"); err != nil {
|
||||
return fmt.Errorf("未找到 kubectl，请先安装并配置 kubeconfig")
|
||||
}
|
||||
|
||||
namespace := src.namespace
|
||||
if namespace == "" {
|
||||
namespace, _ = s.kubeNamespace(ctx, src.context)
|
||||
}
|
||||
|
||||
res, err := s.run(ctx, "kubectl", kubectlArgs(src.context, "get", "pod", target, "-n", namespace, "-o", "json")...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var p pod
|
||||
if err := json.Unmarshal(res.Stdout, &p); err != nil {
|
||||
return fmt.Errorf("解析 kubectl get pod 输出失败: %w", err)
|
||||
}
|
||||
|
||||
name := p.Metadata.Name
|
||||
if name == "" {
|
||||
name = target
|
||||
}
|
||||
if p.Metadata.Namespace != "" {
|
||||
namespace = p.Metadata.Namespace
|
||||
}
|
||||
|
||||
contextLabel := src.context
|
||||
if contextLabel == "" {
|
||||
contextLabel = "当前上下文"
|
||||
}
|
||||
|
||||
ready, total, restarts := 0, len(p.Status.ContainerStatuses), 0
|
||||
for _, cs := range p.Status.ContainerStatuses {
|
||||
if cs.Ready {
|
||||
ready++
|
||||
}
|
||||
restarts += cs.RestartCount
|
||||
}
|
||||
|
||||
fmt.Fprintf(s.out, "目标: %s\n", name)
|
||||
fmt.Fprintf(s.out, "来源: Kubernetes context=%s namespace=%s\n", contextLabel, namespace)
|
||||
fmt.Fprintf(s.out, "状态: phase=%s ready=%d/%d restarts=%d\n", p.Status.Phase, ready, total, restarts)
|
||||
s.renderFinding("", judgePod(p))
|
||||
|
||||
selected := p.Status.ContainerStatuses
|
||||
if opts.Container != "" {
|
||||
selected = nil
|
||||
for _, cs := range p.Status.ContainerStatuses {
|
||||
if cs.Name == opts.Container {
|
||||
selected = append(selected, cs)
|
||||
}
|
||||
}
|
||||
if len(selected) == 0 {
|
||||
fmt.Fprintf(s.errOut, "警告: Pod %s 中没有名为 %q 的容器\n", name, opts.Container)
|
||||
}
|
||||
}
|
||||
|
||||
for _, cs := range selected {
|
||||
fmt.Fprintf(s.out, "\n容器 %s:\n", cs.Name)
|
||||
fmt.Fprintf(s.out, " 状态: %s ready=%t restarts=%d\n", containerState(cs), cs.Ready, cs.RestartCount)
|
||||
s.renderFinding(" ", judgeContainer(cs))
|
||||
if !opts.NoLogs {
|
||||
s.writeContainerLogs(ctx, src, name, namespace, cs, opts)
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.NoEvents {
|
||||
eventRes, err := s.run(ctx, "kubectl", kubectlArgs(src.context,
|
||||
"get", "events", "-n", namespace,
|
||||
"--field-selector", "involvedObject.name="+name,
|
||||
"--sort-by=.lastTimestamp")...)
|
||||
if err != nil {
|
||||
fmt.Fprintf(s.errOut, "警告: 获取事件失败: %v\n", err)
|
||||
} else {
|
||||
events := tailLines(string(eventRes.Stdout), 20)
|
||||
s.writeSection(fmt.Sprintf("最近事件 (namespace=%s)", namespace), events)
|
||||
if hint := eventHint(events); hint != "" {
|
||||
fmt.Fprintf(s.out, "\n事件提示: %s\n", hint)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// judgePod reports Pod-level health; per-container verdicts follow separately.
|
||||
func judgePod(p pod) Finding {
|
||||
evidence := "status.phase=" + p.Status.Phase
|
||||
if p.Status.Reason != "" {
|
||||
evidence += ", reason=" + p.Status.Reason
|
||||
}
|
||||
|
||||
switch p.Status.Phase {
|
||||
case "Pending":
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "Pod 处于 Pending：尚未成功调度或容器还未创建",
|
||||
Evidence: evidence,
|
||||
Advice: "查看下方事件：常见原因是资源不足、亲和性/污点不满足、PVC 未绑定或镜像拉取慢",
|
||||
}
|
||||
case "Failed":
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "Pod 已失败",
|
||||
Evidence: evidence,
|
||||
Advice: "结合下方容器状态、上一次日志与事件定位失败原因",
|
||||
}
|
||||
case "Succeeded":
|
||||
return Finding{
|
||||
Severity: "ok",
|
||||
Verdict: "Pod 已正常完成",
|
||||
Evidence: evidence,
|
||||
}
|
||||
case "Running":
|
||||
return Finding{
|
||||
Severity: "ok",
|
||||
Verdict: "Pod 已调度并在运行（容器级状态见下）",
|
||||
Evidence: evidence,
|
||||
}
|
||||
default:
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "Pod 状态未识别：" + p.Status.Phase,
|
||||
Evidence: evidence,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// judgeContainer maps containerStatus to a root cause.
|
||||
func judgeContainer(cs containerStatus) Finding {
|
||||
if cs.State.Waiting != nil {
|
||||
w := cs.State.Waiting
|
||||
evidence := "state.waiting.reason=" + w.Reason
|
||||
if w.Message != "" {
|
||||
evidence += ", message=" + truncate(w.Message, 200)
|
||||
}
|
||||
|
||||
switch w.Reason {
|
||||
case "CrashLoopBackOff":
|
||||
if cs.LastState.Terminated != nil {
|
||||
t := cs.LastState.Terminated
|
||||
evidence += fmt.Sprintf(", lastState.reason=%s, lastState.exitCode=%d", t.Reason, t.ExitCode)
|
||||
}
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器反复崩溃并重启（CrashLoopBackOff）",
|
||||
Evidence: evidence,
|
||||
Advice: "查看下方“上一次”日志与退出码；检查启动命令、依赖服务与资源限制",
|
||||
}
|
||||
case "ImagePullBackOff", "ErrImagePull":
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "镜像拉取失败（" + w.Reason + "）",
|
||||
Evidence: evidence,
|
||||
Advice: "检查镜像名与 tag、imagePullSecrets、仓库可达性与节点网络",
|
||||
}
|
||||
case "CreateContainerConfigError":
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器配置错误：ConfigMap/Secret 缺失或键不存在",
|
||||
Evidence: evidence,
|
||||
Advice: "核对该 Pod 引用的 ConfigMap/Secret 是否存在，键名是否正确",
|
||||
}
|
||||
case "CreateContainerError", "RunContainerError":
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器创建/启动失败（" + w.Reason + "）",
|
||||
Evidence: evidence,
|
||||
Advice: "检查容器运行时与事件中的具体错误（挂载、cgroup、权限）",
|
||||
}
|
||||
case "ContainerCreating":
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "容器正在创建",
|
||||
Evidence: evidence,
|
||||
Advice: "通常会自动完成；若长时间不变，查看事件中的挂载/镜像错误",
|
||||
}
|
||||
case "PodInitializing":
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "初始化容器尚未完成",
|
||||
Evidence: evidence,
|
||||
Advice: "查看 initContainer 的日志与状态",
|
||||
}
|
||||
default:
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "容器处于等待状态：" + w.Reason,
|
||||
Evidence: evidence,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if cs.State.Terminated != nil {
|
||||
t := cs.State.Terminated
|
||||
evidence := fmt.Sprintf("state.terminated.reason=%s, exitCode=%d", t.Reason, t.ExitCode)
|
||||
if t.Message != "" {
|
||||
evidence += ", message=" + truncate(t.Message, 200)
|
||||
}
|
||||
|
||||
switch t.Reason {
|
||||
case "OOMKilled":
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器因内存超限被 OOMKilled",
|
||||
Evidence: evidence,
|
||||
Advice: "提高 resources.limits.memory，或降低应用内存占用",
|
||||
}
|
||||
case "Completed":
|
||||
return Finding{
|
||||
Severity: "ok",
|
||||
Verdict: "容器已正常结束（Completed）",
|
||||
Evidence: evidence,
|
||||
}
|
||||
case "Error":
|
||||
return Finding{
|
||||
Severity: "error",
|
||||
Verdict: "容器以非零状态退出（Error）",
|
||||
Evidence: evidence,
|
||||
Advice: "查看下方日志末尾定位应用错误",
|
||||
}
|
||||
default:
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "容器已终止：" + t.Reason,
|
||||
Evidence: evidence,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if cs.State.Running != nil {
|
||||
evidence := "state.running"
|
||||
if cs.LastState.Terminated != nil {
|
||||
t := cs.LastState.Terminated
|
||||
evidence += fmt.Sprintf(", lastState.reason=%s, lastState.exitCode=%d", t.Reason, t.ExitCode)
|
||||
}
|
||||
if !cs.Ready {
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "容器运行中但未就绪（就绪探针未通过）",
|
||||
Evidence: evidence + ", ready=false",
|
||||
Advice: "检查 readinessProbe 配置、启动时间与依赖服务",
|
||||
}
|
||||
}
|
||||
return Finding{
|
||||
Severity: "ok",
|
||||
Verdict: "容器运行中且已就绪",
|
||||
Evidence: evidence + ", ready=true",
|
||||
}
|
||||
}
|
||||
|
||||
return Finding{
|
||||
Severity: "warn",
|
||||
Verdict: "容器状态未知（state 为空）",
|
||||
Evidence: "state 为空",
|
||||
Advice: "查看 kubectl describe pod 与事件",
|
||||
}
|
||||
}
|
||||
|
||||
func containerState(cs containerStatus) string {
|
||||
switch {
|
||||
case cs.State.Waiting != nil:
|
||||
return "waiting(" + cs.State.Waiting.Reason + ")"
|
||||
case cs.State.Terminated != nil:
|
||||
return "terminated(" + cs.State.Terminated.Reason + ")"
|
||||
case cs.State.Running != nil:
|
||||
return "running"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) writeContainerLogs(ctx context.Context, src source, podName, namespace string, cs containerStatus, opts Options) {
|
||||
base := []string{"logs", podName, "-n", namespace, "-c", cs.Name, "--tail", strconv.Itoa(opts.Lines)}
|
||||
|
||||
res, err := s.run(ctx, "kubectl", kubectlArgs(src.context, base...)...)
|
||||
if err != nil {
|
||||
fmt.Fprintf(s.errOut, "警告: 获取容器 %s 日志失败: %v\n", cs.Name, err)
|
||||
} else {
|
||||
s.writeSection(fmt.Sprintf("日志 (%s, 最近 %d 行)", cs.Name, opts.Lines), string(res.Stdout))
|
||||
}
|
||||
|
||||
if cs.LastState.Terminated != nil {
|
||||
previous := append(append([]string{}, base...), "--previous")
|
||||
res, err := s.run(ctx, "kubectl", kubectlArgs(src.context, previous...)...)
|
||||
if err != nil {
|
||||
fmt.Fprintf(s.errOut, "警告: 获取容器 %s 上一次日志失败: %v\n", cs.Name, err)
|
||||
} else {
|
||||
s.writeSection(fmt.Sprintf("日志 (%s, 上一次)", cs.Name), string(res.Stdout))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// eventHint turns common event fragments into an actionable hint.
|
||||
func eventHint(events string) string {
|
||||
lower := strings.ToLower(events)
|
||||
switch {
|
||||
case strings.Contains(lower, "failedscheduling"), strings.Contains(lower, "insufficient"):
|
||||
return "事件显示调度失败：检查节点资源、亲和性/反亲和性、污点与 PVC 绑定"
|
||||
case strings.Contains(lower, "failedmount"), strings.Contains(lower, "failedattachvolume"):
|
||||
return "事件显示存储挂载失败：检查 PVC/PV、StorageClass 与 ConfigMap/Secret 挂载"
|
||||
case strings.Contains(lower, "unhealthy"):
|
||||
return "事件显示探针失败：检查 liveness/readiness 探针路径、端口与启动时间"
|
||||
case strings.Contains(lower, "backoff"):
|
||||
return "事件显示退避重试：结合容器退出码与日志确认根因"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package diagnose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const crashLoopPodJSON = `{
|
||||
"metadata":{"name":"api-0","namespace":"prod"},
|
||||
"status":{
|
||||
"phase":"Running",
|
||||
"containerStatuses":[
|
||||
{"name":"app","ready":false,"restartCount":5,
|
||||
"state":{"waiting":{"reason":"CrashLoopBackOff","message":"back-off 5m0s"}},
|
||||
"lastState":{"terminated":{"reason":"Error","exitCode":1,"message":"boom"}}}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
func waitingContainer(reason string, lastExit int) containerStatus {
|
||||
cs := containerStatus{Name: "app", RestartCount: 2}
|
||||
cs.State.Waiting = &podWaiting{Reason: reason, Message: "message"}
|
||||
if lastExit >= 0 {
|
||||
cs.LastState.Terminated = &podTerminated{Reason: "Error", ExitCode: lastExit}
|
||||
}
|
||||
return cs
|
||||
}
|
||||
|
||||
func TestJudgeContainer(t *testing.T) {
|
||||
oom := containerStatus{Name: "app"}
|
||||
oom.State.Terminated = &podTerminated{Reason: "OOMKilled", ExitCode: 137}
|
||||
|
||||
completed := containerStatus{Name: "app"}
|
||||
completed.State.Terminated = &podTerminated{Reason: "Completed", ExitCode: 0}
|
||||
|
||||
notReady := containerStatus{Name: "app"}
|
||||
notReady.State.Running = &podRunning{}
|
||||
|
||||
ready := containerStatus{Name: "app", Ready: true}
|
||||
ready.State.Running = &podRunning{}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
status containerStatus
|
||||
wantSeverity string
|
||||
wantContains string
|
||||
}{
|
||||
{"crash loop", waitingContainer("CrashLoopBackOff", 1), "error", "CrashLoopBackOff"},
|
||||
{"image pull", waitingContainer("ImagePullBackOff", -1), "error", "镜像拉取"},
|
||||
{"err image pull", waitingContainer("ErrImagePull", -1), "error", "镜像拉取"},
|
||||
{"config error", waitingContainer("CreateContainerConfigError", -1), "error", "配置"},
|
||||
{"creating", waitingContainer("ContainerCreating", -1), "warn", "创建"},
|
||||
{"oom killed", oom, "error", "OOMKilled"},
|
||||
{"completed", completed, "ok", "正常结束"},
|
||||
{"not ready", notReady, "warn", "未就绪"},
|
||||
{"ready", ready, "ok", "已就绪"},
|
||||
{"unknown", containerStatus{Name: "app"}, "warn", "未知"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := judgeContainer(tc.status)
|
||||
if f.Severity != tc.wantSeverity {
|
||||
t.Errorf("severity = %q, want %q (verdict %q)", f.Severity, tc.wantSeverity, f.Verdict)
|
||||
}
|
||||
if !strings.Contains(f.Verdict, tc.wantContains) {
|
||||
t.Errorf("verdict %q does not contain %q", f.Verdict, tc.wantContains)
|
||||
}
|
||||
if f.Evidence == "" {
|
||||
t.Error("evidence must not be empty")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJudgePod(t *testing.T) {
|
||||
cases := []struct {
|
||||
phase string
|
||||
wantSeverity string
|
||||
}{
|
||||
{"Pending", "warn"},
|
||||
{"Failed", "error"},
|
||||
{"Succeeded", "ok"},
|
||||
{"Running", "ok"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
var p pod
|
||||
p.Status.Phase = tc.phase
|
||||
if got := judgePod(p); got.Severity != tc.wantSeverity {
|
||||
t.Errorf("phase %s: severity = %q, want %q", tc.phase, got.Severity, tc.wantSeverity)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnoseK8sOutput(t *testing.T) {
|
||||
r := &fakeRunner{
|
||||
available: map[string]bool{"kubectl": true},
|
||||
respond: func(name string, args []string) (CommandResult, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case strings.HasPrefix(joined, "get pod"):
|
||||
return CommandResult{Stdout: []byte(crashLoopPodJSON)}, nil
|
||||
case strings.HasPrefix(joined, "logs"):
|
||||
return CommandResult{Stdout: []byte("panic: boom\n")}, nil
|
||||
case strings.HasPrefix(joined, "get events"):
|
||||
return CommandResult{Stdout: []byte("0s Warning FailedScheduling insufficient cpu\n")}, nil
|
||||
case strings.HasPrefix(joined, "config view"):
|
||||
return CommandResult{}, nil
|
||||
}
|
||||
return CommandResult{}, fmt.Errorf("unexpected call %s", joined)
|
||||
},
|
||||
}
|
||||
|
||||
s, out, errOut := newTestService(r)
|
||||
err := s.Run(context.Background(), []string{"api-0", "--kind", "k8s", "--namespace", "prod"})
|
||||
if err != nil {
|
||||
t.Fatalf("Run error: %v (stderr=%s)", err, errOut.String())
|
||||
}
|
||||
|
||||
text := out.String()
|
||||
for _, want := range []string{
|
||||
"目标: api-0",
|
||||
"来源: Kubernetes",
|
||||
"namespace=prod",
|
||||
"phase=Running",
|
||||
"容器 app:",
|
||||
"CrashLoopBackOff",
|
||||
"lastState.exitCode=1",
|
||||
"日志 (app, 最近 50 行)",
|
||||
"panic: boom",
|
||||
"日志 (app, 上一次)",
|
||||
"FailedScheduling",
|
||||
"事件提示",
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("output missing %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnoseK8sFiltersContainer(t *testing.T) {
|
||||
podJSON := `{
|
||||
"metadata":{"name":"api-0","namespace":"prod"},
|
||||
"status":{"phase":"Running","containerStatuses":[
|
||||
{"name":"one","ready":true,"state":{"running":{}}},
|
||||
{"name":"two","ready":true,"state":{"running":{}}}
|
||||
]}
|
||||
}`
|
||||
|
||||
r := &fakeRunner{
|
||||
available: map[string]bool{"kubectl": true},
|
||||
respond: func(name string, args []string) (CommandResult, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
if strings.HasPrefix(joined, "get pod") {
|
||||
return CommandResult{Stdout: []byte(podJSON)}, nil
|
||||
}
|
||||
return CommandResult{Stdout: []byte("log\n")}, nil
|
||||
},
|
||||
}
|
||||
|
||||
s, out, _ := newTestService(r)
|
||||
if err := s.Run(context.Background(), []string{"api-0", "--kind", "k8s", "-c", "one"}); err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
text := out.String()
|
||||
if !strings.Contains(text, "容器 one:") || strings.Contains(text, "容器 two:") {
|
||||
t.Errorf("container filter not applied:\n%s", text)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package diagnose
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Options controls what a diagnosis collects.
|
||||
type Options struct {
|
||||
Kind string // "", "docker" or "k8s"
|
||||
Namespace string
|
||||
Context string
|
||||
Lines int
|
||||
NoLogs bool
|
||||
NoEvents bool
|
||||
Container string
|
||||
}
|
||||
|
||||
// DefaultLines is the number of log lines collected by default.
|
||||
const DefaultLines = 50
|
||||
|
||||
// ParseArgs parses the arguments forwarded by the CLI. It returns the target
|
||||
// (container or Pod name) and the collection options.
|
||||
func ParseArgs(args []string) (string, Options, error) {
|
||||
opts := Options{Lines: DefaultLines}
|
||||
var targets []string
|
||||
positionalOnly := false
|
||||
|
||||
for i := 0; i < len(args); i++ {
|
||||
arg := args[i]
|
||||
switch {
|
||||
case positionalOnly:
|
||||
targets = append(targets, arg)
|
||||
|
||||
case arg == "--":
|
||||
positionalOnly = true
|
||||
|
||||
case arg == "--kind" || arg == "--kind=":
|
||||
value, err := valueFor(arg, args, &i, "--kind")
|
||||
if err != nil {
|
||||
return "", opts, err
|
||||
}
|
||||
opts.Kind = strings.ToLower(value)
|
||||
if opts.Kind != "docker" && opts.Kind != "k8s" {
|
||||
return "", opts, fmt.Errorf("--kind 只支持 docker 或 k8s，收到 %q", value)
|
||||
}
|
||||
|
||||
case strings.HasPrefix(arg, "--kind="):
|
||||
opts.Kind = strings.ToLower(strings.TrimPrefix(arg, "--kind="))
|
||||
if opts.Kind != "docker" && opts.Kind != "k8s" {
|
||||
return "", opts, fmt.Errorf("--kind 只支持 docker 或 k8s，收到 %q", opts.Kind)
|
||||
}
|
||||
|
||||
case arg == "--namespace" || arg == "-N":
|
||||
value, err := valueFor(arg, args, &i, arg)
|
||||
if err != nil {
|
||||
return "", opts, err
|
||||
}
|
||||
opts.Namespace = value
|
||||
|
||||
case strings.HasPrefix(arg, "--namespace="):
|
||||
opts.Namespace = strings.TrimPrefix(arg, "--namespace=")
|
||||
|
||||
case arg == "--context":
|
||||
value, err := valueFor(arg, args, &i, arg)
|
||||
if err != nil {
|
||||
return "", opts, err
|
||||
}
|
||||
opts.Context = value
|
||||
|
||||
case strings.HasPrefix(arg, "--context="):
|
||||
opts.Context = strings.TrimPrefix(arg, "--context=")
|
||||
|
||||
case arg == "--lines":
|
||||
value, err := valueFor(arg, args, &i, arg)
|
||||
if err != nil {
|
||||
return "", opts, err
|
||||
}
|
||||
n, err := strconv.Atoi(value)
|
||||
if err != nil || n <= 0 {
|
||||
return "", opts, fmt.Errorf("--lines 需要正整数，收到 %q", value)
|
||||
}
|
||||
opts.Lines = n
|
||||
|
||||
case strings.HasPrefix(arg, "--lines="):
|
||||
n, err := strconv.Atoi(strings.TrimPrefix(arg, "--lines="))
|
||||
if err != nil || n <= 0 {
|
||||
return "", opts, fmt.Errorf("--lines 需要正整数，收到 %q", strings.TrimPrefix(arg, "--lines="))
|
||||
}
|
||||
opts.Lines = n
|
||||
|
||||
case arg == "--no-logs":
|
||||
opts.NoLogs = true
|
||||
|
||||
case arg == "--no-events":
|
||||
opts.NoEvents = true
|
||||
|
||||
case arg == "--container" || arg == "-c":
|
||||
value, err := valueFor(arg, args, &i, arg)
|
||||
if err != nil {
|
||||
return "", opts, err
|
||||
}
|
||||
opts.Container = value
|
||||
|
||||
case strings.HasPrefix(arg, "--container="):
|
||||
opts.Container = strings.TrimPrefix(arg, "--container=")
|
||||
|
||||
case strings.HasPrefix(arg, "-") && arg != "-":
|
||||
return "", opts, fmt.Errorf("未知选项 %q", arg)
|
||||
|
||||
default:
|
||||
targets = append(targets, arg)
|
||||
}
|
||||
}
|
||||
|
||||
if len(targets) == 0 {
|
||||
return "", opts, fmt.Errorf("必须提供容器名或 Pod 名，例如 kopsc diagnose web")
|
||||
}
|
||||
if len(targets) > 1 {
|
||||
return "", opts, fmt.Errorf("一次只能诊断一个目标，收到 %d 个：%s", len(targets), strings.Join(targets, ", "))
|
||||
}
|
||||
|
||||
if opts.Kind == "docker" && (opts.Namespace != "" || opts.Context != "") {
|
||||
return "", opts, fmt.Errorf("Docker 诊断不支持 --namespace/--context；请改用 --kind k8s")
|
||||
}
|
||||
|
||||
return targets[0], opts, nil
|
||||
}
|
||||
|
||||
// valueFor reads the value of a flag written as "--flag value" or "--flag=".
|
||||
func valueFor(flag string, args []string, index *int, name string) (string, error) {
|
||||
if strings.HasSuffix(flag, "=") {
|
||||
return "", fmt.Errorf("%s 缺少参数", name)
|
||||
}
|
||||
if *index+1 >= len(args) {
|
||||
return "", fmt.Errorf("%s 缺少参数，例如 %s value", name, name)
|
||||
}
|
||||
*index = *index + 1
|
||||
return args[*index], nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package diagnose
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseArgs(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args []string
|
||||
wantTarget string
|
||||
wantOpts Options
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "target only",
|
||||
args: []string{"web"},
|
||||
wantTarget: "web",
|
||||
wantOpts: Options{Lines: DefaultLines},
|
||||
},
|
||||
{
|
||||
name: "namespace",
|
||||
args: []string{"api-0", "--namespace", "prod"},
|
||||
wantTarget: "api-0",
|
||||
wantOpts: Options{Lines: DefaultLines, Namespace: "prod"},
|
||||
},
|
||||
{
|
||||
name: "short namespace",
|
||||
args: []string{"api-0", "-N", "prod"},
|
||||
wantTarget: "api-0",
|
||||
wantOpts: Options{Lines: DefaultLines, Namespace: "prod"},
|
||||
},
|
||||
{
|
||||
name: "namespace equals",
|
||||
args: []string{"api-0", "--namespace=prod"},
|
||||
wantTarget: "api-0",
|
||||
wantOpts: Options{Lines: DefaultLines, Namespace: "prod"},
|
||||
},
|
||||
{
|
||||
name: "context and lines",
|
||||
args: []string{"api-0", "--context", "staging", "--lines", "200"},
|
||||
wantTarget: "api-0",
|
||||
wantOpts: Options{Lines: 200, Context: "staging"},
|
||||
},
|
||||
{
|
||||
name: "flags",
|
||||
args: []string{"web", "--no-logs", "--no-events", "--container", "app"},
|
||||
wantTarget: "web",
|
||||
wantOpts: Options{Lines: DefaultLines, NoLogs: true, NoEvents: true, Container: "app"},
|
||||
},
|
||||
{
|
||||
name: "short container",
|
||||
args: []string{"web", "-c", "app"},
|
||||
wantTarget: "web",
|
||||
wantOpts: Options{Lines: DefaultLines, Container: "app"},
|
||||
},
|
||||
{
|
||||
name: "kind",
|
||||
args: []string{"web", "--kind", "docker"},
|
||||
wantTarget: "web",
|
||||
wantOpts: Options{Lines: DefaultLines, Kind: "docker"},
|
||||
},
|
||||
{
|
||||
name: "after separator",
|
||||
args: []string{"--", "-n"},
|
||||
wantTarget: "-n",
|
||||
wantOpts: Options{Lines: DefaultLines},
|
||||
},
|
||||
{name: "missing target", args: nil, wantErr: "必须提供"},
|
||||
{name: "two targets", args: []string{"a", "b"}, wantErr: "一次只能诊断一个"},
|
||||
{name: "bad kind", args: []string{"a", "--kind", "nomad"}, wantErr: "只支持 docker 或 k8s"},
|
||||
{name: "docker with namespace", args: []string{"a", "--kind", "docker", "-N", "prod"}, wantErr: "不支持 --namespace"},
|
||||
{name: "bad lines", args: []string{"a", "--lines", "0"}, wantErr: "正整数"},
|
||||
{name: "unknown option", args: []string{"a", "--wat"}, wantErr: "未知选项"},
|
||||
{name: "missing value", args: []string{"a", "--namespace"}, wantErr: "缺少参数"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
target, opts, err := ParseArgs(tc.args)
|
||||
if tc.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
|
||||
t.Fatalf("err = %v, want %q", err, tc.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if target != tc.wantTarget {
|
||||
t.Errorf("target = %q, want %q", target, tc.wantTarget)
|
||||
}
|
||||
if opts != tc.wantOpts {
|
||||
t.Errorf("options = %+v, want %+v", opts, tc.wantOpts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Package diagnose explains why a container or Pod is in its current state.
|
||||
// It knows how to inspect Docker/Podman containers and Kubernetes Pods, and
|
||||
// falls back to a built-in implementation so the output is identical on every
|
||||
// platform.
|
||||
package diagnose
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
// CommandResult is the output of one external command.
|
||||
type CommandResult struct {
|
||||
Stdout []byte
|
||||
Stderr []byte
|
||||
ExitCode int
|
||||
}
|
||||
|
||||
// Runner executes external programs. It is an interface so diagnoses can be
|
||||
// tested without docker or kubectl.
|
||||
type Runner interface {
|
||||
LookPath(name string) (string, error)
|
||||
Run(ctx context.Context, name string, args ...string) (CommandResult, error)
|
||||
}
|
||||
|
||||
// ExecRunner runs commands on the host.
|
||||
type ExecRunner struct{}
|
||||
|
||||
// LookPath reports whether a program is available on PATH.
|
||||
func (ExecRunner) LookPath(name string) (string, error) {
|
||||
return exec.LookPath(name)
|
||||
}
|
||||
|
||||
// Run executes a command and captures its output.
|
||||
func (ExecRunner) Run(ctx context.Context, name string, args ...string) (CommandResult, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
err := cmd.Run()
|
||||
result := CommandResult{Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) {
|
||||
result.ExitCode = exitErr.ExitCode()
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package diagnose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.odboy.cn/odboy/kenaito-kopsc/internal/platform"
|
||||
)
|
||||
|
||||
// Finding is one judgement about a container.
|
||||
type Finding struct {
|
||||
Severity string // "ok", "warn" or "error"
|
||||
Verdict string
|
||||
Evidence string
|
||||
Advice string
|
||||
}
|
||||
|
||||
// Service collects and renders diagnoses.
|
||||
type Service struct {
|
||||
info platform.Info
|
||||
runner Runner
|
||||
out io.Writer
|
||||
errOut io.Writer
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// New builds a Service. runner is usually ExecRunner; tests inject a fake.
|
||||
func New(info platform.Info, runner Runner, out, errOut io.Writer) *Service {
|
||||
if runner == nil {
|
||||
runner = ExecRunner{}
|
||||
}
|
||||
if out == nil {
|
||||
out = io.Discard
|
||||
}
|
||||
if errOut == nil {
|
||||
errOut = io.Discard
|
||||
}
|
||||
return &Service{info: info, runner: runner, out: out, errOut: errOut, now: time.Now}
|
||||
}
|
||||
|
||||
// Run parses the arguments forwarded by kopsc and diagnoses the target.
|
||||
func (s *Service) Run(ctx context.Context, args []string) error {
|
||||
target, opts, err := ParseArgs(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.diagnose(ctx, target, opts)
|
||||
}
|
||||
|
||||
type source struct {
|
||||
kind string // "docker" or "k8s"
|
||||
program string // docker or podman
|
||||
namespace string
|
||||
context string
|
||||
}
|
||||
|
||||
func (s *Service) diagnose(ctx context.Context, target string, opts Options) error {
|
||||
src, err := s.resolve(ctx, target, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch src.kind {
|
||||
case "docker":
|
||||
return s.diagnoseDocker(ctx, src, target, opts)
|
||||
case "k8s":
|
||||
return s.diagnoseK8s(ctx, src, target, opts)
|
||||
default:
|
||||
return fmt.Errorf("未知诊断来源 %q", src.kind)
|
||||
}
|
||||
}
|
||||
|
||||
// resolve decides whether the target is a container or a Pod.
|
||||
func (s *Service) resolve(ctx context.Context, target string, opts Options) (source, error) {
|
||||
switch opts.Kind {
|
||||
case "docker":
|
||||
program, err := s.dockerProgram()
|
||||
if err != nil {
|
||||
return source{}, err
|
||||
}
|
||||
return source{kind: "docker", program: program}, nil
|
||||
|
||||
case "k8s":
|
||||
ns := opts.Namespace
|
||||
if ns == "" {
|
||||
ns, _ = s.kubeNamespace(ctx, opts.Context)
|
||||
}
|
||||
return source{kind: "k8s", namespace: ns, context: opts.Context}, nil
|
||||
}
|
||||
|
||||
// Providing namespace/context is an explicit signal for Kubernetes.
|
||||
if opts.Namespace != "" || opts.Context != "" {
|
||||
ns := opts.Namespace
|
||||
if ns == "" {
|
||||
ns, _ = s.kubeNamespace(ctx, opts.Context)
|
||||
}
|
||||
return source{kind: "k8s", namespace: ns, context: opts.Context}, nil
|
||||
}
|
||||
|
||||
var dockerErr error
|
||||
if program, err := s.dockerProgram(); err == nil {
|
||||
if res, err := s.run(ctx, program, "inspect", "--type", "container", target); err == nil {
|
||||
var items []json.RawMessage
|
||||
if json.Unmarshal(res.Stdout, &items) == nil && len(items) > 0 {
|
||||
return source{kind: "docker", program: program}, nil
|
||||
}
|
||||
dockerErr = fmt.Errorf("Docker 中没有名为 %q 的容器", target)
|
||||
} else {
|
||||
dockerErr = err
|
||||
}
|
||||
} else {
|
||||
dockerErr = err
|
||||
}
|
||||
|
||||
if _, err := s.runner.LookPath("kubectl"); err == nil {
|
||||
ns, _ := s.kubeNamespace(ctx, opts.Context)
|
||||
if res, err := s.run(ctx, "kubectl", kubectlArgs(opts.Context, "get", "pod", target, "-n", ns, "-o", "name")...); err == nil &&
|
||||
strings.TrimSpace(string(res.Stdout)) != "" {
|
||||
return source{kind: "k8s", namespace: ns, context: opts.Context}, nil
|
||||
}
|
||||
return source{}, fmt.Errorf("未找到目标 %q：%v；Kubernetes 命名空间 %s 中也没有该 Pod", target, dockerErr, ns)
|
||||
}
|
||||
|
||||
return source{}, fmt.Errorf("未找到目标 %q：%v；同时未安装 kubectl", target, dockerErr)
|
||||
}
|
||||
|
||||
func (s *Service) dockerProgram() (string, error) {
|
||||
for _, name := range []string{"docker", "podman"} {
|
||||
if _, err := s.runner.LookPath(name); err == nil {
|
||||
return name, nil
|
||||
}
|
||||
}
|
||||
for _, name := range s.info.Runtimes {
|
||||
if name == "docker" || name == "podman" {
|
||||
return name, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("未找到 docker 或 podman")
|
||||
}
|
||||
|
||||
// run executes a command with a per-command timeout.
|
||||
func (s *Service) run(ctx context.Context, name string, args ...string) (CommandResult, error) {
|
||||
callCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
res, err := s.runner.Run(callCtx, name, args...)
|
||||
if err != nil {
|
||||
message := summarizeStderr(string(res.Stderr))
|
||||
if message == "" {
|
||||
message = err.Error()
|
||||
}
|
||||
return res, fmt.Errorf("%s %s 失败: %s", name, strings.Join(args, " "), message)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// summarizeStderr keeps the actionable last line of a multi-line error (kubectl
|
||||
// prints its connection error last) and trims the rest.
|
||||
func summarizeStderr(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
lines := strings.Split(raw, "\n")
|
||||
return truncate(strings.TrimSpace(lines[len(lines)-1]), 300)
|
||||
}
|
||||
|
||||
func kubectlArgs(context string, args ...string) []string {
|
||||
if context == "" {
|
||||
return args
|
||||
}
|
||||
out := make([]string, 0, 2+len(args))
|
||||
out = append(out, "--context", context)
|
||||
return append(out, args...)
|
||||
}
|
||||
|
||||
// kubeNamespace resolves the effective namespace: --namespace, then the
|
||||
// context's kubeconfig namespace, then "default".
|
||||
func (s *Service) kubeNamespace(ctx context.Context, context string) (string, error) {
|
||||
res, err := s.run(ctx, "kubectl", kubectlArgs(context, "config", "view", "--minify", "-o", "jsonpath={..namespace}")...)
|
||||
if err != nil {
|
||||
return "default", err
|
||||
}
|
||||
ns := strings.TrimSpace(string(res.Stdout))
|
||||
if ns == "" {
|
||||
return "default", nil
|
||||
}
|
||||
return ns, nil
|
||||
}
|
||||
|
||||
func (s *Service) renderFinding(indent string, f Finding) {
|
||||
icon := "✅"
|
||||
switch f.Severity {
|
||||
case "error":
|
||||
icon = "❌"
|
||||
case "warn":
|
||||
icon = "⚠️"
|
||||
}
|
||||
fmt.Fprintf(s.out, "%s判定: %s %s\n", indent, icon, f.Verdict)
|
||||
if f.Evidence != "" {
|
||||
fmt.Fprintf(s.out, "%s依据: %s\n", indent, f.Evidence)
|
||||
}
|
||||
if f.Advice != "" {
|
||||
fmt.Fprintf(s.out, "%s建议: %s\n", indent, f.Advice)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) writeSection(title, content string) {
|
||||
fmt.Fprintf(s.out, "\n%s:\n", title)
|
||||
if strings.TrimSpace(content) == "" {
|
||||
fmt.Fprintln(s.out, "(无)")
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(s.out, content)
|
||||
}
|
||||
|
||||
func tailLines(text string, n int) string {
|
||||
text = strings.TrimRight(text, "\n")
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return ""
|
||||
}
|
||||
lines := strings.Split(text, "\n")
|
||||
if n > 0 && len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package diagnose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolveAutoDocker(t *testing.T) {
|
||||
inspectJSON := `[{"Id":"abc","Name":"/web","State":{"Status":"running"}}]`
|
||||
r := &fakeRunner{
|
||||
available: map[string]bool{"docker": true, "kubectl": true},
|
||||
respond: func(name string, args []string) (CommandResult, error) {
|
||||
switch {
|
||||
case len(args) > 0 && args[0] == "inspect":
|
||||
return CommandResult{Stdout: []byte(inspectJSON)}, nil
|
||||
case len(args) > 0 && args[0] == "logs":
|
||||
return CommandResult{Stdout: []byte("ok\n")}, nil
|
||||
case len(args) > 0 && args[0] == "events":
|
||||
return CommandResult{}, nil
|
||||
}
|
||||
return CommandResult{}, fmt.Errorf("unexpected call %s %v", name, args)
|
||||
},
|
||||
}
|
||||
|
||||
s, out, _ := newTestService(r)
|
||||
if err := s.Run(context.Background(), []string{"web"}); err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "来源: Docker (docker)") {
|
||||
t.Errorf("expected docker source:\n%s", out.String())
|
||||
}
|
||||
if r.called("kubectl") {
|
||||
t.Error("kubectl should not be consulted when docker hits")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAutoKubernetes(t *testing.T) {
|
||||
r := &fakeRunner{
|
||||
available: map[string]bool{"docker": true, "kubectl": true},
|
||||
respond: func(name string, args []string) (CommandResult, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case len(args) > 0 && args[0] == "inspect":
|
||||
return CommandResult{Stdout: []byte("[]")}, nil
|
||||
case strings.HasPrefix(joined, "config view"):
|
||||
return CommandResult{}, nil
|
||||
case strings.HasPrefix(joined, "get pod"):
|
||||
return CommandResult{Stdout: []byte("pod/api-0\n")}, nil
|
||||
case strings.HasPrefix(joined, "logs"):
|
||||
return CommandResult{Stdout: []byte("log\n")}, nil
|
||||
case strings.HasPrefix(joined, "get events"):
|
||||
return CommandResult{}, nil
|
||||
}
|
||||
return CommandResult{}, fmt.Errorf("unexpected call %s", joined)
|
||||
},
|
||||
}
|
||||
|
||||
s, out, _ := newTestService(r)
|
||||
// The fake returns name output for the first pod probe but diagnoseK8s
|
||||
// needs JSON; swap the responder after resolve would be brittle, so return
|
||||
// JSON for every get pod call instead.
|
||||
r.respond = func(name string, args []string) (CommandResult, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case len(args) > 0 && args[0] == "inspect":
|
||||
return CommandResult{Stdout: []byte("[]")}, nil
|
||||
case strings.HasPrefix(joined, "config view"):
|
||||
return CommandResult{}, nil
|
||||
case strings.HasPrefix(joined, "get pod"):
|
||||
return CommandResult{Stdout: []byte(crashLoopPodJSON)}, nil
|
||||
case strings.HasPrefix(joined, "logs"):
|
||||
return CommandResult{Stdout: []byte("log\n")}, nil
|
||||
case strings.HasPrefix(joined, "get events"):
|
||||
return CommandResult{}, nil
|
||||
}
|
||||
return CommandResult{}, fmt.Errorf("unexpected call %s", joined)
|
||||
}
|
||||
|
||||
if err := s.Run(context.Background(), []string{"api-0"}); err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "来源: Kubernetes") {
|
||||
t.Errorf("expected kubernetes source:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveNotFound(t *testing.T) {
|
||||
r := &fakeRunner{
|
||||
available: map[string]bool{"docker": true, "kubectl": true},
|
||||
respond: func(name string, args []string) (CommandResult, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case len(args) > 0 && args[0] == "inspect":
|
||||
return CommandResult{Stdout: []byte("[]")}, nil
|
||||
case strings.HasPrefix(joined, "config view"):
|
||||
return CommandResult{}, nil
|
||||
case strings.HasPrefix(joined, "get pod"):
|
||||
return CommandResult{Stderr: []byte("not found")}, fmt.Errorf("exit status 1")
|
||||
}
|
||||
return CommandResult{}, nil
|
||||
},
|
||||
}
|
||||
|
||||
s, _, _ := newTestService(r)
|
||||
err := s.Run(context.Background(), []string{"ghost"})
|
||||
if err == nil || !strings.Contains(err.Error(), "未找到目标") {
|
||||
t.Fatalf("err = %v, want not-found error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveNamespaceSkipsDocker(t *testing.T) {
|
||||
r := &fakeRunner{
|
||||
available: map[string]bool{"docker": true, "kubectl": true},
|
||||
respond: func(name string, args []string) (CommandResult, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case strings.HasPrefix(joined, "get pod"):
|
||||
return CommandResult{Stdout: []byte(crashLoopPodJSON)}, nil
|
||||
case strings.HasPrefix(joined, "logs"):
|
||||
return CommandResult{}, nil
|
||||
case strings.HasPrefix(joined, "get events"):
|
||||
return CommandResult{}, nil
|
||||
}
|
||||
return CommandResult{}, fmt.Errorf("unexpected call %s", joined)
|
||||
},
|
||||
}
|
||||
|
||||
s, out, _ := newTestService(r)
|
||||
if err := s.Run(context.Background(), []string{"api-0", "--namespace", "prod"}); err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if r.called("inspect") {
|
||||
t.Error("namespace should skip the docker probe")
|
||||
}
|
||||
if !strings.Contains(out.String(), "namespace=prod") {
|
||||
t.Errorf("namespace missing from output:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user