feat: 扫描整理并行化、WebSocket 实时进度与统计看板

- 移动阶段并行处理文件,按原子计数上报累计进度
- 扫描遍历阶段实时上报已发现文件数,界面不再卡在 0/0
- 新增 /api/scan/ws WebSocket 实时推送扫描状态;前端断线指数退避自动重连,并以低频轮询兜底
- 新增统计看板与媒体库筛选/排序,目录选择器路径不存在时回退到最近可用父目录
- 存放目录支持自定义并持久化,默认输出到 KMP已整理
This commit is contained in:
2025-05-08 21:16:20 +08:00
parent 36d7ecced8
commit f915a9b1ac
22 changed files with 1342 additions and 281 deletions
+1 -1
View File
@@ -173,7 +173,7 @@ lightScore = 亮度 > 175 的像素占比
## 注意事项
- 默认使用**移动**模式,原文件会从源目录移走;同名文件跨磁盘时自动降级为复制 + 删除
- 已整理文件全部移动到 `{当前目录名}-picker` 白名单目录内,重复运行会跳过该目录,并将新文件并入已有的分类目录
- 已整理文件全部移动到 `KMP已整理` 白名单目录内,重复运行会跳过该目录,并将新文件并入已有的分类目录
- 扫描整理由页面点击触发,整理期间服务会短暂占用较多 CPU/磁盘,属正常现象
- 同一时间只允许一次扫描整理;重复点击会提示「正在运行中」
- GIF 图片单独归入「图片_GIF」;GIF 视频按方向归入「视频_*_GIF」
-42
View File
@@ -463,9 +463,6 @@
"x64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -567,9 +564,6 @@
"arm"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -584,9 +578,6 @@
"arm"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -601,9 +592,6 @@
"arm64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -618,9 +606,6 @@
"arm64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -635,9 +620,6 @@
"loong64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -652,9 +634,6 @@
"loong64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -669,9 +648,6 @@
"ppc64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -686,9 +662,6 @@
"ppc64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -703,9 +676,6 @@
"riscv64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -720,9 +690,6 @@
"riscv64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -737,9 +704,6 @@
"s390x"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -754,9 +718,6 @@
"x64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -771,9 +732,6 @@
"x64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
+4 -4
View File
@@ -12,14 +12,14 @@
text-color="#4a5a78"
active-text-color="#1677ff"
>
<el-menu-item index="/dashboard">
<i class="el-icon-data-line"></i>
<span slot="title">统计看板</span>
</el-menu-item>
<el-menu-item index="/scan">
<i class="el-icon-video-play"></i>
<span slot="title">扫描整理</span>
</el-menu-item>
<el-menu-item index="/dashboard">
<i class="el-icon-data-line"></i>
<span slot="title">统计看板</span>
</el-menu-item>
<el-menu-item index="/media">
<i class="el-icon-film"></i>
<span slot="title">媒体库</span>
+1 -1
View File
@@ -8,7 +8,7 @@ Vue.use(Router)
export default new Router({
routes: [
{path: '/', redirect: '/dashboard'},
{path: '/', redirect: '/scan'},
{path: '/dashboard', component: Dashboard},
{path: '/media', component: MediaList},
{path: '/scan', component: Scan}
+123 -1
View File
@@ -16,6 +16,21 @@
<div class="stat-value">{{ fmtSize(stats.totalSize) }}</div>
<div class="stat-sub">所有文件占用</div>
</div>
<div class="stat-card">
<div class="stat-label">平均文件</div>
<div class="stat-value">{{ fmtSize(stats.avgSize) }}</div>
<div class="stat-sub">单文件平均体积</div>
</div>
<div class="stat-card">
<div class="stat-label">最大文件</div>
<div class="stat-value">{{ fmtSize(stats.maxSize) }}</div>
<div class="stat-sub">体积最大的文件</div>
</div>
<div class="stat-card">
<div class="stat-label">最小文件</div>
<div class="stat-value">{{ fmtSize(stats.minSize) }}</div>
<div class="stat-sub">体积最小的文件</div>
</div>
<div class="stat-card">
<div class="stat-label">重复文件</div>
<div class="stat-value warning">{{ stats.duplicate }}</div>
@@ -26,6 +41,11 @@
<div class="stat-value warning">{{ fmtSize(stats.dupSize) }}</div>
<div class="stat-sub">可释放空间(约)</div>
</div>
<div class="stat-card">
<div class="stat-label">总时长</div>
<div class="stat-value">{{ fmtDuration(stats.durationTotal) }}</div>
<div class="stat-sub">平均 {{ fmtDuration(stats.durationAvg) }}</div>
</div>
</div>
<div class="chart-grid">
@@ -86,6 +106,66 @@
</div>
<div v-if="stats.formatCount.length === 0" class="empty-tip">暂无格式数据</div>
</div>
<div class="panel">
<div class="panel-title">时长分布(视频/音频)</div>
<div v-for="item in stats.durationBins" :key="item.key" class="dist-row">
<span class="dist-label">{{ item.key }}</span>
<el-progress :percentage="percent(item.count, durationTotal)" :stroke-width="14" color="#722ed1"/>
<span class="dist-count">{{ item.count }}</span>
</div>
<div v-if="stats.durationBins.length === 0" class="empty-tip">当前媒体库暂无时长数据,扫描含视频/音频的目录后显示</div>
</div>
<div class="panel top-panel">
<div class="panel-title">入库趋势(近30天)</div>
<div v-for="item in (stats.createdCounts || [])" :key="item.key" class="dist-row">
<span class="dist-label">{{ item.key }}</span>
<el-progress :percentage="percent(item.count, createdTotal)" :stroke-width="14" color="#13c2c2"/>
<span class="dist-count">{{ item.count }}</span>
</div>
<div v-if="!stats.createdCounts || stats.createdCounts.length === 0" class="empty-tip">暂无入库记录</div>
</div>
<div class="panel top-panel">
<div class="panel-title">归类目录分布</div>
<div v-for="item in (stats.categoryCounts || [])" :key="item.key" class="dist-row">
<span class="dist-label">{{ item.key }}</span>
<el-progress :percentage="percent(item.count, categoryTotal)" :stroke-width="14" color="#fa8c16"/>
<span class="dist-count">{{ item.count }}</span>
</div>
<div v-if="!stats.categoryCounts || stats.categoryCounts.length === 0" class="empty-tip">暂无归类记录(扫描整理后显示)</div>
</div>
<div class="panel top-panel">
<div class="panel-title">扩展名分布</div>
<div v-for="item in stats.extCounts" :key="item.key" class="dist-row">
<span class="dist-label">{{ item.key }}</span>
<el-progress :percentage="percent(item.count, extTotal)" :stroke-width="14" color="#eb2f96"/>
<span class="dist-count">{{ item.count }}</span>
</div>
<div v-if="stats.extCounts.length === 0" class="empty-tip">暂无扩展名数据</div>
</div>
<div class="panel">
<div class="panel-title">MIME 类型分布</div>
<div v-for="item in stats.mimeCounts" :key="item.key" class="dist-row">
<span class="dist-label">{{ item.key }}</span>
<el-progress :percentage="percent(item.count, mimeTotal)" :stroke-width="14" color="#2f54eb"/>
<span class="dist-count">{{ item.count }}</span>
</div>
<div v-if="stats.mimeCounts.length === 0" class="empty-tip">暂无 MIME 数据</div>
</div>
<div class="panel">
<div class="panel-title">音频编码分布</div>
<div v-for="item in stats.audioCodecCount" :key="item.key" class="dist-row">
<span class="dist-label">{{ item.key }}</span>
<el-progress :percentage="percent(item.count, audioCodecTotal)" :stroke-width="14" color="#13c2c2"/>
<span class="dist-count">{{ item.count }}</span>
</div>
<div v-if="stats.audioCodecCount.length === 0" class="empty-tip">当前媒体库暂无音频数据,扫描含音频文件的目录后显示</div>
</div>
</div>
<div class="panel top-panel">
@@ -119,7 +199,18 @@ export default {
formatCount: [],
duplicate: 0,
dupSize: 0,
topLarge: []
topLarge: [],
durationTotal: 0,
durationAvg: 0,
avgSize: 0,
maxSize: 0,
minSize: 0,
durationBins: [],
createdCounts: [],
categoryCounts: [],
extCounts: [],
mimeCounts: [],
audioCodecCount: []
}
}
},
@@ -135,6 +226,24 @@ export default {
},
formatTotal() {
return this.sumCount(this.stats.formatCount)
},
durationTotal() {
return this.sumCount(this.stats.durationBins)
},
createdTotal() {
return this.sumCount(this.stats.createdCounts)
},
categoryTotal() {
return this.sumCount(this.stats.categoryCounts)
},
extTotal() {
return this.sumCount(this.stats.extCounts)
},
mimeTotal() {
return this.sumCount(this.stats.mimeCounts)
},
audioCodecTotal() {
return this.sumCount(this.stats.audioCodecCount)
}
},
mounted() {
@@ -168,6 +277,15 @@ export default {
}
return size.toFixed(size >= 100 || i === 0 ? 0 : 1) + ' ' + units[i]
},
fmtDuration(sec) {
if (!sec && sec !== 0) return '-'
const h = Math.floor(sec / 3600)
const m = Math.floor((sec % 3600) / 60)
const s = Math.floor(sec % 60)
if (h > 0) return h + 'h' + (m > 0 ? m + 'm' : '')
if (m > 0) return m + 'm' + s + 's'
return s + 's'
},
typeName(type) {
return {image: '图片', video: '视频', audio: '音频'}[type] || type
},
@@ -266,6 +384,10 @@ export default {
text-overflow: ellipsis;
}
.top-panel .dist-label {
width: 260px;
}
.dist-count {
width: 44px;
text-align: right;
+7 -7
View File
@@ -32,7 +32,7 @@
</div>
<el-table :data="list" v-loading="loading" border stripe @sort-change="handleSortChange">
<el-table-column prop="fileName" label="文件名" min-width="220" show-overflow-tooltip>
<el-table-column prop="fileName" label="文件名" min-width="100" show-overflow-tooltip>
<template slot-scope="scope">
<span class="file-name" :title="scope.row.filePath">{{ scope.row.fileName }}</span>
</template>
@@ -48,7 +48,7 @@
<span v-else>{{ scope.row.orientation }}</span>
</template>
</el-table-column>
<el-table-column prop="resolution" label="分辨率" width="110" align="center" sortable="custom">
<el-table-column prop="resolution" label="分辨率" width="120" align="center" sortable="custom">
<template slot-scope="scope">
<span class="muted" v-if="!scope.row.resolution">-</span>
<span v-else>{{ scope.row.resolution }}</span>
@@ -60,19 +60,19 @@
<span v-else class="muted">-</span>
</template>
</el-table-column>
<el-table-column prop="codec" label="编码" width="90" align="center">
<el-table-column prop="codec" label="编码" width="120" align="center">
<template slot-scope="scope">
<span class="muted" v-if="!scope.row.codec">-</span>
<span v-else>{{ scope.row.codec }}</span>
</template>
</el-table-column>
<el-table-column prop="format" label="格式" width="80" align="center">
<el-table-column prop="format" label="格式" width="120" align="center">
<template slot-scope="scope">
<span class="muted" v-if="!scope.row.format">-</span>
<span v-else>{{ scope.row.format }}</span>
</template>
</el-table-column>
<el-table-column prop="duration" label="时长" width="90" align="center" sortable="custom">
<el-table-column prop="duration" label="时长" width="120" align="center" sortable="custom">
<template slot-scope="scope">
<span class="muted" v-if="!scope.row.durationText">-</span>
<span v-else>{{ scope.row.durationText }}</span>
@@ -87,7 +87,7 @@
<span v-else>{{ scope.row.destDir }}</span>
</template>
</el-table-column>
<el-table-column prop="createdAt" label="入库时间" width="150" align="center" sortable="custom">
<el-table-column prop="createdAt" label="入库时间" width="200" align="center" sortable="custom">
<template slot-scope="scope">
<span class="muted" v-if="!scope.row.createdAt">-</span>
<span v-else>{{ formatTime(scope.row.createdAt) }}</span>
@@ -134,7 +134,7 @@ export default {
formatOptions: [],
query: {
page: 1,
pageSize: 10,
pageSize: 20,
mediaType: '',
orientation: '',
resolution: '',
+249 -37
View File
@@ -2,12 +2,25 @@
<div class="page-card">
<div class="toolbar">
<span class="title">扫描整理</span>
<span class="conn-hint" :class="{off: !wsAlive && !usingPolling}">
<i :class="connText.icon"></i>{{ connText.text }}
</span>
</div>
<div class="scan-tip">
<i class="el-icon-warning-outline"></i>
点击开始后,程序将扫描所选目录并自动归类媒体文件(<b>会移动文件</b>,整理前请先备份)。
整理期间请保持服务运行,结束后可在「媒体库」查看结果。
<div class="scan-tip-body">
<p><b>会移动文件,整理前请先备份!</b></p>
<ul>
<li>程序将<b>递归扫描</b>扫描目录下的全部文件,图片/视频/音频会被探测并按规则归类移动</li>
<li>归类后的文件<b>移动</b>到「存放目录」下的分类子目录(未填写时默认在扫描目录下创建 <b>KMP已整理</b> 文件夹)</li>
<li>跨磁盘(移动失败时)自动降级为<b>复制后删除</b>,原文件仍会从源目录移除</li>
<li>与库中已有记录 <b>MD5 相同的重复文件</b>会被移动到「重复文件」目录</li>
<li>无法读取/不支持的格式会被移动到「读取异常」「已忽略」目录</li>
<li>扫描会自动跳过 <b>KMP已整理</b> 输出目录、<b>media-info.db</b> 数据库与 ffprobe 临时文件</li>
<li>整理期间请保持服务运行,结束后可在「媒体库」查看结果</li>
</ul>
</div>
</div>
<div class="scan-form">
@@ -19,7 +32,18 @@
clearable
size="medium"
@keyup.enter.native="handleStart">
<el-button slot="append" icon="el-icon-folder-opened" @click="openDirPicker">选择目录</el-button>
<el-button slot="append" icon="el-icon-folder-opened" @click="openDirPicker('scan')">选择目录</el-button>
</el-input>
</div>
<div class="form-row" style="margin-top: 10px">
<span class="form-label">存放目录</span>
<el-input
v-model="destDir"
placeholder="留空则默认存放到扫描目录下的「KMP已整理」文件夹"
clearable
size="medium"
@keyup.enter.native="handleStart">
<el-button slot="append" icon="el-icon-folder-opened" @click="openDirPicker('dest')">选择目录</el-button>
</el-input>
</div>
</div>
@@ -33,8 +57,16 @@
:loading="starting"
@click="handleStart">开始扫描</el-button>
<div v-else class="running-hint">
<span>
<i class="el-icon-loading"></i>
<span>扫描整理中,已运行 {{ status.elapsed }} 秒...</span>
扫描整理中,已运行 {{ status.elapsed }} 秒...({{ stageText }} {{ progressText }})
<span v-if="etaText" class="eta-text">预计剩余 {{ etaText }}</span>
</span>
<el-progress
class="scan-progress"
:percentage="status.progress"
:stroke-width="14"
:status="status.progress >= 100 ? 'success' : ''"/>
</div>
</div>
@@ -56,7 +88,7 @@
class="state-alert"/>
<el-dialog
title="选择要整理的目录"
title="选择目录"
:visible.sync="dirPickerVisible"
width="520px"
append-to-body>
@@ -115,8 +147,10 @@
<span class="muted" v-if="status.rootDir">扫描目录:{{ status.rootDir }}</span>
<span class="muted" v-if="status.started">开始于 {{ status.started }}</span>
</div>
<div ref="logBox" class="log-body" v-loading="status.state === 'running'">
<div v-if="status.lines.length === 0" class="muted empty-log">暂无日志,点击「开始扫描」后在此查看</div>
<div ref="logBox" class="log-body">
<div v-if="status.lines.length === 0" class="muted empty-log">
{{ status.state === 'running' ? '扫描进行中,等待日志输出…' : '暂无日志,点击「开始扫描」后在此查看' }}
</div>
<div v-for="(line, idx) in status.lines" :key="idx" class="log-line">{{ line }}</div>
</div>
</div>
@@ -132,16 +166,30 @@ export default {
return {
starting: false,
scanDir: localStorage.getItem('scan-dir') || '',
destDir: localStorage.getItem('scan-dest-dir') || '',
status: {
state: 'idle',
message: '',
started: '',
elapsed: 0,
rootDir: '',
lines: []
destDir: '',
lines: [],
initialized: false,
stage: '',
progress: 0,
done: 0,
total: 0
},
ws: null,
wsAlive: false,
usingPolling: false,
wsClosed: false,
wsReconnectTimer: null,
wsReconnectDelay: 1000,
pollTimer: null,
dirPickerVisible: false,
pickerTarget: 'scan',
dirLoading: false,
fsData: {
current: '',
@@ -152,44 +200,162 @@ export default {
}
},
mounted() {
this.loadStatus()
this.fetchStatus()
this.connectWs()
},
destroyed() {
this.stopPoll()
this.closeWs()
},
computed: {
stageText() {
return {
scan: '扫描文件',
organize: '整理归类',
move: '移动文件'
}[this.status.stage] || ''
},
// 总数未知(遍历目录树阶段)时只显示已发现数,避免卡在 0/0
progressText() {
const {total, done} = this.status
if (total > 0) return done + '/' + total
if (done > 0) return '已发现 ' + done + ' 个文件'
return '准备中…'
},
// 后端按当前阶段速率估算的剩余时间;0 表示无法估算
etaText() {
const eta = this.status.eta
if (!eta || eta <= 0) return ''
return this.fmtDuration(Math.round(eta))
},
// 实时推送连接状态文案
connText() {
if (this.wsAlive) return {icon: 'el-icon-link', text: '实时推送已连接'}
if (this.usingPolling) return {icon: 'el-icon-refresh', text: '轮询模式'}
return {icon: 'el-icon-loading', text: '实时连接中断,自动重连中…'}
}
},
methods: {
async loadStatus() {
try {
const res = await request.get('/api/scan/status')
const data = res.data || {}
// 统一处理状态更新(HTTP 响应与 WebSocket 消息共用)
applyStatus(data) {
if (!data) return
const wasRunning = this.status.state === 'running'
const isRunning = data.state === 'running'
const prevLineCount = (this.status.lines || []).length
if (!this.status.initialized && data.destDir) {
this.destDir = data.destDir
}
this.status = data
if (isRunning && !wasRunning) this.startPoll()
this.status.initialized = true
if (isRunning && data.lines && data.lines.length !== prevLineCount) {
this.scrollBottom()
}
if (wasRunning && !isRunning) {
this.stopPoll()
this.$message.success(data.message || '扫描完成')
}
if (isRunning) this.scrollBottom()
},
async fetchStatus() {
try {
const res = await request.get('/api/scan/status')
this.applyStatus(res.data)
} catch (e) {
// 拦截器已提示
// 网络异常:看门狗/轮询会继续重试,WS 模式忽略(等待 WS 重连)
}
},
// WebSocket 连接:页面刷新后自动重连,失败指数退避重试(上限 5 秒)。
// 同时维持一个低频 HTTP 轮询作为看门狗:WS 正常时保证兜底刷新,
// WS 不可用/断开时也能低频更新,界面永不冻结。
connectWs() {
if (this.ws || this.wsClosed) return
if (!window.WebSocket) {
this.usingPolling = true
this.wsAlive = false
this.startPoll(1000)
return
}
const proto = location.protocol === 'https:' ? 'wss://' : 'ws://'
const ws = new WebSocket(proto + location.host + '/api/scan/ws')
this.ws = ws
ws.onopen = () => {
this.wsAlive = true
this.usingPolling = false
this.wsReconnectDelay = 1000
this.fetchStatus()
this.startPoll(3000)
}
ws.onmessage = (e) => {
try {
this.applyStatus(JSON.parse(e.data))
} catch (err) {
// 忽略无法解析的消息
}
}
ws.onerror = () => {
ws.close()
}
ws.onclose = () => {
this.ws = null
this.wsAlive = false
if (this.wsClosed) return
this.wsReconnectTimer = setTimeout(() => {
this.wsReconnectDelay = Math.min(this.wsReconnectDelay * 2, 5000)
this.connectWs()
}, this.wsReconnectDelay)
}
},
closeWs() {
this.wsClosed = true
this.wsAlive = false
if (this.ws) {
this.ws.close()
this.ws = null
}
if (this.wsReconnectTimer) {
clearTimeout(this.wsReconnectTimer)
this.wsReconnectTimer = null
}
this.stopPoll()
},
// 周期轮询:WS 正常时每 3 秒兜底,无 WS 环境时每 1 秒
startPoll(interval) {
this.stopPoll()
this.pollTimer = setInterval(() => this.fetchStatus(), interval || 3000)
},
stopPoll() {
if (this.pollTimer) {
clearInterval(this.pollTimer)
this.pollTimer = null
}
},
// 秒 → "X 分 Y 秒" / "X 小时 Y 分"
fmtDuration(sec) {
if (!sec) return ''
const h = Math.floor(sec / 3600)
const m = Math.floor((sec % 3600) / 60)
const s = sec % 60
if (h > 0) return h + ' 小时 ' + m + ' 分'
if (m > 0) return m + ' 分 ' + s + ' 秒'
return s + ' 秒'
},
async handleStart() {
this.starting = true
try {
localStorage.setItem('scan-dir', this.scanDir)
const res = await request.post('/api/scan/start', { dir: this.scanDir.trim() })
this.status = res.data || this.status
this.startPoll()
localStorage.setItem('scan-dest-dir', this.destDir)
const res = await request.post('/api/scan/start', {
dir: this.scanDir.trim(),
destDir: this.destDir.trim()
})
this.applyStatus(res.data)
this.$message.success('已开始扫描整理')
} finally {
this.starting = false
}
},
openDirPicker() {
openDirPicker(target) {
this.pickerTarget = target || 'scan'
this.dirPickerVisible = true
this.loadFs(this.scanDir.trim())
const initial = target === 'dest' ? this.destDir.trim() : this.scanDir.trim()
this.loadFs(initial)
},
async loadFs(dir) {
this.dirLoading = true
@@ -204,19 +370,14 @@ export default {
},
confirmDir() {
if (!this.fsData.current) return
if (this.pickerTarget === 'dest') {
this.destDir = this.fsData.current
localStorage.setItem('scan-dest-dir', this.destDir)
} else {
this.scanDir = this.fsData.current
localStorage.setItem('scan-dir', this.scanDir)
this.dirPickerVisible = false
},
startPoll() {
this.stopPoll()
this.pollTimer = setInterval(() => this.loadStatus(), 1000)
},
stopPoll() {
if (this.pollTimer) {
clearInterval(this.pollTimer)
this.pollTimer = null
}
this.dirPickerVisible = false
},
scrollBottom() {
this.$nextTick(() => {
@@ -229,7 +390,25 @@ export default {
</script>
<style scoped>
.conn-hint {
display: inline-flex;
align-items: center;
gap: 4px;
font-size: 12px;
color: #67c23a;
}
.conn-hint.off {
color: #e6a23c;
}
.conn-hint i {
font-size: 13px;
}
.scan-tip {
display: flex;
gap: 10px;
background: #f8fbff;
border: 1px solid #e6efff;
border-radius: 8px;
@@ -242,7 +421,25 @@ export default {
.scan-tip .el-icon-warning-outline {
color: #faad14;
margin-right: 4px;
margin-top: 3px;
flex-shrink: 0;
}
.scan-tip-body {
flex: 1;
}
.scan-tip-body p {
margin: 0 0 6px;
}
.scan-tip-body ul {
margin: 0;
padding-left: 18px;
}
.scan-tip-body li {
margin-bottom: 2px;
}
.scan-form {
@@ -321,13 +518,28 @@ export default {
}
.running-hint {
display: inline-flex;
align-items: center;
gap: 8px;
display: flex;
flex-direction: column;
align-items: flex-start;
gap: 10px;
max-width: 640px;
}
.running-hint > span {
color: #1677ff;
font-size: 14px;
}
.scan-progress {
width: 100%;
}
.eta-text {
color: #909399;
font-size: 12px;
margin-left: 6px;
}
.state-alert {
margin-bottom: 16px;
}
+2 -1
View File
@@ -8,7 +8,8 @@ export default defineConfig({
proxy: {
'/api': {
target: 'http://localhost:8080',
changeOrigin: true
changeOrigin: true,
ws: true
}
}
},
+1 -1
View File
@@ -7,8 +7,8 @@ require (
github.com/gabriel-vasile/mimetype v1.4.12
github.com/gin-gonic/gin v1.12.0
github.com/glebarez/go-sqlite v1.20.3
github.com/gorilla/websocket v1.5.3
github.com/magiconair/properties v1.8.10
github.com/redmask-hb/GoSimplePrint v0.0.0-20210302075413-3a3af92bcb7d
github.com/satori/go.uuid v1.2.0
golang.org/x/image v0.16.0
golang.org/x/sys v0.41.0
+2 -2
View File
@@ -74,6 +74,8 @@ github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S3
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -174,8 +176,6 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/redmask-hb/GoSimplePrint v0.0.0-20210302075413-3a3af92bcb7d h1:h/hohIqMUCML2Rp9BXXAu0I3ZR68d7eqMHLPNq7N2tg=
github.com/redmask-hb/GoSimplePrint v0.0.0-20210302075413-3a3af92bcb7d/go.mod h1:LiYo3EFlYfk46Re4zgQysMo6yO3/kAXslB2fyMdl+uw=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 h1:VstopitMQi3hZP0fzvnsLmzXZdQGc4bEcgu24cp+d4M=
github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+16 -3
View File
@@ -52,11 +52,24 @@ func (c *FsController) List(ctx *gin.Context) {
Error(ctx, http.StatusBadRequest, "路径解析失败: "+err.Error())
return
}
st, err := os.Stat(abs)
if err != nil || !st.IsDir() {
Error(ctx, http.StatusBadRequest, "目录不存在或不可访问: "+abs)
// 目录不存在/不可访问时向上回退到最近的已存在父目录,避免阻塞目录选择器
for {
st, statErr := os.Stat(abs)
if statErr == nil && st.IsDir() {
break
}
parent := filepath.Dir(abs)
if parent == abs {
// 已到根仍不存在:回退到盘符层
if runtime.GOOS == "windows" {
Success(ctx, FsListResult{Drives: listLogicalDrives()})
return
}
abs = string(os.PathSeparator)
break
}
abs = parent
}
entries, err := os.ReadDir(abs)
if err != nil {
+1
View File
@@ -28,6 +28,7 @@ func NewRouter(st *store.Store) *gin.Engine {
sc := NewScanController(service.NewScanService(st))
apiGroup.POST("/scan/start", sc.Start)
apiGroup.GET("/scan/status", sc.Status)
apiGroup.GET("/scan/ws", sc.Ws)
fc := NewFsController()
apiGroup.GET("/fs/list", fc.List)
+3 -2
View File
@@ -18,9 +18,10 @@ func NewScanController(svc *service.ScanService) *ScanController {
return &ScanController{service: svc}
}
// scanStartReq 扫描启动请求;dir 为空时使用服务当前运行目录
// scanStartReq 扫描启动请求;dir 为空时使用服务当前运行目录,destDir 为空时默认输出到扫描目录下的「KMP已整理」
type scanStartReq struct {
Dir string `json:"dir"`
DestDir string `json:"destDir"`
}
// Start POST /api/scan/start 触发一次扫描整理
@@ -30,7 +31,7 @@ func (c *ScanController) Start(ctx *gin.Context) {
Error(ctx, http.StatusBadRequest, "请求参数错误:"+err.Error())
return
}
if err := c.service.Start(req.Dir); err != nil {
if err := c.service.Start(req.Dir, req.DestDir); err != nil {
Error(ctx, http.StatusBadRequest, err.Error())
return
}
+93
View File
@@ -0,0 +1,93 @@
// 扫描状态 WebSocket 实时推送端点。
package api
import (
"encoding/json"
"net/http"
"time"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
)
// wsUpgrader WebSocket 升级器
var wsUpgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 4096,
// 同源管理页,允许任意来源,便于局域网访问
CheckOrigin: func(r *http.Request) bool { return true },
}
const (
wsWriteWait = 10 * time.Second
wsPongWait = 60 * time.Second
wsPingPeriod = (wsPongWait * 9) / 10
)
// Ws GET /api/scan/ws 建立 WebSocket 连接,持续推送扫描状态。
// 连接建立后立即推送当前状态快照;断开后前端可自动重连(服务端不做重连保证)。
func (c *ScanController) Ws(ctx *gin.Context) {
conn, err := wsUpgrader.Upgrade(ctx.Writer, ctx.Request, nil)
if err != nil {
return
}
ch, cancel := c.service.Subscribe()
defer func() {
cancel()
conn.Close()
}()
// 读协程:仅用于感知客户端断开/失活(配合 pong 心跳),不处理业务消息
conn.SetReadLimit(1024)
_ = conn.SetReadDeadline(time.Now().Add(wsPongWait))
conn.SetPongHandler(func(string) error {
return conn.SetReadDeadline(time.Now().Add(wsPongWait))
})
// 写协程:先推送快照,再转发订阅更新,并周期性 ping 保活
writeDone := make(chan struct{})
go func() {
defer close(writeDone)
write := func(msgType int, data []byte) bool {
if err := conn.WriteMessage(msgType, data); err != nil {
conn.Close() // 写失败即断开,让读协程退出
return false
}
return true
}
if b, err := json.Marshal(c.service.Status()); err == nil {
if !write(websocket.TextMessage, b) {
return
}
}
ticker := time.NewTicker(wsPingPeriod)
defer ticker.Stop()
for {
select {
case st, ok := <-ch:
if !ok {
return
}
b, err := json.Marshal(st)
if err != nil {
continue
}
if !write(websocket.TextMessage, b) {
return
}
case <-ticker.C:
if !write(websocket.PingMessage, nil) {
return
}
}
}
}()
for {
if _, _, err := conn.ReadMessage(); err != nil {
break
}
}
cancel() // 先关闭订阅频道,让写协程退出,避免 <-writeDone 永久阻塞
<-writeDone
}
+56 -36
View File
@@ -6,27 +6,34 @@ import (
"io"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
"github.com/redmask-hb/GoSimplePrint/goPrint"
uuid "github.com/satori/go.uuid"
)
// Mover 文件移动器
type Mover struct{}
// moveResult 移动结果状态
type moveResult int
// workerCount 并行移动的 worker 数量:不超过 CPU 核数,限制在 1~4 之间。
// 移动/复制是磁盘 I/O 密集型操作,过高的并行度反而会降低吞吐(尤其机械硬盘)。
func workerCount() int {
n := runtime.NumCPU()
if n < 1 {
return 1
}
if n > 4 {
return 4
}
return n
}
const (
moveOK moveResult = iota // 移动成功(rename 或 copy+delete 完整完成)
movePartial // 复制成功但删除源失败,源与目标同时存在
moveFailed // 移动/复制失败
)
// MoveToDir 将一批文件移动到目标目录(目标文件名使用 uuid 前缀防止同名覆盖)。
// 返回移动成功的 源路径 → 目标目录 映射,供记录去向使用。
func (m *Mover) MoveToDir(filePaths []string, targetDir string) map[string]string {
// MoveToDir 将一批文件并行移动到目标目录(目标文件名使用 uuid 前缀防止同名覆盖)。
// 返回移动成功的 源路径 → 完整目标文件路径 映射,供记录去向使用。
// onProgress 在每移动一个文件后调用(done 为累计成功移动数,并发安全、单调递增)。
func (m *Mover) MoveToDir(filePaths []string, targetDir string, onProgress func(done int)) map[string]string {
moved := make(map[string]string)
if len(filePaths) == 0 {
return moved
@@ -36,37 +43,58 @@ func (m *Mover) MoveToDir(filePaths []string, targetDir string) map[string]strin
return moved
}
bar := goPrint.NewBar(100)
bar.SetNotice("=== 移动文件到目录:")
bar.SetGraph(">")
total := len(filePaths)
for i, src := range filePaths {
switch moveOne(src, targetDir) {
case moveOK, movePartial:
moved[src] = targetDir
results := make([]string, len(filePaths))
workers := workerCount()
if workers > len(filePaths) {
workers = len(filePaths)
}
var wg sync.WaitGroup
var doneCount int32
jobs := make(chan int)
for w := 0; w < workers; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := range jobs {
if target, ok := moveOne(filePaths[i], targetDir); ok {
results[i] = target
}
if onProgress != nil {
onProgress(int(atomic.AddInt32(&doneCount, 1)))
}
}
}()
}
for i := range filePaths {
jobs <- i
}
close(jobs)
wg.Wait()
for i, target := range results {
if target != "" {
moved[filePaths[i]] = target
}
bar.PrintBar(percent(i+1, total))
}
bar.PrintEnd("=== Finish")
return moved
}
// moveOne 移动单个文件;跨文件系统时 fallback 为 copy+delete
func moveOne(src, targetDir string) moveResult {
// moveOne 移动单个文件;跨文件系统时 fallback 为 copy+delete。
// 成功时返回完整目标文件路径;部分成功(复制成功删除源失败)同样返回目标路径。
func moveOne(src, targetDir string) (string, bool) {
fileName := strings.ReplaceAll(uuid.NewV4().String(), "-", "") + "_" + filepath.Base(src)
target := filepath.Join(targetDir, fileName)
if err := os.Rename(src, target); err == nil {
return moveOK
return target, true
}
if err := copyFile(src, target); err != nil {
fmt.Printf("=== 复制文件失败, 源: %s, 目标: %s, 错误: %v\n", src, target, err)
return moveFailed
return "", false
}
if err := os.Remove(src); err != nil {
fmt.Printf("=== 警告: 复制成功但删除源文件失败, 源与目标同时存在, 源: %s, 错误: %v\n", src, err)
return movePartial
}
return moveOK
return target, true
}
// copyFile 逐块复制文件;失败时清理已写入的目标文件,避免残留半截文件
@@ -92,11 +120,3 @@ func copyFile(src, dst string) error {
}
return nil
}
// percent 计算百分比(0~100)
func percent(part, total int) int {
if total <= 0 {
return 100
}
return int(float64(part) / float64(total) * 100)
}
+251 -55
View File
@@ -5,7 +5,10 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
"kenaito-media-picker/internal/classify"
"kenaito-media-picker/internal/digest"
@@ -34,6 +37,10 @@ type Organizer struct {
mover *mover.Mover
buckets map[string][]string // 目标目录 → 文件路径
order []string // 目标目录的创建顺序(保证输出稳定)
seenMd5 map[string]bool // 本次扫描已入库的 MD5(同一扫描内去重)
onProgress func(done, total int)
pending []*store.MediaInfo // 待批量入库的媒体记录
pendingDup []*store.MediaInfo // 待批量入库的重复文件记录
}
// New 创建整理器;store 可为 nil(数据库初始化失败时仅不记录信息)
@@ -44,9 +51,69 @@ func New(pickerDir, ffprobePath string, st *store.Store) *Organizer {
store: st,
mover: &mover.Mover{},
buckets: make(map[string][]string),
seenMd5: make(map[string]bool),
}
}
// SetProgress 设置进度回调(done/total 为已处理文件数与总数)
func (o *Organizer) SetProgress(fn func(done, total int)) {
o.onProgress = fn
}
// report 上报进度
func (o *Organizer) report(done, total int) {
if o.onProgress != nil && total > 0 {
o.onProgress(done, total)
}
}
// workerCount 并行探测的 worker 数量:不超过 CPU 核数,且限制在 1~8 之间
func workerCount() int {
n := runtime.NumCPU()
if n < 1 {
return 1
}
if n > 8 {
return 8
}
return n
}
// runParallel 用固定 worker 数并行处理切片,按原顺序收集结果。
// onProgress 在每次完成一个任务后调用(done 为已完成任务数,并发安全)。
// 返回 results[i] 对应 items[i] 的处理结果。
func runParallel[T any, R any](items []T, fn func(T) R, onProgress func(done int)) []R {
results := make([]R, len(items))
if len(items) == 0 {
return results
}
workers := workerCount()
if workers > len(items) {
workers = len(items)
}
var wg sync.WaitGroup
var doneCount int32
jobs := make(chan int)
for w := 0; w < workers; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := range jobs {
results[i] = fn(items[i])
if onProgress != nil {
onProgress(int(atomic.AddInt32(&doneCount, 1)))
}
}
}()
}
for i := range items {
jobs <- i
}
close(jobs)
wg.Wait()
return results
}
// add 将文件归入某个子目录
func (o *Organizer) add(f scanner.File, subDir string) {
dir := filepath.Join(o.pickerDir, subDir)
@@ -86,25 +153,66 @@ func (o *Organizer) calcFileInfo(f scanner.File) (md5Str, headMd5, tailMd5 strin
return md5Str, headMd5, tailMd5, size
}
// isDuplicate 判断文件是否与已入库记录重复(按 MD5)
// isDuplicate 判断文件是否重复:
// 1. 本次扫描已入库的 MD5 → 重复(同一扫描内去重)
// 2. 库中已有记录且其副本在磁盘上仍存在(DestPath)→ 重复
// 若库中副本已被移动/改名/删除,说明是同一文件被重新扫描,不算重复。
func (o *Organizer) isDuplicate(md5Str string) bool {
if o.store == nil || md5Str == "" {
if md5Str == "" {
return false
}
_, ok := o.store.FindMediaByMd5(md5Str)
return ok
if o.seenMd5[md5Str] {
return true
}
if o.store == nil {
return false
}
existing, ok := o.store.FindMediaByMd5(md5Str)
if !ok {
return false
}
if existing.DestPath != "" {
if _, err := os.Stat(existing.DestPath); err == nil {
return true
}
}
return false
}
// record 将媒体信息写入数据库
// markSeen 记录本次扫描已入库的 MD5(去重用)
func (o *Organizer) markSeen(md5Str string) {
if md5Str != "" {
o.seenMd5[md5Str] = true
}
}
// record 收集媒体信息到待入库缓冲(不立即写库,ProcessImages/ProcessMedia 结束统一批量提交)
func (o *Organizer) record(f scanner.File, mediaType string, res *probe.ProbeResult, md5Str, headMd5, tailMd5 string, size int64) {
if o.store == nil || md5Str == "" {
return
}
o.pending = append(o.pending, buildMediaInfo(f, mediaType, res, md5Str, headMd5, tailMd5, size))
}
// recordDuplicate 收集重复文件信息到待入库缓冲
func (o *Organizer) recordDuplicate(f scanner.File, mediaType string, res *probe.ProbeResult, md5Str, headMd5, tailMd5 string, size int64) {
if o.store == nil || md5Str == "" {
return
}
o.pendingDup = append(o.pendingDup, buildMediaInfo(f, mediaType, res, md5Str, headMd5, tailMd5, size))
}
// buildMediaInfo 构造入库记录
func buildMediaInfo(f scanner.File, mediaType string, res *probe.ProbeResult, md5Str, headMd5, tailMd5 string, size int64) *store.MediaInfo {
codec := res.VideoCodec
if mediaType == "audio" {
codec = res.AudioCodec
}
info := &store.MediaInfo{
format := res.Format
if format == "" && f.Ext != "" {
format = strings.TrimPrefix(f.Ext, ".")
}
return &store.MediaInfo{
FileMd5: md5Str,
FilePath: f.Path,
FileName: f.Name,
@@ -116,69 +224,144 @@ func (o *Organizer) record(f scanner.File, mediaType string, res *probe.ProbeRes
Height: res.Height,
Duration: res.Duration,
Codec: codec,
Format: res.Format,
Format: format,
RawInfo: res.RawJSON,
HeadMd5: headMd5,
TailMd5: tailMd5,
}
if err := o.store.SaveMedia(info); err != nil {
fmt.Printf("=== 保存媒体信息失败: %s, 错误: %v\n", f.Path, err)
}
// flushRecords 将缓冲的媒体/重复记录批量提交入库
func (o *Organizer) flushRecords() {
if o.store == nil {
return
}
if len(o.pending) > 0 {
if err := o.store.SaveMediaBatch(o.pending); err != nil {
fmt.Printf("=== 批量保存媒体信息失败: %v\n", err)
}
o.pending = nil
}
if len(o.pendingDup) > 0 {
if err := o.store.InsertDuplicateBatch(o.pendingDup); err != nil {
fmt.Printf("=== 批量保存重复文件信息失败: %v\n", err)
}
o.pendingDup = nil
}
}
// ProcessImages 处理全部图片:去重 → 探测尺寸并归类
func (o *Organizer) ProcessImages(images []scanner.File) {
success, fail, ignore, dup := 0, 0, 0, 0
for _, f := range images {
switch {
case strings.EqualFold(f.Ext, ".psd"):
o.add(f, dirImagePSD)
success++
continue
case !probe.IsSupportImage(f.Ext) && o.ffprobePath == "":
// 非白名单扩展名且无 ffprobe 兜底时直接忽略
o.add(f, dirImageIgnored)
ignore++
continue
// imageProbeResult 图片探测结果(并行阶段产出,串行阶段归类)
type imageProbeResult struct {
psd bool // PSD 特殊归类
unsupported bool // 非白名单且无 ffprobe 兜底,忽略
probeErr bool // 探测失败,归入异常
res *probe.ProbeResult
md5Str string
headMd5 string
tailMd5 string
size int64
darkScore float64
lightScore float64
}
// probeImage 并行探测一张图片:探测尺寸 + MD5 + 明暗分析(不含重复判断/归类/入库)。
// 注意:不访问 o.seenMd5 等共享可变状态,仅读 store(指纹复用),并行安全。
func (o *Organizer) probeImage(f scanner.File) imageProbeResult {
r := imageProbeResult{}
if strings.EqualFold(f.Ext, ".psd") {
r.psd = true
return r
}
if !probe.IsSupportImage(f.Ext) && o.ffprobePath == "" {
r.unsupported = true
return r
}
res, err := probe.ProbeImage(f.Path, o.ffprobePath)
if err != nil {
o.add(f, dirImageErr)
fail++
fmt.Printf("=== 异常图片: %s \n", f.Path)
continue
r.probeErr = true
return r
}
md5Str, headMd5, tailMd5, size := o.calcFileInfo(f)
if o.isDuplicate(md5Str) {
o.add(f, dirDuplicate)
dup++
fmt.Printf("=== 重复图片: %s \n", f.Path)
continue
}
o.record(f, "image", res, md5Str, headMd5, tailMd5, size)
darkScore, lightScore := 0.0, 0.0
r.res = res
r.md5Str, r.headMd5, r.tailMd5, r.size = o.calcFileInfo(f)
if !strings.EqualFold(f.Ext, ".gif") {
dark, light, berr := probe.AnalyzeBrightness(f.Path)
if berr != nil {
fmt.Printf("=== 明暗分析失败(按中性mid归类): %s \n", f.Path)
} else {
darkScore, lightScore = dark, light
r.darkScore, r.lightScore = dark, light
}
}
o.add(f, classify.ImageDir(f.Ext, res.Width, res.Height, darkScore, lightScore))
return r
}
// ProcessImages 处理全部图片:并行探测(尺寸/MD5/明暗)→ 串行去重归类
func (o *Organizer) ProcessImages(images []scanner.File) {
success, fail, ignore, dup := 0, 0, 0, 0
total := len(images)
results := runParallel(images, o.probeImage, func(done int) {
o.report(done, total)
})
for i, r := range results {
f := images[i]
switch {
case r.psd:
o.add(f, dirImagePSD)
success++
case r.unsupported:
o.add(f, dirImageIgnored)
ignore++
case r.probeErr:
o.add(f, dirImageErr)
fail++
fmt.Printf("=== 异常图片: %s \n", f.Path)
case o.isDuplicate(r.md5Str):
o.recordDuplicate(f, "image", r.res, r.md5Str, r.headMd5, r.tailMd5, r.size)
o.add(f, dirDuplicate)
dup++
fmt.Printf("=== 重复图片: %s \n", f.Path)
default:
o.record(f, "image", r.res, r.md5Str, r.headMd5, r.tailMd5, r.size)
o.markSeen(r.md5Str)
o.add(f, classify.ImageDir(f.Ext, r.res.Width, r.res.Height, r.darkScore, r.lightScore))
success++
}
}
fmt.Printf("=== 图片处理完毕,成功: %d, 读取异常: %d, 忽略: %d, 重复: %d \n\n", success, fail, ignore, dup)
}
// ProcessMedia 处理媒体候选:去重 → ffprobe 判定视频/音频/不支持
func (o *Organizer) ProcessMedia(media []scanner.File) {
video, audio, fail, ignore, dup := 0, 0, 0, 0, 0
for _, f := range media {
// mediaProbeResult 媒体探测结果(并行阶段产出,串行阶段归类)
type mediaProbeResult struct {
res *probe.ProbeResult
err error
md5 string
head string
tail string
size int64
}
// probeMedia 并行探测一个媒体文件(ffprobe + MD5)。不访问共享可变状态。
func (o *Organizer) probeMedia(f scanner.File) mediaProbeResult {
res, err := probe.ProbeMediaFile(o.ffprobePath, f.Path)
if err != nil {
return mediaProbeResult{err: err}
}
if !res.HasVideo && !res.HasAudio {
return mediaProbeResult{res: res}
}
md5, head, tail, size := o.calcFileInfo(f)
return mediaProbeResult{res: res, md5: md5, head: head, tail: tail, size: size}
}
// ProcessMedia 处理媒体候选:并行 ffprobe 判定 → 串行去重归类
func (o *Organizer) ProcessMedia(media []scanner.File) {
video, audio, fail, ignore, dup := 0, 0, 0, 0, 0
total := len(media)
results := runParallel(media, o.probeMedia, func(done int) {
o.report(done, total)
})
for i, r := range results {
f := media[i]
if r.err != nil {
// 仅 MIME 明确为视频/音频的文件才移入异常/忽略目录;
// octet-stream/空 MIME 且 ffprobe 无法确认的原地忽略,不移动
if strings.HasPrefix(f.Mime, "video/") {
@@ -195,27 +378,29 @@ func (o *Organizer) ProcessMedia(media []scanner.File) {
continue
}
switch {
case res.HasVideo:
md5Str, headMd5, tailMd5, size := o.calcFileInfo(f)
if o.isDuplicate(md5Str) {
case r.res.HasVideo:
if o.isDuplicate(r.md5) {
o.recordDuplicate(f, "video", r.res, r.md5, r.head, r.tail, r.size)
dup++
o.add(f, dirDuplicate)
fmt.Printf("=== 重复媒体: %s \n", f.Path)
continue
}
video++
o.record(f, "video", res, md5Str, headMd5, tailMd5, size)
o.add(f, classify.VideoDir(f.Ext, res.Width, res.Height))
case res.HasAudio:
md5Str, headMd5, tailMd5, size := o.calcFileInfo(f)
if o.isDuplicate(md5Str) {
o.record(f, "video", r.res, r.md5, r.head, r.tail, r.size)
o.markSeen(r.md5)
o.add(f, classify.VideoDir(f.Ext, r.res.Width, r.res.Height))
case r.res.HasAudio:
if o.isDuplicate(r.md5) {
o.recordDuplicate(f, "audio", r.res, r.md5, r.head, r.tail, r.size)
dup++
o.add(f, dirDuplicate)
fmt.Printf("=== 重复媒体: %s \n", f.Path)
continue
}
audio++
o.record(f, "audio", res, md5Str, headMd5, tailMd5, size)
o.record(f, "audio", r.res, r.md5, r.head, r.tail, r.size)
o.markSeen(r.md5)
o.add(f, dirAudio)
default:
// ffprobe 无音视频流:MIME 明确为媒体的归入忽略目录,其余原地不动
@@ -231,12 +416,23 @@ func (o *Organizer) ProcessMedia(media []scanner.File) {
fmt.Printf("=== 媒体处理完毕,视频: %d, 音频: %d, 读取异常: %d, 忽略: %d, 重复: %d \n\n", video, audio, fail, ignore, dup)
}
// Flush 将所有收集到的文件移动到对应目录,并回填归类去向
// Flush 先将缓冲的媒体记录批量入库,再将收集到的文件移动到对应目录并回填归类去向。
// 移动期间按累计完成数上报进度。
func (o *Organizer) Flush() {
o.flushRecords()
total := 0
for _, dir := range o.order {
moved := o.mover.MoveToDir(o.buckets[dir], dir)
total += len(o.buckets[dir])
}
movedAll := 0
for _, dir := range o.order {
onProgress := func(done int) {
o.report(movedAll+done, total)
}
moved := o.mover.MoveToDir(o.buckets[dir], dir, onProgress)
movedAll += len(o.buckets[dir])
for src, dest := range moved {
if err := o.store.UpdateMediaDest(src, dest); err != nil {
if err := o.store.UpdateMediaDest(src, dir, dest); err != nil {
fmt.Printf("=== 更新媒体去向失败: %s, 错误: %v\n", src, err)
}
}
+70 -12
View File
@@ -18,9 +18,9 @@ const (
// brightnessDecodeLimit 明暗分析允许解码的最大像素数,超出时跳过分析(按 mid 归类),避免超大图 OOM
brightnessDecodeLimit = 40_000_000
// thumbnailMaxEdge 明暗分析时大图缩放后的最长边像素,缩小到该尺寸内再统计
thumbnailMaxEdge = 512
thumbnailMaxEdge = 256
// thumbnailScaleThreshold 像素数超过该值才缩放为缩略图再统计
thumbnailScaleThreshold = 1_000_000
thumbnailScaleThreshold = 250_000
)
// AnalyzeBrightness 打开并解码图片,分析明暗程度。
@@ -78,6 +78,7 @@ func scaleToThumbnail(img image.Image, maxEdge int) image.Image {
// analyzeBrightness 统计暗色/亮色像素占比(ITU-R BT.601 亮度公式)。
// 完全透明的像素(alpha=0)不计入统计。
// 对常见内存布局(NRGBA/RGBA/Gray)直接访问像素数组,避免 image.At 接口逐像素开销。
func analyzeBrightness(img image.Image) (darkScore float64, lightScore float64) {
bounds := img.Bounds()
totalPixels := bounds.Dx() * bounds.Dy()
@@ -85,28 +86,85 @@ func analyzeBrightness(img image.Image) (darkScore float64, lightScore float64)
return 0, 0
}
darkCount, lightCount, sampleCount := 0, 0, 0
var darkCount, lightCount, sampleCount int64
switch m := img.(type) {
case *image.NRGBA:
pix := m.Pix
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
offset := m.PixOffset(bounds.Min.X, y)
for x := bounds.Min.X; x < bounds.Max.X; x++ {
r, g, b, a := pix[offset], pix[offset+1], pix[offset+2], pix[offset+3]
if a == 0 {
offset += 4
continue
}
countBrightness(float64(r), float64(g), float64(b), &darkCount, &lightCount)
sampleCount++
offset += 4
}
}
case *image.RGBA:
pix := m.Pix
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
offset := m.PixOffset(bounds.Min.X, y)
for x := bounds.Min.X; x < bounds.Max.X; x++ {
// 预乘 alpha:若完全不透明则按直接值,否则近似取非预乘分量
a := pix[offset+3]
if a == 0 {
offset += 4
continue
}
r, g, b := pix[offset], pix[offset+1], pix[offset+2]
if a != 255 {
r = uint8(int(r) * 255 / int(a))
g = uint8(int(g) * 255 / int(a))
b = uint8(int(b) * 255 / int(a))
}
countBrightness(float64(r), float64(g), float64(b), &darkCount, &lightCount)
sampleCount++
offset += 4
}
}
case *image.Gray:
pix := m.Pix
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
offset := m.PixOffset(bounds.Min.X, y)
for x := bounds.Min.X; x < bounds.Max.X; x++ {
v := float64(pix[offset])
if v < 80 {
darkCount++
} else if v > 175 {
lightCount++
}
sampleCount++
offset++
}
}
default:
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
for x := bounds.Min.X; x < bounds.Max.X; x++ {
r, g, b, a := img.At(x, y).RGBA()
if a == 0 {
continue
}
r8 := uint8(r >> 8)
g8 := uint8(g >> 8)
b8 := uint8(b >> 8)
brightness := 0.299*float64(r8) + 0.587*float64(g8) + 0.114*float64(b8)
if brightness < 80 {
darkCount++
} else if brightness > 175 {
lightCount++
}
countBrightness(float64(r>>8), float64(g>>8), float64(b>>8), &darkCount, &lightCount)
sampleCount++
}
}
}
if sampleCount == 0 {
return 0, 0
}
return float64(darkCount) / float64(sampleCount), float64(lightCount) / float64(sampleCount)
}
// countBrightness 按 BT.601 亮度公式累计暗/亮像素计数
func countBrightness(r, g, b float64, darkCount, lightCount *int64) {
brightness := 0.299*r + 0.587*g + 0.114*b
if brightness < 80 {
*darkCount++
} else if brightness > 175 {
*lightCount++
}
}
+24 -15
View File
@@ -6,11 +6,11 @@ import (
"os"
"path/filepath"
"strings"
"time"
"kenaito-media-picker/internal/probe"
"github.com/gabriel-vasile/mimetype"
"github.com/redmask-hb/GoSimplePrint/goPrint"
)
// File 扫描到的文件信息
@@ -32,12 +32,18 @@ type Result struct {
type Options struct {
RootDir string // 扫描根目录
SkipDir string // 白名单输出目录,整目录跳过
SkipDirNames []string // 按目录名跳过(如 KMP已整理),任意位置命中即跳过
SkipNames []string // 按文件名跳过(数据库、ffprobe 等)
OnProgress func(done, total int)
}
// Scan 递归扫描目录并读取每个文件的 MIME 信息
func Scan(opts Options) (*Result, error) {
res := &Result{}
// 遍历阶段尚不知道总文件数,先按「已发现文件数」实时上报(total=0),
// 并节流上报频率,避免目录很大时界面长时间无动静或推送过密。
walkCount := 0
lastReport := time.Now()
err := filepath.Walk(opts.RootDir, func(currentPath string, info os.FileInfo, err error) error {
if err != nil {
// 单个路径访问失败(权限不足/损坏)不中止整体扫描,仅告警后跳过该子树
@@ -48,6 +54,11 @@ func Scan(opts Options) (*Result, error) {
if opts.SkipDir != "" && currentPath == opts.SkipDir {
return filepath.SkipDir
}
for _, n := range opts.SkipDirNames {
if info.Name() == n {
return filepath.SkipDir
}
}
return nil
}
for _, n := range opts.SkipNames {
@@ -64,6 +75,11 @@ func Scan(opts Options) (*Result, error) {
Name: info.Name(),
Ext: filepath.Ext(currentPath),
})
walkCount++
if opts.OnProgress != nil && time.Since(lastReport) >= 100*time.Millisecond {
lastReport = time.Now()
opts.OnProgress(walkCount, 0)
}
return nil
})
if err != nil {
@@ -72,15 +88,16 @@ func Scan(opts Options) (*Result, error) {
total := len(res.Files)
fmt.Printf("=== 文件总数: %d \n", total)
bar := goPrint.NewBar(100)
bar.SetNotice("=== 扫描文件Mime:")
bar.SetGraph(">")
bar.SetNoticeColor(goPrint.FontColor.Red)
// 遍历结束、开始 MIME 探测前,上报一次完整总数
if opts.OnProgress != nil {
opts.OnProgress(total, total)
}
for i := range res.Files {
res.Files[i].Mime = detectMime(res.Files[i].Path)
bar.PrintBar(percent(i+1, total))
if opts.OnProgress != nil {
opts.OnProgress(i+1, total)
}
}
bar.PrintEnd("=== Finish")
return res, nil
}
@@ -138,11 +155,3 @@ func isPotentialMedia(fileMime string) bool {
}
return true
}
// percent 计算百分比(0~100),total 为 0 时返回 100
func percent(part, total int) int {
if total <= 0 {
return 100
}
return int(float64(part) / float64(total) * 100)
}
+52
View File
@@ -0,0 +1,52 @@
// 状态订阅发布:将 ScanService 的状态变更实时推送给订阅者(WebSocket 等)。
package service
import "sync"
// StatusListener 状态订阅通道(缓冲 1,保底容纳一条未消费状态)
type StatusListener chan ScanStatus
// StatusHub 管理状态订阅者;发布时若消费者来不及消费则丢弃旧消息,
// 保证订阅者拿到的始终是最新状态。
type StatusHub struct {
mu sync.RWMutex
subs map[StatusListener]struct{}
}
// NewStatusHub 创建状态订阅中心
func NewStatusHub() *StatusHub {
return &StatusHub{subs: make(map[StatusListener]struct{})}
}
// Subscribe 注册订阅,返回频道与取消函数。取消后频道被关闭,写方需停止发送。
func (h *StatusHub) Subscribe() (StatusListener, func()) {
h.mu.Lock()
defer h.mu.Unlock()
ch := make(StatusListener, 1)
h.subs[ch] = struct{}{}
return ch, func() {
h.mu.Lock()
if _, ok := h.subs[ch]; ok {
delete(h.subs, ch)
close(ch)
}
h.mu.Unlock()
}
}
// Publish 向所有订阅者推送最新状态(非阻塞;缓冲满时覆盖为最新状态)
func (h *StatusHub) Publish(st ScanStatus) {
h.mu.RLock()
defer h.mu.RUnlock()
for ch := range h.subs {
select {
case ch <- st:
default:
select {
case <-ch:
default:
}
ch <- st
}
}
}
+156 -9
View File
@@ -35,27 +35,73 @@ type ScanStatus struct {
Started string `json:"started"` // 开始时间
Elapsed int64 `json:"elapsed"` // 已运行秒数
RootDir string `json:"rootDir"` // 本次扫描的根目录
DestDir string `json:"destDir"` // 上次使用的存放目录
Lines []string `json:"lines"` // 运行日志
Stage string `json:"stage"` // 当前阶段:scan/organize/move
StageText string `json:"stageText"` // 阶段中文说明
Progress int `json:"progress"` // 总体进度百分比 0-100
Done int `json:"done"` // 已处理数
Total int `json:"total"` // 总数
Eta int64 `json:"eta"` // 预计剩余秒数(按当前阶段速率估算,0 表示无法估算)
}
// 应用级配置键
const (
ConfigKeyDestDir = "scan.destDir"
)
// DefaultPickerDirName 默认存放目录名(未指定 destDir 时创建于扫描目录下)
const DefaultPickerDirName = "KMP已整理"
// ScanService 管理媒体库扫描整理任务(同一时间只允许一次)
type ScanService struct {
store *store.Store
hub *StatusHub
mu sync.Mutex
state ScanState
msg string
started time.Time
rootDir string
destDir string
lines []string
stage string
done int
total int
stageStart time.Time
stageDone int
lastPublish time.Time
}
// publishInterval 运行中状态推送的节流间隔;非运行状态(done/error)不受限,保证结束状态必达
const publishInterval = 100 * time.Millisecond
// NewScanService 创建扫描服务;store 可为 nil(数据库不可用时仅不记录媒体信息)
func NewScanService(st *store.Store) *ScanService {
return &ScanService{store: st, state: ScanIdle}
return &ScanService{store: st, hub: NewStatusHub(), state: ScanIdle}
}
// Start 启动一次扫描整理;已在运行时返回错误。rootDir 为空时默认扫描当前运行目录。
func (s *ScanService) Start(rootDir string) error {
// Subscribe 订阅扫描状态变更(供 WebSocket 等实时推送);取消后返回的频道被关闭
func (s *ScanService) Subscribe() (StatusListener, func()) {
return s.hub.Subscribe()
}
// publish 将当前状态推送给所有订阅者。
// 运行中按 publishInterval 节流(约 10 次/秒),避免海量文件时 WebSocket 消息泛滥;
// 非运行状态(done/error)无条件推送,确保结束状态不丢失。
func (s *ScanService) publish() {
s.mu.Lock()
if s.state == ScanRunning && time.Since(s.lastPublish) < publishInterval {
s.mu.Unlock()
return
}
s.lastPublish = time.Now()
s.mu.Unlock()
s.hub.Publish(s.Status())
}
// Start 启动一次扫描整理;已在运行时返回错误。rootDir 为空时默认扫描当前运行目录,
// destDir 为空时默认输出到扫描目录下的「KMP已整理」文件夹。
func (s *ScanService) Start(rootDir, destDir string) error {
s.mu.Lock()
if s.state == ScanRunning {
s.mu.Unlock()
@@ -66,14 +112,28 @@ func (s *ScanService) Start(rootDir string) error {
s.mu.Unlock()
return fmt.Errorf("扫描目录不存在或不可访问: %s", rootDir)
}
if destDir != "" {
destDir = resolveRootDir(destDir)
}
s.state = ScanRunning
s.msg = ""
s.started = time.Now()
s.rootDir = rootDir
s.destDir = destDir
s.lines = nil
s.stage = "scan"
s.done = 0
s.total = 0
s.mu.Unlock()
go s.run(rootDir)
// 持久化存放目录,下次打开页面自动回填
if s.store != nil {
if err := s.store.SaveConfig(ConfigKeyDestDir, destDir); err != nil {
fmt.Printf("=== 保存存放目录配置失败: %v\n", err)
}
}
go s.run(rootDir, destDir)
return nil
}
@@ -85,14 +145,51 @@ func (s *ScanService) Status() ScanStatus {
if !s.started.IsZero() {
elapsed = int64(time.Since(s.started).Seconds())
}
return ScanStatus{
progress := 0
if s.total > 0 {
progress = int(float64(s.done) / float64(s.total) * 100)
if progress > 100 {
progress = 100
}
}
st := ScanStatus{
State: s.state,
Message: s.msg,
Started: s.started.Format("2006-01-02 15:04:05"),
Elapsed: elapsed,
RootDir: s.rootDir,
DestDir: s.destDir,
Lines: append([]string{}, s.lines...),
Stage: s.stage,
Progress: progress,
Done: s.done,
Total: s.total,
Eta: s.estimateEtaLocked(),
}
return st
}
// estimateEtaLocked 按当前阶段速率估算剩余秒数;需持有锁。返回 0 表示无法估算。
func (s *ScanService) estimateEtaLocked() int64 {
if s.state != ScanRunning || s.total <= 0 || s.done <= 0 || s.done >= s.total {
return 0
}
if s.stageStart.IsZero() || s.done <= s.stageDone {
return 0
}
elapsedSec := time.Since(s.stageStart).Seconds()
if elapsedSec <= 0 {
return 0
}
rate := float64(s.done-s.stageDone) / elapsedSec // 文件/秒
if rate <= 0 {
return 0
}
remain := float64(s.total-s.done) / rate
if remain <= 0 {
return 0
}
return int64(remain + 0.5)
}
// resolveRootDir 解析扫描根目录:空值回退当前工作目录,并做绝对路径归一化
@@ -111,8 +208,11 @@ func resolveRootDir(dir string) string {
return abs
}
// maxLogLines 日志保留的最大行数,超出后丢弃最旧行,避免前端渲染卡顿
const maxLogLines = 500
// run 后台执行:捕获 stdout 日志流式写入 lines,结束后更新状态
func (s *ScanService) run(rootDir string) {
func (s *ScanService) run(rootDir, destDir string) {
r, w, err := os.Pipe()
if err != nil {
s.finish(ScanError, "创建日志管道失败: "+err.Error())
@@ -132,12 +232,16 @@ func (s *ScanService) run(rootDir string) {
continue
}
s.mu.Lock()
if len(s.lines) >= maxLogLines {
// 丢弃最旧一半,保留最近日志,控制内存与前端渲染规模
s.lines = append([]string{}, s.lines[len(s.lines)/2:]...)
}
s.lines = append(s.lines, line)
s.mu.Unlock()
}
}()
scanErr := s.runScan(rootDir)
scanErr := s.runScan(rootDir, destDir)
os.Stdout = oldStdout
_ = w.Close()
@@ -156,6 +260,26 @@ func (s *ScanService) finish(state ScanState, msg string) {
s.state = state
s.msg = msg
s.mu.Unlock()
s.publish()
}
// setProgress 更新阶段与进度(线程安全,供各阶段回调调用)。
// 阶段切换或进度回退(新阶段从头计数)时重置阶段起点,用于估算剩余时间。
func (s *ScanService) setProgress(stage string, done, total int) {
s.mu.Lock()
if stage != s.stage {
s.stageStart = time.Now()
s.stageDone = done
} else if done < s.stageDone {
// 阶段未变但进度回退(如开始新一轮计数),重置起点
s.stageStart = time.Now()
s.stageDone = done
}
s.stage = stage
s.done = done
s.total = total
s.mu.Unlock()
s.publish()
}
// ansiRe 匹配 ANSI 颜色转义序列
@@ -169,15 +293,20 @@ func sanitizeLine(line string) string {
}
// runScan 执行完整整理流程:扫描 → 过滤 → 释放解码器 → 归类 → 移动
func (s *ScanService) runScan(rootDir string) error {
pickerDir := filepath.Join(rootDir, filepath.Base(rootDir)+"-picker")
func (s *ScanService) runScan(rootDir, destDir string) error {
pickerDir := destDir
if pickerDir == "" {
pickerDir = filepath.Join(rootDir, DefaultPickerDirName)
}
fmt.Printf("=== 扫描根目录: %s \n", rootDir)
fmt.Printf("=== 整理输出目录(白名单): %s \n", pickerDir)
res, err := scanner.Scan(scanner.Options{
RootDir: rootDir,
SkipDir: pickerDir,
SkipDirNames: []string{DefaultPickerDirName},
SkipNames: []string{store.DBFileName, probe.FFprobeExeName},
OnProgress: func(done, total int) { s.setProgress("scan", done, total) },
})
if err != nil {
return fmt.Errorf("扫描失败: %v", err)
@@ -196,13 +325,31 @@ func (s *ScanService) runScan(rootDir string) error {
}
org := organize.New(pickerDir, ffprobePath, s.store)
totalWork := len(res.Images) + len(res.Media)
doneWork := 0
currentStage := "organize"
org.SetProgress(func(done, total int) {
// organize 阶段:doneWork 累计图片/媒体已处理数,加上本阶段内 done;
// move 阶段:Flush 上报的 done 已是累计移动数,直接使用,避免叠加导致溢出。
if currentStage == "move" {
s.setProgress("move", done, total)
} else {
s.setProgress(currentStage, doneWork+done, totalWork)
}
})
if len(res.Images) > 0 {
org.ProcessImages(res.Images)
doneWork += len(res.Images)
}
if len(res.Media) > 0 && ffprobePath != "" {
org.ProcessMedia(res.Media)
doneWork += len(res.Media)
}
// 移动阶段:进度为累计已移动文件数
currentStage = "move"
org.Flush()
currentStage = "done"
s.setProgress("move", totalWork, totalWork)
fmt.Println("=== 整理完成")
return nil
}
+191 -13
View File
@@ -17,6 +17,9 @@ import (
// DBFileName 数据库文件名(存放于被整理目录的根目录,扫描时跳过)
const DBFileName = "media-info.db"
// CategoryDuplicate 重复文件的归类目录名(入库记录用于统计重复数量,媒体统计中排除)
const CategoryDuplicate = "重复文件"
// init 将纯Go的sqlite驱动注册为 xorm 认识的 sqlite3 驱动名(避免CGO依赖)
func init() {
sql.Register("sqlite3", &sqlite.Driver{})
@@ -42,6 +45,7 @@ type MediaInfo struct {
TailMd5 string `xorm:"char(32) index('idx_fp') 'tail_md5'" json:"tailMd5"` // 文件尾256KB的MD5(指纹加速)
Category string `xorm:"varchar(64) index 'category'" json:"category"` // 归类的目录名(去向)
DestDir string `xorm:"text 'dest_dir'" json:"destDir"` // 移动后的目录路径(去向)
DestPath string `xorm:"text 'dest_path'" json:"destPath"` // 移动后的完整文件路径(判断库中副本是否仍存在)
CreatedAt time.Time `xorm:"created 'created_at'" json:"createdAt"` // 首次入库时间
UpdatedAt time.Time `xorm:"updated 'updated_at'" json:"updatedAt"` // 最近更新时间
}
@@ -51,6 +55,12 @@ type Store struct {
engine *xorm.Engine
}
// AppConfig 应用级键值配置(如上次使用的存放目录),持久化到数据库便于复用
type AppConfig struct {
Key string `xorm:"pk varchar(64) 'key'"`
Value string `xorm:"text 'value'"`
}
// Open 打开(或创建)数据库并同步表结构
func Open(dbPath string) (*Store, error) {
e, err := xorm.NewEngine("sqlite3", dbPath)
@@ -59,7 +69,7 @@ func Open(dbPath string) (*Store, error) {
}
e.ShowSQL(false)
e.SetMaxOpenConns(1) // sqlite同一时刻只允许一个写连接
if err := e.Sync(new(MediaInfo)); err != nil {
if err := e.Sync(new(MediaInfo), new(AppConfig)); err != nil {
_ = e.Close()
return nil, err
}
@@ -76,6 +86,37 @@ func (s *Store) Close() error {
return err
}
// SaveConfig 保存/更新应用级配置项
func (s *Store) SaveConfig(key, value string) error {
if s == nil || s.engine == nil || key == "" {
return nil
}
existing := new(AppConfig)
has, err := s.engine.Where("key = ?", key).Get(existing)
if err != nil {
return err
}
if has {
_, err = s.engine.Where("key = ?", key).Update(&AppConfig{Value: value})
return err
}
_, err = s.engine.Insert(&AppConfig{Key: key, Value: value})
return err
}
// GetConfig 读取应用级配置项;不存在时返回空串
func (s *Store) GetConfig(key string) string {
if s == nil || s.engine == nil || key == "" {
return ""
}
cfg := new(AppConfig)
has, err := s.engine.Where("key = ?", key).Get(cfg)
if err != nil || !has {
return ""
}
return cfg.Value
}
// SaveMedia 按MD5去重保存/更新媒体信息
func (s *Store) SaveMedia(info *MediaInfo) error {
if s == nil || s.engine == nil {
@@ -95,6 +136,65 @@ func (s *Store) SaveMedia(info *MediaInfo) error {
return err
}
// InsertDuplicate 记录重复文件(与已入库内容 MD5 相同,独立入库便于统计重复数量)
func (s *Store) InsertDuplicate(info *MediaInfo) error {
if s == nil || s.engine == nil {
return nil
}
_, err := s.engine.Insert(info)
return err
}
// SaveMediaBatch 在一个事务内批量按MD5去重保存/更新,避免大量文件逐个提交事务的 fsync 开销
func (s *Store) SaveMediaBatch(infos []*MediaInfo) error {
if s == nil || s.engine == nil || len(infos) == 0 {
return nil
}
sess := s.engine.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
for _, info := range infos {
existing := new(MediaInfo)
has, err := sess.Where("file_md5 = ?", info.FileMd5).Get(existing)
if err != nil {
_ = sess.Rollback()
return err
}
if has {
info.Id = existing.Id
if _, err = sess.ID(existing.Id).Update(info); err != nil {
_ = sess.Rollback()
return err
}
} else if _, err = sess.Insert(info); err != nil {
_ = sess.Rollback()
return err
}
}
return sess.Commit()
}
// InsertDuplicateBatch 在一个事务内批量插入重复文件记录
func (s *Store) InsertDuplicateBatch(infos []*MediaInfo) error {
if s == nil || s.engine == nil || len(infos) == 0 {
return nil
}
sess := s.engine.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
for _, info := range infos {
if _, err := sess.Insert(info); err != nil {
_ = sess.Rollback()
return err
}
}
return sess.Commit()
}
// FindMediaByFingerprint 按 (文件大小 + 头256KB + 尾256KB) 查找已入库记录,避免对大文件重复计算全量MD5
func (s *Store) FindMediaByFingerprint(size int64, headMd5, tailMd5 string) (*MediaInfo, bool) {
if s == nil || s.engine == nil || headMd5 == "" || tailMd5 == "" {
@@ -121,14 +221,15 @@ func (s *Store) FindMediaByMd5(md5 string) (*MediaInfo, bool) {
return info, true
}
// UpdateMediaDest 按源路径回填文件的归类目录与去向
func (s *Store) UpdateMediaDest(filePath, destDir string) error {
// UpdateMediaDest 按源路径回填文件的归类目录与去向(destPath 为移动后的完整文件路径)
func (s *Store) UpdateMediaDest(filePath, destDir, destPath string) error {
if s == nil || s.engine == nil {
return nil
}
_, err := s.engine.Where("file_path = ?", filePath).Update(&MediaInfo{
Category: filepath.Base(destDir),
DestDir: destDir,
DestPath: destPath,
})
return err
}
@@ -181,9 +282,10 @@ func (s *Store) CountMedia(q MediaQuery) (int64, error) {
return sess.Count(new(MediaInfo))
}
// newQuerySession 创建带查询条件的会话
// newQuerySession 创建带查询条件的会话(默认排除重复文件分类记录)
func (s *Store) newQuerySession(q MediaQuery) *xorm.Session {
sess := s.engine.NewSession()
sess.Where("COALESCE(category,'') <> ?", CategoryDuplicate)
cond, args := buildMediaCond(q)
if cond != "" {
sess.Where(cond, args...)
@@ -271,6 +373,19 @@ type MediaStats struct {
Duplicate int64 `json:"duplicate"` // 重复文件数(同一MD5出现多次的去重后计数)
DupSize int64 `json:"dupSize"` // 重复文件占用的总字节
TopLarge []KV `json:"topLarge"` // 体积最大的 10 个文件
DurationTotal int64 `json:"durationTotal"` // 视频/音频总时长(秒)
DurationAvg int64 `json:"durationAvg"` // 视频/音频平均时长(秒)
AvgSize int64 `json:"avgSize"` // 平均文件体积(字节)
MaxSize int64 `json:"maxSize"` // 最大文件体积(字节)
MinSize int64 `json:"minSize"` // 最小文件体积(字节)
DurationBins []KV `json:"durationBins"` // 时长区间分布(视频/音频)
CreatedCounts []KV `json:"createdCounts"` // 入库趋势(按日,近 30 天)
CategoryCounts []KV `json:"categoryCounts"` // 归类目录分布
ExtCounts []KV `json:"extCounts"` // 扩展名分布
MimeCounts []KV `json:"mimeCounts"` // MIME 类型分布
AudioCodecCount []KV `json:"audioCodecCount"` // 音频编码分布
}
// KV 统计键值对
@@ -287,45 +402,54 @@ func (s *Store) Stats() (*MediaStats, error) {
}
st := &MediaStats{
ResCount: []KV{}, // 避免空库时返回 null,前端 .slice 直接可用
DurationBins: []KV{},
CreatedCounts: []KV{},
CategoryCounts: []KV{},
ExtCounts: []KV{},
MimeCounts: []KV{},
AudioCodecCount: []KV{},
}
if total, err := s.engine.Count(new(MediaInfo)); err == nil {
st.TotalFiles = total
var totalCnt struct {
C int64 `xorm:"c"`
}
if _, err := s.engine.SQL("SELECT COUNT(*) AS c FROM media_info WHERE COALESCE(category,'') <> '重复文件'").Get(&totalCnt); err == nil {
st.TotalFiles = totalCnt.C
}
var sizeTotal struct {
Size int64 `xorm:"size"`
}
if _, err := s.engine.SQL("SELECT COALESCE(SUM(file_size),0) AS size FROM media_info").Get(&sizeTotal); err == nil {
if _, err := s.engine.SQL("SELECT COALESCE(SUM(file_size),0) AS size FROM media_info WHERE COALESCE(category,'') <> '重复文件'").Get(&sizeTotal); err == nil {
st.TotalSize = sizeTotal.Size
}
// 类型统计(数量 + 体积)
typeRows := make([]KV, 0)
if err := s.engine.SQL("SELECT media_type AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info GROUP BY media_type ORDER BY count DESC").Find(&typeRows); err == nil {
if err := s.engine.SQL("SELECT media_type AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE COALESCE(category,'') <> '重复文件' GROUP BY media_type ORDER BY count DESC").Find(&typeRows); err == nil {
st.TypeCounts = typeRows
}
// 方向统计(仅含有效尺寸的图片/视频)
orientRows := make([]KV, 0)
if err := s.engine.SQL("SELECT CASE WHEN width>height THEN 'horizontal' WHEN width<height THEN 'vertical' ELSE 'square' END AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE width>0 AND height>0 GROUP BY 1 ORDER BY count DESC").Find(&orientRows); err == nil {
if err := s.engine.SQL("SELECT CASE WHEN width>height THEN 'horizontal' WHEN width<height THEN 'vertical' ELSE 'square' END AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE width>0 AND height>0 AND COALESCE(category,'') <> '重复文件' GROUP BY 1 ORDER BY count DESC").Find(&orientRows); err == nil {
st.OrientCount = orientRows
}
// 编码 / 格式 / 分辨率 / 大文件统计
codecRows := make([]KV, 0)
if err := s.engine.SQL("SELECT COALESCE(NULLIF(codec,''),'未知') AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE media_type='video' AND codec <> '' GROUP BY codec ORDER BY count DESC LIMIT 10").Find(&codecRows); err == nil {
if err := s.engine.SQL("SELECT COALESCE(NULLIF(codec,''),'未知') AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE media_type='video' AND codec <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY codec ORDER BY count DESC LIMIT 10").Find(&codecRows); err == nil {
st.CodecCount = codecRows
}
formatRows := make([]KV, 0)
if err := s.engine.SQL("SELECT COALESCE(NULLIF(format,''),'未知') AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE format <> '' GROUP BY format ORDER BY count DESC LIMIT 10").Find(&formatRows); err == nil {
if err := s.engine.SQL("SELECT COALESCE(NULLIF(format,''), LOWER(ltrim(file_ext,'.'))) AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE (format <> '' OR file_ext <> '') AND COALESCE(category,'') <> '重复文件' GROUP BY key ORDER BY count DESC LIMIT 10").Find(&formatRows); err == nil {
st.FormatCount = formatRows
}
largeRows := make([]KV, 0)
if err := s.engine.SQL("SELECT file_name AS key, 1 AS count, file_size AS size FROM media_info ORDER BY file_size DESC LIMIT 10").Find(&largeRows); err == nil {
if err := s.engine.SQL("SELECT file_name AS key, 1 AS count, file_size AS size FROM media_info WHERE COALESCE(category,'') <> '重复文件' ORDER BY file_size DESC LIMIT 10").Find(&largeRows); err == nil {
st.TopLarge = largeRows
}
// 分辨率分布(Go 侧计算,复用归类规则)
resMap := make(map[string]int64)
rows, err := s.engine.SQL("SELECT width, height FROM media_info WHERE width>0 AND height>0").Query()
rows, err := s.engine.SQL("SELECT width, height FROM media_info WHERE width>0 AND height>0 AND COALESCE(category,'') <> '重复文件'").Query()
if err != nil {
return st, err
}
@@ -346,6 +470,60 @@ func (s *Store) Stats() (*MediaStats, error) {
}
sortKVs(st.ResCount)
// 时长统计(视频/音频)
var dur struct {
Total float64 `xorm:"total"`
Avg float64 `xorm:"avg"`
}
if _, err := s.engine.SQL("SELECT COALESCE(SUM(duration),0) AS total, COALESCE(AVG(CASE WHEN duration>0 THEN duration END),0) AS avg FROM media_info WHERE duration>0 AND COALESCE(category,'') <> '重复文件'").Get(&dur); err == nil {
st.DurationTotal = int64(dur.Total)
st.DurationAvg = int64(dur.Avg)
}
// 时长区间分布
durRows := make([]KV, 0)
if err := s.engine.SQL("SELECT CASE WHEN duration<60 THEN '<1分钟' WHEN duration<300 THEN '1-5分钟' WHEN duration<1800 THEN '5-30分钟' WHEN duration<3600 THEN '30分钟-1小时' ELSE '>1小时' END AS key, COUNT(*) AS count, COALESCE(SUM(duration),0) AS size FROM media_info WHERE duration>0 AND COALESCE(category,'') <> '重复文件' GROUP BY key ORDER BY count DESC").Find(&durRows); err == nil {
st.DurationBins = durRows
}
// 平均/最大/最小文件体积
var sizeStat struct {
Avg int64 `xorm:"avg"`
Max int64 `xorm:"max"`
Min int64 `xorm:"min"`
}
if _, err := s.engine.SQL("SELECT COALESCE(AVG(file_size),0) AS avg, COALESCE(MAX(file_size),0) AS max, COALESCE(MIN(file_size),0) AS min FROM media_info WHERE COALESCE(category,'') <> '重复文件'").Get(&sizeStat); err == nil {
st.AvgSize = sizeStat.Avg
st.MaxSize = sizeStat.Max
st.MinSize = sizeStat.Min
}
// 入库趋势(近 30 天按日分组)
createdRows := make([]KV, 0)
if err := s.engine.SQL("SELECT substr(created_at,1,10) AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE created_at <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY key ORDER BY key DESC LIMIT 30").Find(&createdRows); err == nil {
for i, j := 0, len(createdRows)-1; i < j; i, j = i+1, j-1 {
createdRows[i], createdRows[j] = createdRows[j], createdRows[i]
}
st.CreatedCounts = createdRows
}
// 归类目录分布
catRows := make([]KV, 0)
if err := s.engine.SQL("SELECT category AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE category <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY category ORDER BY count DESC LIMIT 15").Find(&catRows); err == nil {
st.CategoryCounts = catRows
}
// 扩展名分布
extRows := make([]KV, 0)
if err := s.engine.SQL("SELECT LOWER(ltrim(file_ext,'.')) AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE file_ext <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY key ORDER BY count DESC LIMIT 10").Find(&extRows); err == nil {
st.ExtCounts = extRows
}
// MIME 类型分布
mimeRows := make([]KV, 0)
if err := s.engine.SQL("SELECT mime_type AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE mime_type <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY mime_type ORDER BY count DESC LIMIT 10").Find(&mimeRows); err == nil {
st.MimeCounts = mimeRows
}
// 音频编码分布
audioRows := make([]KV, 0)
if err := s.engine.SQL("SELECT COALESCE(NULLIF(codec,''),'未知') AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE media_type='audio' AND codec <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY codec ORDER BY count DESC LIMIT 10").Find(&audioRows); err == nil {
st.AudioCodecCount = audioRows
}
// 重复文件统计(同一 MD5 出现多次)
dupRows := make([]KV, 0)
if err := s.engine.SQL("SELECT file_md5 AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info GROUP BY file_md5 HAVING COUNT(*)>1").Find(&dupRows); err == nil {
BIN
View File
Binary file not shown.