feat: 扫描整理并行化、WebSocket 实时进度与统计看板
- 移动阶段并行处理文件，按原子计数上报累计进度 - 扫描遍历阶段实时上报已发现文件数，界面不再卡在 0/0 - 新增 /api/scan/ws WebSocket 实时推送扫描状态；前端断线指数退避自动重连，并以低频轮询兜底 - 新增统计看板与媒体库筛选/排序，目录选择器路径不存在时回退到最近可用父目录 - 存放目录支持自定义并持久化，默认输出到 KMP已整理
This commit is contained in:
+192
-14
@@ -17,6 +17,9 @@ import (
|
||||
// DBFileName 数据库文件名（存放于被整理目录的根目录，扫描时跳过）
|
||||
const DBFileName = "media-info.db"
|
||||
|
||||
// CategoryDuplicate 重复文件的归类目录名（入库记录用于统计重复数量，媒体统计中排除）
|
||||
const CategoryDuplicate = "重复文件"
|
||||
|
||||
// init 将纯Go的sqlite驱动注册为 xorm 认识的 sqlite3 驱动名（避免CGO依赖）
|
||||
func init() {
|
||||
sql.Register("sqlite3", &sqlite.Driver{})
|
||||
@@ -42,6 +45,7 @@ type MediaInfo struct {
|
||||
TailMd5 string `xorm:"char(32) index('idx_fp') 'tail_md5'" json:"tailMd5"` // 文件尾256KB的MD5（指纹加速）
|
||||
Category string `xorm:"varchar(64) index 'category'" json:"category"` // 归类的目录名（去向）
|
||||
DestDir string `xorm:"text 'dest_dir'" json:"destDir"` // 移动后的目录路径（去向）
|
||||
DestPath string `xorm:"text 'dest_path'" json:"destPath"` // 移动后的完整文件路径（判断库中副本是否仍存在）
|
||||
CreatedAt time.Time `xorm:"created 'created_at'" json:"createdAt"` // 首次入库时间
|
||||
UpdatedAt time.Time `xorm:"updated 'updated_at'" json:"updatedAt"` // 最近更新时间
|
||||
}
|
||||
@@ -51,6 +55,12 @@ type Store struct {
|
||||
engine *xorm.Engine
|
||||
}
|
||||
|
||||
// AppConfig 应用级键值配置（如上次使用的存放目录），持久化到数据库便于复用
|
||||
type AppConfig struct {
|
||||
Key string `xorm:"pk varchar(64) 'key'"`
|
||||
Value string `xorm:"text 'value'"`
|
||||
}
|
||||
|
||||
// Open 打开（或创建）数据库并同步表结构
|
||||
func Open(dbPath string) (*Store, error) {
|
||||
e, err := xorm.NewEngine("sqlite3", dbPath)
|
||||
@@ -59,7 +69,7 @@ func Open(dbPath string) (*Store, error) {
|
||||
}
|
||||
e.ShowSQL(false)
|
||||
e.SetMaxOpenConns(1) // sqlite同一时刻只允许一个写连接
|
||||
if err := e.Sync(new(MediaInfo)); err != nil {
|
||||
if err := e.Sync(new(MediaInfo), new(AppConfig)); err != nil {
|
||||
_ = e.Close()
|
||||
return nil, err
|
||||
}
|
||||
@@ -76,6 +86,37 @@ func (s *Store) Close() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// SaveConfig 保存/更新应用级配置项
|
||||
func (s *Store) SaveConfig(key, value string) error {
|
||||
if s == nil || s.engine == nil || key == "" {
|
||||
return nil
|
||||
}
|
||||
existing := new(AppConfig)
|
||||
has, err := s.engine.Where("key = ?", key).Get(existing)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if has {
|
||||
_, err = s.engine.Where("key = ?", key).Update(&AppConfig{Value: value})
|
||||
return err
|
||||
}
|
||||
_, err = s.engine.Insert(&AppConfig{Key: key, Value: value})
|
||||
return err
|
||||
}
|
||||
|
||||
// GetConfig 读取应用级配置项；不存在时返回空串
|
||||
func (s *Store) GetConfig(key string) string {
|
||||
if s == nil || s.engine == nil || key == "" {
|
||||
return ""
|
||||
}
|
||||
cfg := new(AppConfig)
|
||||
has, err := s.engine.Where("key = ?", key).Get(cfg)
|
||||
if err != nil || !has {
|
||||
return ""
|
||||
}
|
||||
return cfg.Value
|
||||
}
|
||||
|
||||
// SaveMedia 按MD5去重保存/更新媒体信息
|
||||
func (s *Store) SaveMedia(info *MediaInfo) error {
|
||||
if s == nil || s.engine == nil {
|
||||
@@ -95,6 +136,65 @@ func (s *Store) SaveMedia(info *MediaInfo) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// InsertDuplicate 记录重复文件（与已入库内容 MD5 相同，独立入库便于统计重复数量）
|
||||
func (s *Store) InsertDuplicate(info *MediaInfo) error {
|
||||
if s == nil || s.engine == nil {
|
||||
return nil
|
||||
}
|
||||
_, err := s.engine.Insert(info)
|
||||
return err
|
||||
}
|
||||
|
||||
// SaveMediaBatch 在一个事务内批量按MD5去重保存/更新，避免大量文件逐个提交事务的 fsync 开销
|
||||
func (s *Store) SaveMediaBatch(infos []*MediaInfo) error {
|
||||
if s == nil || s.engine == nil || len(infos) == 0 {
|
||||
return nil
|
||||
}
|
||||
sess := s.engine.NewSession()
|
||||
defer sess.Close()
|
||||
if err := sess.Begin(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, info := range infos {
|
||||
existing := new(MediaInfo)
|
||||
has, err := sess.Where("file_md5 = ?", info.FileMd5).Get(existing)
|
||||
if err != nil {
|
||||
_ = sess.Rollback()
|
||||
return err
|
||||
}
|
||||
if has {
|
||||
info.Id = existing.Id
|
||||
if _, err = sess.ID(existing.Id).Update(info); err != nil {
|
||||
_ = sess.Rollback()
|
||||
return err
|
||||
}
|
||||
} else if _, err = sess.Insert(info); err != nil {
|
||||
_ = sess.Rollback()
|
||||
return err
|
||||
}
|
||||
}
|
||||
return sess.Commit()
|
||||
}
|
||||
|
||||
// InsertDuplicateBatch 在一个事务内批量插入重复文件记录
|
||||
func (s *Store) InsertDuplicateBatch(infos []*MediaInfo) error {
|
||||
if s == nil || s.engine == nil || len(infos) == 0 {
|
||||
return nil
|
||||
}
|
||||
sess := s.engine.NewSession()
|
||||
defer sess.Close()
|
||||
if err := sess.Begin(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, info := range infos {
|
||||
if _, err := sess.Insert(info); err != nil {
|
||||
_ = sess.Rollback()
|
||||
return err
|
||||
}
|
||||
}
|
||||
return sess.Commit()
|
||||
}
|
||||
|
||||
// FindMediaByFingerprint 按 (文件大小 + 头256KB + 尾256KB) 查找已入库记录，避免对大文件重复计算全量MD5
|
||||
func (s *Store) FindMediaByFingerprint(size int64, headMd5, tailMd5 string) (*MediaInfo, bool) {
|
||||
if s == nil || s.engine == nil || headMd5 == "" || tailMd5 == "" {
|
||||
@@ -121,14 +221,15 @@ func (s *Store) FindMediaByMd5(md5 string) (*MediaInfo, bool) {
|
||||
return info, true
|
||||
}
|
||||
|
||||
// UpdateMediaDest 按源路径回填文件的归类目录与去向
|
||||
func (s *Store) UpdateMediaDest(filePath, destDir string) error {
|
||||
// UpdateMediaDest 按源路径回填文件的归类目录与去向（destPath 为移动后的完整文件路径）
|
||||
func (s *Store) UpdateMediaDest(filePath, destDir, destPath string) error {
|
||||
if s == nil || s.engine == nil {
|
||||
return nil
|
||||
}
|
||||
_, err := s.engine.Where("file_path = ?", filePath).Update(&MediaInfo{
|
||||
Category: filepath.Base(destDir),
|
||||
DestDir: destDir,
|
||||
DestPath: destPath,
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -181,9 +282,10 @@ func (s *Store) CountMedia(q MediaQuery) (int64, error) {
|
||||
return sess.Count(new(MediaInfo))
|
||||
}
|
||||
|
||||
// newQuerySession 创建带查询条件的会话
|
||||
// newQuerySession 创建带查询条件的会话（默认排除重复文件分类记录）
|
||||
func (s *Store) newQuerySession(q MediaQuery) *xorm.Session {
|
||||
sess := s.engine.NewSession()
|
||||
sess.Where("COALESCE(category,'') <> ?", CategoryDuplicate)
|
||||
cond, args := buildMediaCond(q)
|
||||
if cond != "" {
|
||||
sess.Where(cond, args...)
|
||||
@@ -271,6 +373,19 @@ type MediaStats struct {
|
||||
Duplicate int64 `json:"duplicate"` // 重复文件数（同一MD5出现多次的去重后计数）
|
||||
DupSize int64 `json:"dupSize"` // 重复文件占用的总字节
|
||||
TopLarge []KV `json:"topLarge"` // 体积最大的 10 个文件
|
||||
|
||||
DurationTotal int64 `json:"durationTotal"` // 视频/音频总时长（秒）
|
||||
DurationAvg int64 `json:"durationAvg"` // 视频/音频平均时长（秒）
|
||||
AvgSize int64 `json:"avgSize"` // 平均文件体积（字节）
|
||||
MaxSize int64 `json:"maxSize"` // 最大文件体积（字节）
|
||||
MinSize int64 `json:"minSize"` // 最小文件体积（字节）
|
||||
|
||||
DurationBins []KV `json:"durationBins"` // 时长区间分布（视频/音频）
|
||||
CreatedCounts []KV `json:"createdCounts"` // 入库趋势（按日，近 30 天）
|
||||
CategoryCounts []KV `json:"categoryCounts"` // 归类目录分布
|
||||
ExtCounts []KV `json:"extCounts"` // 扩展名分布
|
||||
MimeCounts []KV `json:"mimeCounts"` // MIME 类型分布
|
||||
AudioCodecCount []KV `json:"audioCodecCount"` // 音频编码分布
|
||||
}
|
||||
|
||||
// KV 统计键值对
|
||||
@@ -286,46 +401,55 @@ func (s *Store) Stats() (*MediaStats, error) {
|
||||
return &MediaStats{}, nil
|
||||
}
|
||||
st := &MediaStats{
|
||||
ResCount: []KV{}, // 避免空库时返回 null，前端 .slice 直接可用
|
||||
ResCount: []KV{}, // 避免空库时返回 null，前端 .slice 直接可用
|
||||
DurationBins: []KV{},
|
||||
CreatedCounts: []KV{},
|
||||
CategoryCounts: []KV{},
|
||||
ExtCounts: []KV{},
|
||||
MimeCounts: []KV{},
|
||||
AudioCodecCount: []KV{},
|
||||
}
|
||||
|
||||
if total, err := s.engine.Count(new(MediaInfo)); err == nil {
|
||||
st.TotalFiles = total
|
||||
var totalCnt struct {
|
||||
C int64 `xorm:"c"`
|
||||
}
|
||||
if _, err := s.engine.SQL("SELECT COUNT(*) AS c FROM media_info WHERE COALESCE(category,'') <> '重复文件'").Get(&totalCnt); err == nil {
|
||||
st.TotalFiles = totalCnt.C
|
||||
}
|
||||
var sizeTotal struct {
|
||||
Size int64 `xorm:"size"`
|
||||
}
|
||||
if _, err := s.engine.SQL("SELECT COALESCE(SUM(file_size),0) AS size FROM media_info").Get(&sizeTotal); err == nil {
|
||||
if _, err := s.engine.SQL("SELECT COALESCE(SUM(file_size),0) AS size FROM media_info WHERE COALESCE(category,'') <> '重复文件'").Get(&sizeTotal); err == nil {
|
||||
st.TotalSize = sizeTotal.Size
|
||||
}
|
||||
|
||||
// 类型统计（数量 + 体积）
|
||||
typeRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT media_type AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info GROUP BY media_type ORDER BY count DESC").Find(&typeRows); err == nil {
|
||||
if err := s.engine.SQL("SELECT media_type AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE COALESCE(category,'') <> '重复文件' GROUP BY media_type ORDER BY count DESC").Find(&typeRows); err == nil {
|
||||
st.TypeCounts = typeRows
|
||||
}
|
||||
// 方向统计（仅含有效尺寸的图片/视频）
|
||||
orientRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT CASE WHEN width>height THEN 'horizontal' WHEN width<height THEN 'vertical' ELSE 'square' END AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE width>0 AND height>0 GROUP BY 1 ORDER BY count DESC").Find(&orientRows); err == nil {
|
||||
if err := s.engine.SQL("SELECT CASE WHEN width>height THEN 'horizontal' WHEN width<height THEN 'vertical' ELSE 'square' END AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE width>0 AND height>0 AND COALESCE(category,'') <> '重复文件' GROUP BY 1 ORDER BY count DESC").Find(&orientRows); err == nil {
|
||||
st.OrientCount = orientRows
|
||||
}
|
||||
// 编码 / 格式 / 分辨率 / 大文件统计
|
||||
codecRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT COALESCE(NULLIF(codec,''),'未知') AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE media_type='video' AND codec <> '' GROUP BY codec ORDER BY count DESC LIMIT 10").Find(&codecRows); err == nil {
|
||||
if err := s.engine.SQL("SELECT COALESCE(NULLIF(codec,''),'未知') AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE media_type='video' AND codec <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY codec ORDER BY count DESC LIMIT 10").Find(&codecRows); err == nil {
|
||||
st.CodecCount = codecRows
|
||||
}
|
||||
formatRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT COALESCE(NULLIF(format,''),'未知') AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE format <> '' GROUP BY format ORDER BY count DESC LIMIT 10").Find(&formatRows); err == nil {
|
||||
if err := s.engine.SQL("SELECT COALESCE(NULLIF(format,''), LOWER(ltrim(file_ext,'.'))) AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE (format <> '' OR file_ext <> '') AND COALESCE(category,'') <> '重复文件' GROUP BY key ORDER BY count DESC LIMIT 10").Find(&formatRows); err == nil {
|
||||
st.FormatCount = formatRows
|
||||
}
|
||||
largeRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT file_name AS key, 1 AS count, file_size AS size FROM media_info ORDER BY file_size DESC LIMIT 10").Find(&largeRows); err == nil {
|
||||
if err := s.engine.SQL("SELECT file_name AS key, 1 AS count, file_size AS size FROM media_info WHERE COALESCE(category,'') <> '重复文件' ORDER BY file_size DESC LIMIT 10").Find(&largeRows); err == nil {
|
||||
st.TopLarge = largeRows
|
||||
}
|
||||
|
||||
// 分辨率分布（Go 侧计算，复用归类规则）
|
||||
resMap := make(map[string]int64)
|
||||
rows, err := s.engine.SQL("SELECT width, height FROM media_info WHERE width>0 AND height>0").Query()
|
||||
rows, err := s.engine.SQL("SELECT width, height FROM media_info WHERE width>0 AND height>0 AND COALESCE(category,'') <> '重复文件'").Query()
|
||||
if err != nil {
|
||||
return st, err
|
||||
}
|
||||
@@ -346,6 +470,60 @@ func (s *Store) Stats() (*MediaStats, error) {
|
||||
}
|
||||
sortKVs(st.ResCount)
|
||||
|
||||
// 时长统计（视频/音频）
|
||||
var dur struct {
|
||||
Total float64 `xorm:"total"`
|
||||
Avg float64 `xorm:"avg"`
|
||||
}
|
||||
if _, err := s.engine.SQL("SELECT COALESCE(SUM(duration),0) AS total, COALESCE(AVG(CASE WHEN duration>0 THEN duration END),0) AS avg FROM media_info WHERE duration>0 AND COALESCE(category,'') <> '重复文件'").Get(&dur); err == nil {
|
||||
st.DurationTotal = int64(dur.Total)
|
||||
st.DurationAvg = int64(dur.Avg)
|
||||
}
|
||||
// 时长区间分布
|
||||
durRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT CASE WHEN duration<60 THEN '<1分钟' WHEN duration<300 THEN '1-5分钟' WHEN duration<1800 THEN '5-30分钟' WHEN duration<3600 THEN '30分钟-1小时' ELSE '>1小时' END AS key, COUNT(*) AS count, COALESCE(SUM(duration),0) AS size FROM media_info WHERE duration>0 AND COALESCE(category,'') <> '重复文件' GROUP BY key ORDER BY count DESC").Find(&durRows); err == nil {
|
||||
st.DurationBins = durRows
|
||||
}
|
||||
// 平均/最大/最小文件体积
|
||||
var sizeStat struct {
|
||||
Avg int64 `xorm:"avg"`
|
||||
Max int64 `xorm:"max"`
|
||||
Min int64 `xorm:"min"`
|
||||
}
|
||||
if _, err := s.engine.SQL("SELECT COALESCE(AVG(file_size),0) AS avg, COALESCE(MAX(file_size),0) AS max, COALESCE(MIN(file_size),0) AS min FROM media_info WHERE COALESCE(category,'') <> '重复文件'").Get(&sizeStat); err == nil {
|
||||
st.AvgSize = sizeStat.Avg
|
||||
st.MaxSize = sizeStat.Max
|
||||
st.MinSize = sizeStat.Min
|
||||
}
|
||||
// 入库趋势（近 30 天按日分组）
|
||||
createdRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT substr(created_at,1,10) AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE created_at <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY key ORDER BY key DESC LIMIT 30").Find(&createdRows); err == nil {
|
||||
for i, j := 0, len(createdRows)-1; i < j; i, j = i+1, j-1 {
|
||||
createdRows[i], createdRows[j] = createdRows[j], createdRows[i]
|
||||
}
|
||||
st.CreatedCounts = createdRows
|
||||
}
|
||||
// 归类目录分布
|
||||
catRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT category AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE category <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY category ORDER BY count DESC LIMIT 15").Find(&catRows); err == nil {
|
||||
st.CategoryCounts = catRows
|
||||
}
|
||||
// 扩展名分布
|
||||
extRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT LOWER(ltrim(file_ext,'.')) AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE file_ext <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY key ORDER BY count DESC LIMIT 10").Find(&extRows); err == nil {
|
||||
st.ExtCounts = extRows
|
||||
}
|
||||
// MIME 类型分布
|
||||
mimeRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT mime_type AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE mime_type <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY mime_type ORDER BY count DESC LIMIT 10").Find(&mimeRows); err == nil {
|
||||
st.MimeCounts = mimeRows
|
||||
}
|
||||
// 音频编码分布
|
||||
audioRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT COALESCE(NULLIF(codec,''),'未知') AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info WHERE media_type='audio' AND codec <> '' AND COALESCE(category,'') <> '重复文件' GROUP BY codec ORDER BY count DESC LIMIT 10").Find(&audioRows); err == nil {
|
||||
st.AudioCodecCount = audioRows
|
||||
}
|
||||
|
||||
// 重复文件统计（同一 MD5 出现多次）
|
||||
dupRows := make([]KV, 0)
|
||||
if err := s.engine.SQL("SELECT file_md5 AS key, COUNT(*) AS count, COALESCE(SUM(file_size),0) AS size FROM media_info GROUP BY file_md5 HAVING COUNT(*)>1").Find(&dupRows); err == nil {
|
||||
|
||||
Reference in New Issue
Block a user