Files
bj_power/bj_power_wms/internal/backup/backup.go
T

638 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package backup 实现「备份到外部存储(U盘/移动硬盘)」与「按时间窗物理清理」两大能力(WMS 端)。
// 与 MES 端对称:时间字段统一 Int64 unix 秒;各端附件文件由各端独占写入与引用
// (UUID 文件名,物料同步不含附件),不存在跨系统共享文件,清理不依赖对端服务。
package backup
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"time"
"bj_power_wms/ent"
att "bj_power_wms/ent/attachment"
br "bj_power_wms/ent/backuprecord"
"bj_power_wms/ent/eventlog"
evlog "bj_power_wms/internal/eventlog"
"golang.org/x/sys/windows"
)
// ItemDef 可选备份/清理数据项(质量 4 表有意不出现)。
type ItemDef struct {
Key string `json:"key"`
Name string `json:"name"`
Kind string `json:"kind"` // log | attachment
}
// WMSItems WMS 端可选数据项(event_log/attachments)。
var WMSItems = []ItemDef{
{Key: "event_log", Name: "操作日志", Kind: "log"},
{Key: "attachments", Name: "业务附件", Kind: "attachment"},
}
// BackupRequest 备份请求。
type BackupRequest struct {
TargetPath string `json:"targetPath"`
Start int64 `json:"start"`
End int64 `json:"end"`
Items []string `json:"items"`
Operator string `json:"operator"`
}
// CleanupRequest 清理请求。
type CleanupRequest struct {
Start int64 `json:"start"`
End int64 `json:"end"`
Items []string `json:"items"`
Operator string `json:"operator"`
}
// ItemResult 单项导出结果。
type ItemResult struct {
Name string `json:"name"`
Key string `json:"key"`
Rows int64 `json:"rows"`
Bytes int64 `json:"bytes"`
}
// BackupResult 备份结果。
type BackupResult struct {
BackupPath string `json:"backupPath"`
TotalSize int64 `json:"totalSize"`
Checksum string `json:"checksum"`
Items []ItemResult `json:"items"`
}
// CleanupResult 清理结果。
type CleanupResult struct {
RangeStart int64 `json:"rangeStart"`
RangeEnd int64 `json:"rangeEnd"`
Deleted []ItemResult `json:"deleted"`
Skipped []ItemResult `json:"skipped"`
}
// MissingSeg 缺失时间段。
type MissingSeg struct {
Start int64 `json:"start"`
End int64 `json:"end"`
}
// CoverageResult 覆盖校验结果。
type CoverageResult struct {
OK bool `json:"ok"`
Missing []MissingSeg `json:"missing"`
}
// Progress 任务进度。
type Progress struct {
JobID string `json:"jobId"`
Phase string `json:"phase"`
Stage string `json:"stage"`
Percent int `json:"percent"`
Message string `json:"message"`
Done bool `json:"done"`
Failed bool `json:"failed"`
Result any `json:"result,omitempty"`
}
// Service 备份/清理服务。
type Service struct {
client *ent.Client
ev *evlog.Writer
attRoot string
mu sync.Mutex
jobs map[string]*Progress
}
// New 构造服务。
func New(client *ent.Client, ev *evlog.Writer, attRoot string) *Service {
return &Service{
client: client,
ev: ev,
attRoot: attRoot,
jobs: make(map[string]*Progress),
}
}
// ListItems 返回可选数据项。
func (s *Service) ListItems() []ItemDef { return WMSItems }
// ---------------- 备份 ----------------
// StartBackup 启动异步备份,返回 jobID。
func (s *Service) StartBackup(req BackupRequest) (string, error) {
if strings.TrimSpace(req.TargetPath) == "" {
return "", fmt.Errorf("请选择备份目标路径")
}
if !filepath.IsAbs(req.TargetPath) {
return "", fmt.Errorf("备份目标路径必须为绝对路径")
}
if req.End <= req.Start {
return "", fmt.Errorf("时间窗不合法(结束须晚于开始)")
}
if len(req.Items) == 0 {
return "", fmt.Errorf("请至少选择一个数据项")
}
jobID := newJobID()
s.mu.Lock()
s.jobs[jobID] = &Progress{JobID: jobID, Phase: "backup", Stage: "preparing", Percent: 0}
s.mu.Unlock()
go s.runBackup(jobID, req)
return jobID, nil
}
func (s *Service) runBackup(jobID string, req BackupRequest) {
ctx := context.Background()
setStage := func(stage string, pct int, msg string) {
s.mu.Lock()
if j, ok := s.jobs[jobID]; ok {
j.Stage, j.Percent, j.Message = stage, pct, msg
}
s.mu.Unlock()
}
est, err := s.estimateBytes(ctx, req)
if err != nil {
s.failBackup(jobID, req, err)
return
}
free, derr := diskFree(req.TargetPath)
if derr == nil && free < uint64(est) {
need := est - int64(free)
s.failBackup(jobID, req, fmt.Errorf("目标盘空间不足:需要约 %s,当前剩余 %s(还差 %s)",
humanBytes(est), humanBytes(int64(free)), humanBytes(need)))
return
}
setStage("writing", 5, "准备写入备份包…")
stamp := time.Now().Format("20060102_150405")
zipPath := filepath.Join(req.TargetPath, "backup_wms_"+stamp+".zip")
if err := os.MkdirAll(req.TargetPath, 0o755); err != nil {
s.failBackup(jobID, req, fmt.Errorf("无法创建目标目录:%w", err))
return
}
f, err := os.Create(zipPath)
if err != nil {
s.failBackup(jobID, req, fmt.Errorf("无法创建备份文件(U盘是否已拔出/无写入权限):%w", err))
return
}
h := sha256.New()
zw := newZipWriter(f, h)
results := make([]ItemResult, 0, len(req.Items))
total := int64(0)
itemNames := make(map[string]string, len(WMSItems))
for _, it := range WMSItems {
itemNames[it.Key] = it.Name
}
for i, key := range req.Items {
setStage("writing", 5+int(float64(i)/float64(len(req.Items))*80), "导出 "+itemName(itemNames, key)+" …")
var ir ItemResult
var e error
if key == "attachments" {
ir, e = s.exportAttachment(ctx, zw, req.Start, req.End)
} else {
ir, e = s.exportLog(ctx, zw, key, req.Start, req.End)
}
if e != nil {
_ = zw.Close()
_ = f.Close()
_ = os.Remove(zipPath)
s.failBackup(jobID, req, fmt.Errorf("导出 %s 失败(U盘是否已拔出):%w", itemName(itemNames, key), e))
return
}
ir.Key = key
ir.Name = itemName(itemNames, key)
results = append(results, ir)
total += ir.Bytes
}
manifest := map[string]any{
"generatedAt": time.Now().Format(time.RFC3339),
"scope": "wms",
"rangeStart": req.Start,
"rangeEnd": req.End,
"rangeStartText": time.Unix(req.Start, 0).Format("2006-01-02 15:04:05"),
"rangeEndText": time.Unix(req.End, 0).Format("2006-01-02 15:04:05"),
"operator": req.Operator,
"items": results,
"totalSize": total,
"checksum": "",
}
manifestBytes, _ := json.MarshalIndent(manifest, "", " ")
if err := zw.WriteFile("manifest.json", manifestBytes); err != nil {
_ = zw.Close()
_ = f.Close()
_ = os.Remove(zipPath)
s.failBackup(jobID, req, fmt.Errorf("写入 manifest 失败:%w", err))
return
}
indexHTML := buildIndexHTML("WMS 备份清单", manifest, "")
if err := zw.WriteFile("index.html", []byte(indexHTML)); err != nil {
_ = zw.Close()
_ = f.Close()
_ = os.Remove(zipPath)
s.failBackup(jobID, req, fmt.Errorf("写入 index.html 失败:%w", err))
return
}
if err := zw.Close(); err != nil {
_ = f.Close()
_ = os.Remove(zipPath)
// 备份包已随 os.Remove 删除,无残留产物:状态=failed(非 partial)
s.failBackup(jobID, req, fmt.Errorf("备份写入中断(U盘是否已拔出):%w", err))
return
}
if err := f.Close(); err != nil {
_ = os.Remove(zipPath)
// 备份包已随 os.Remove 删除,无残留产物:状态=failed(非 partial)
s.failBackup(jobID, req, fmt.Errorf("备份文件关闭失败:%w", err))
return
}
fi, _ := os.Stat(zipPath)
if fi != nil {
total = fi.Size()
}
s.finishBackup(jobID, req, zipPath, results, total, hex.EncodeToString(h.Sum(nil)), false, nil)
}
func (s *Service) exportLog(ctx context.Context, zw *zipWriter, key string, start, end int64) (ItemResult, error) {
if key != "event_log" {
return ItemResult{}, fmt.Errorf("未知数据项 %s", key)
}
rows, err := s.client.EventLog.Query().
Where(eventlog.CreatedAtGTE(start), eventlog.CreatedAtLTE(end)).All(ctx)
if err != nil {
return ItemResult{}, err
}
ir := ItemResult{Rows: int64(len(rows))}
if err := zw.WriteJSONL(key+"/data.jsonl", sliceToAny(rows), &ir.Bytes); err != nil {
return ir, err
}
return ir, nil
}
func (s *Service) exportAttachment(ctx context.Context, zw *zipWriter, start, end int64) (ItemResult, error) {
rows, err := s.client.Attachment.Query().
Where(att.CreatedAtGTE(start), att.CreatedAtLTE(end), att.DeletedEQ(false)).All(ctx)
if err != nil {
return ItemResult{}, err
}
ir := ItemResult{Rows: int64(len(rows))}
meta := make([]map[string]any, 0, len(rows))
for _, r := range rows {
abs, ok := safeAbsPath(s.attRoot, r.FilePath)
if !ok {
continue
}
_, ferr := os.Stat(abs)
if ferr != nil {
meta = append(meta, map[string]any{
"id": r.ID, "file_name": r.FileName, "file_path": r.FilePath,
"file_size": r.FileSize, "missing": true,
})
continue
}
src, oerr := os.Open(abs)
if oerr != nil {
return ir, oerr
}
entryName := "attachments/" + strings.ReplaceAll(r.FilePath, "\\", "/")
n, cerr := zw.CopyFrom(entryName, src)
_ = src.Close()
if cerr != nil {
return ir, cerr
}
ir.Bytes += n
meta = append(meta, map[string]any{
"id": r.ID, "file_name": r.FileName, "file_path": r.FilePath,
"file_size": r.FileSize, "bytes": n,
})
}
mb, _ := json.MarshalIndent(meta, "", " ")
if err := zw.WriteFile("attachment/data.jsonl", mb); err != nil {
return ir, err
}
ir.Bytes += int64(len(mb))
return ir, nil
}
func (s *Service) finishBackup(jobID string, req BackupRequest, zipPath string, results []ItemResult, total int64, checksum string, partial bool, runErr error) {
status := "success"
if partial {
status = "partial"
}
itemsJSON, _ := json.Marshal(results)
_, _ = s.client.BackupRecord.Create().
SetExecutedAt(time.Now().Unix()).
SetRangeStart(req.Start).
SetRangeEnd(req.End).
SetBackupPath(zipPath).
SetTotalSize(total).
SetItemsJSON(string(itemsJSON)).
SetStatus(status).
SetOperator(req.Operator).
SetChecksum(checksum).
SetScope("wms").
Save(context.Background())
s.ev.Write(context.Background(), "backup.create", req.Operator, "backup", "",
fmt.Sprintf("备份到 %s(范围 %s ~ %s,共 %d 项,%s,校验和 %s)",
zipPath, time.Unix(req.Start, 0).Format("2006-01-02"), time.Unix(req.End, 0).Format("2006-01-02"),
len(results), humanBytes(total), checksum), nil)
res := BackupResult{BackupPath: zipPath, TotalSize: total, Checksum: checksum, Items: results}
s.mu.Lock()
if j, ok := s.jobs[jobID]; ok {
j.Done = true
if partial {
j.Failed = true
j.Message = "备份部分成功(U盘可能已拔出):" + runErr.Error()
} else {
j.Message = "备份完成"
}
j.Result = res
j.Percent = 100
}
s.mu.Unlock()
}
func (s *Service) failBackup(jobID string, req BackupRequest, runErr error) {
_, _ = s.client.BackupRecord.Create().
SetExecutedAt(time.Now().Unix()).
SetRangeStart(req.Start).
SetRangeEnd(req.End).
SetBackupPath("").
SetTotalSize(0).
SetItemsJSON("[]").
SetStatus("failed").
SetOperator(req.Operator).
SetChecksum("").
SetScope("wms").
Save(context.Background())
s.mu.Lock()
if j, ok := s.jobs[jobID]; ok {
j.Done = true
j.Failed = true
j.Message = runErr.Error()
j.Percent = 100
}
s.mu.Unlock()
}
func (s *Service) estimateBytes(ctx context.Context, req BackupRequest) (int64, error) {
var total int64
for _, key := range req.Items {
switch key {
case "attachments":
rows, err := s.client.Attachment.Query().
Where(att.CreatedAtGTE(req.Start), att.CreatedAtLTE(req.End)).All(ctx)
if err != nil {
return 0, err
}
for _, r := range rows {
total += r.FileSize
}
total += int64(len(rows)) * 256
case "event_log":
n, err := s.client.EventLog.Query().
Where(eventlog.CreatedAtGTE(req.Start), eventlog.CreatedAtLTE(req.End)).Count(ctx)
if err != nil {
return 0, err
}
total += int64(n) * 1024
}
}
total += 4096
return total, nil
}
// ---------------- 清理 ----------------
// StartCleanup 启动异步清理,返回 jobID。
func (s *Service) StartCleanup(req CleanupRequest) (string, error) {
if req.End <= req.Start {
return "", fmt.Errorf("时间窗不合法")
}
if len(req.Items) == 0 {
return "", fmt.Errorf("请至少选择一个数据项")
}
jobID := newJobID()
s.mu.Lock()
s.jobs[jobID] = &Progress{JobID: jobID, Phase: "cleanup", Stage: "checking", Percent: 0}
s.mu.Unlock()
go s.runCleanup(jobID, req)
return jobID, nil
}
// CheckCoverage 校验备份覆盖(供前端预检)。
func (s *Service) CheckCoverage(start, end int64) (CoverageResult, error) {
recs, err := s.client.BackupRecord.Query().Where(br.StatusEQ("success")).All(context.Background())
if err != nil {
return CoverageResult{}, err
}
ivs := make([][2]int64, 0, len(recs))
for _, r := range recs {
ivs = append(ivs, [2]int64{r.RangeStart, r.RangeEnd})
}
missing := complement(ivs, start, end)
return CoverageResult{OK: len(missing) == 0, Missing: missing}, nil
}
func (s *Service) runCleanup(jobID string, req CleanupRequest) {
ctx := context.Background()
setStage := func(stage string, pct int, msg string) {
s.mu.Lock()
if j, ok := s.jobs[jobID]; ok {
j.Stage, j.Percent, j.Message = stage, pct, msg
}
s.mu.Unlock()
}
cov, err := s.CheckCoverage(req.Start, req.End)
if err != nil {
s.failCleanup(jobID, req, fmt.Errorf("覆盖校验失败:%w", err))
return
}
if !cov.OK {
s.failCleanup(jobID, req, fmt.Errorf("该时间窗未被完整备份,禁止清理。缺失时间段:%s", formatMissing(cov.Missing)))
return
}
deleted := make([]ItemResult, 0, len(req.Items))
skipped := make([]ItemResult, 0, len(req.Items))
itemNames := make(map[string]string, len(WMSItems))
for _, it := range WMSItems {
itemNames[it.Key] = it.Name
}
for i, key := range req.Items {
setStage("cleaning", int(float64(i)/float64(len(req.Items))*90), "清理 "+itemName(itemNames, key)+" …")
if key == "attachments" {
d, sk, e := s.cleanupAttachment(ctx, req.Start, req.End)
if e != nil {
s.failCleanup(jobID, req, fmt.Errorf("清理附件失败:%w", e))
return
}
d.Key, d.Name = key, itemName(itemNames, key)
sk.Key, sk.Name = key, itemName(itemNames, key)
deleted = append(deleted, d)
if sk.Rows > 0 {
skipped = append(skipped, sk)
}
} else {
n, e := s.cleanupLog(ctx, key, req.Start, req.End)
if e != nil {
s.failCleanup(jobID, req, fmt.Errorf("清理 %s 失败:%w", itemName(itemNames, key), e))
return
}
deleted = append(deleted, ItemResult{Key: key, Name: itemName(itemNames, key), Rows: n, Bytes: 0})
}
}
s.ev.Write(ctx, "backup.cleanup", req.Operator, "backup", "",
fmt.Sprintf("按时间窗清理(%s ~ %s):%s",
time.Unix(req.Start, 0).Format("2006-01-02"), time.Unix(req.End, 0).Format("2006-01-02"),
summaryDeleted(deleted, skipped)), nil)
setStage("done", 100, "清理完成")
s.mu.Lock()
if j, ok := s.jobs[jobID]; ok {
j.Done = true
j.Result = CleanupResult{RangeStart: req.Start, RangeEnd: req.End, Deleted: deleted, Skipped: skipped}
}
s.mu.Unlock()
}
func (s *Service) cleanupLog(ctx context.Context, key string, start, end int64) (int64, error) {
if key != "event_log" {
return 0, nil
}
n, err := s.client.EventLog.Delete().
Where(eventlog.CreatedAtGTE(start), eventlog.CreatedAtLTE(end)).Exec(ctx)
return int64(n), err
}
// cleanupAttachment 物理删除文件 + DELETE 记录(附件文件由本端独占写入与引用,无需跨库校验)。
func (s *Service) cleanupAttachment(ctx context.Context, start, end int64) (ItemResult, ItemResult, error) {
rows, err := s.client.Attachment.Query().
Where(att.CreatedAtGTE(start), att.CreatedAtLTE(end)).All(ctx)
if err != nil {
return ItemResult{}, ItemResult{}, err
}
del := ItemResult{}
sk := ItemResult{}
for _, r := range rows {
abs, ok := safeAbsPath(s.attRoot, r.FilePath)
if ok {
if rerr := os.Remove(abs); rerr != nil && !os.IsNotExist(rerr) {
sk.Rows++
continue
}
}
if _, derr := s.client.Attachment.Delete().Where(att.ID(r.ID)).Exec(ctx); derr != nil {
sk.Rows++
continue
}
del.Rows++
del.Bytes += r.FileSize
}
return del, sk, nil
}
func (s *Service) failCleanup(jobID string, req CleanupRequest, runErr error) {
s.ev.Write(context.Background(), "backup.cleanup.reject", req.Operator, "backup", "",
"清理被拒:"+runErr.Error(), nil)
s.mu.Lock()
if j, ok := s.jobs[jobID]; ok {
j.Done = true
j.Failed = true
j.Message = runErr.Error()
j.Percent = 100
}
s.mu.Unlock()
}
// ---------------- 进度 ----------------
// Progress 返回任务进度。
func (s *Service) Progress(jobID string) (Progress, bool) {
s.mu.Lock()
defer s.mu.Unlock()
j, ok := s.jobs[jobID]
if !ok {
return Progress{}, false
}
cp := *j
return cp, true
}
// ---------------- 工具 ----------------
func itemName(m map[string]string, key string) string {
if n, ok := m[key]; ok {
return n
}
return key
}
func safeAbsPath(root, rel string) (string, bool) {
rel = strings.ReplaceAll(rel, "\\", "/")
clean := filepath.Clean(filepath.Join(root, rel))
rootAbs, err := filepath.Abs(root)
if err != nil {
return "", false
}
abs, err := filepath.Abs(clean)
if err != nil {
return "", false
}
if abs != rootAbs && !strings.HasPrefix(abs, rootAbs+string(os.PathSeparator)) {
return "", false
}
return abs, true
}
func diskFree(path string) (uint64, error) {
var free uint64
p, err := windows.UTF16PtrFromString(path)
if err != nil {
return 0, err
}
if err := windows.GetDiskFreeSpaceEx(p, &free, nil, nil); err != nil {
return 0, err
}
return free, nil
}
func humanBytes(n int64) string {
const unit = 1024
if n < unit {
return fmt.Sprintf("%d B", n)
}
div, exp := int64(unit), 0
for x := n / unit; x >= unit; x /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.2f %cB", float64(n)/float64(div), "KMGTPE"[exp])
}
func newJobID() string {
b := make([]byte, 8)
_, _ = io.ReadFull(randReader(), b)
return hex.EncodeToString(b) + fmt.Sprintf("%d", time.Now().UnixNano()%100000)
}