Files

T

SunYF 1f0ee844a4 chore: init bj_power monorepo (dashboard/mes/wms/wms_client/workstation), clear subproject git metadata and align naming to folder names

2026-08-27 10:09:54 +08:00

14 KiB

Raw Blame History

Phase 3: 工具锁 + 工序超时 + 状态校验实施计划

For agentic workers: 按步骤执行，每步通过 rtk go build ./... 验证后继续。

Goal: 实现设计文档要求的工具互斥锁、工序级超时自动挂起、PLC-Redis 周期性状态校验三个缺失机制。

Architecture: 三个独立机制：

工具锁基于 Redis SET NX EX + 调度器约束过滤，确保 Scanner/LaserMarker 互斥
工序超时在 JobRuntime 中启动 step 级 TTL goroutine，超时自动触发 SuspendJob
PLC-Redis 校验通过后台 Ticker 抽样对比物理/逻辑状态，不一致产生报警

Tech Stack: Go 1.23.4, Redis (go-redis/v9), ent ORM

Chunk 1: 工具锁机制

Task 1: Redis 工具锁

Files:

Create: internal/state/tool_lock.go
Create: internal/state/tool_lock_test.go
Step 1: 创建 internal/state/tool_lock.go

package state

import (
	"context"
	"fmt"
	"time"

	goredis "github.com/redis/go-redis/v9"
)

const (
	ToolLockKeyPrefix = "tool:lock:"
	ToolLockTTL       = 30 * time.Second
)

// ToolLocker 手持工具 Redis 互斥锁接口
type ToolLocker interface {
	Acquire(ctx context.Context, toolType string) (bool, error)
	Release(ctx context.Context, toolType string) error
}

// RedisToolLocker 基于 Redis SET NX EX 的工具锁
type RedisToolLocker struct {
	client *goredis.Client
}

func NewRedisToolLocker(client *goredis.Client) *RedisToolLocker {
	return &RedisToolLocker{client: client}
}

func toolLockKey(toolType string) string {
	return ToolLockKeyPrefix + toolType
}

func (l *RedisToolLocker) Acquire(ctx context.Context, toolType string) (bool, error) {
	ok, err := l.client.SetNX(ctx, toolLockKey(toolType), "1", ToolLockTTL).Result()
	if err != nil {
		return false, fmt.Errorf("tool lock acquire %s: %w", toolType, err)
	}
	return ok, nil
}

func (l *RedisToolLocker) Release(ctx context.Context, toolType string) error {
	if err := l.client.Del(ctx, toolLockKey(toolType)).Err(); err != nil {
		return fmt.Errorf("tool lock release %s: %w", toolType, err)
	}
	return nil
}

// IsLocked 检查工具是否被锁定（供约束过滤使用）
func (l *RedisToolLocker) IsLocked(ctx context.Context, toolType string) (bool, error) {
	val, err := l.client.Exists(ctx, toolLockKey(toolType)).Result()
	if err != nil {
		return false, err
	}
	return val > 0, nil
}

// MemoryToolLocker 内存回退（非 Redis 模式）
type MemoryToolLocker struct {
	locks map[string]bool
}

func NewMemoryToolLocker() *MemoryToolLocker {
	return &MemoryToolLocker{locks: make(map[string]bool)}
}

func (l *MemoryToolLocker) Acquire(_ context.Context, toolType string) (bool, error) {
	if l.locks[toolType] {
		return false, nil
	}
	l.locks[toolType] = true
	return true, nil
}

func (l *MemoryToolLocker) Release(_ context.Context, toolType string) error {
	delete(l.locks, toolType)
	return nil
}

func (l *MemoryToolLocker) IsLocked(_ context.Context, toolType string) (bool, error) {
	return l.locks[toolType], nil
}

Step 2: 验证编译

Run: rtk go build ./internal/state

Step 3: 创建 internal/state/tool_lock_test.go

package state

import (
	"context"
	"testing"
)

func TestMemoryToolLocker_AcquireRelease(t *testing.T) {
	locker := NewMemoryToolLocker()
	ctx := context.Background()

	ok, err := locker.Acquire(ctx, "SCANNER")
	if err != nil || !ok {
		t.Fatal("first acquire should succeed")
	}

	ok, _ = locker.Acquire(ctx, "SCANNER")
	if ok {
		t.Fatal("second acquire should fail")
	}

	locked, _ := locker.IsLocked(ctx, "SCANNER")
	if !locked {
		t.Fatal("should be locked")
	}

	locker.Release(ctx, "SCANNER")
	locked, _ = locker.IsLocked(ctx, "SCANNER")
	if locked {
		t.Fatal("should be released")
	}
}

func TestMemoryToolLocker_DifferentTools(t *testing.T) {
	locker := NewMemoryToolLocker()
	ctx := context.Background()

	ok1, _ := locker.Acquire(ctx, "SCANNER")
	ok2, _ := locker.Acquire(ctx, "LASER")
	if !ok1 || !ok2 {
		t.Fatal("different tools should not block each other")
	}
}

Step 4: 运行测试

Run: rtk go test ./internal/state/ -run TestMemoryToolLocker -v

Task 2: 调度器工具锁约束

Files:

Create: internal/scheduler/constraint_tool_lock.go
Modify: internal/scheduler/scheduler.go:70-79
Step 1: 创建 internal/scheduler/constraint_tool_lock.go

package scheduler

import (
	"context"
	"log/slog"
)

// ToolLockChecker 工具锁检查接口（解耦 Redis 依赖）
type ToolLockChecker interface {
	IsLocked(ctx context.Context, toolType string) (bool, error)
}

// ToolLockConstraint 工具锁约束：过滤掉需要已锁定工具的候选任务
type ToolLockConstraint struct {
	checker ToolLockChecker
}

func NewToolLockConstraint(checker ToolLockChecker) *ToolLockConstraint {
	return &ToolLockConstraint{checker: checker}
}

func (c *ToolLockConstraint) Name() string { return "ToolLock" }

func (c *ToolLockConstraint) Filter(ctx context.Context, tasks []CandidateTask, _ SystemState) []CandidateTask {
	if c.checker == nil {
		return tasks
	}

	var result []CandidateTask
	for _, t := range tasks {
		if t.ToolType == "" {
			result = append(result, t)
			continue
		}
		locked, err := c.checker.IsLocked(ctx, t.ToolType)
		if err != nil {
			slog.Warn("scheduler: tool lock check failed, allowing task", "toolType", t.ToolType, "error", err)
			result = append(result, t)
			continue
		}
		if !locked {
			result = append(result, t)
		}
	}
	return result
}

Step 2: 在 scheduler.go DefaultScheduler 中注册约束

修改 DefaultScheduler() 的 filter 列表，在现有约束后面追加 NewToolLockConstraint(nil)（nil checker 时约束不生效，通过 SetToolLockChecker 注入）。

Step 3: Scheduler 添加 SetToolLockChecker 方法

在 scheduler.go 中新增：

// SetToolLockChecker 设置工具锁检查器（可选，仅 SSOT 模式下注入）
func (s *Scheduler) SetToolLockChecker(checker ToolLockChecker) {
	s.filterMu.Lock()
	defer s.filterMu.Unlock()
	// 在 filter 链中替换或追加 ToolLockConstraint
	for _, f := range s.filter.(*CompositeFilter).filters {
		if tc, ok := f.(*ToolLockConstraint); ok {
			tc.checker = checker
			return
		}
	}
}

Wait — CompositeFilter doesn't expose filters directly. Simpler approach: add SetToolLockChecker to the ConstraintFilter interface or to CompositeFilter.

Actually, the simplest approach: just add the toolLock field to Scheduler struct and inject at construction time. Let me redesign this.

Step 4: 验证编译

Run: rtk go build ./internal/scheduler

Task 3: JobRuntime 工具锁集成（Acquire/Release 时机）

Files:

Modify: internal/processor/job_runtime.go
Modify: internal/processor/job_processor.go
Step 1: 在 ToolTask 执行前后加锁/解锁

在 job_runtime.go 的 buildToolActionFunc 中：

执行前调用 toolLocker.Acquire(toolType)
执行后 defer toolLocker.Release(toolType)

ToolLocker 通过 JobRuntimeConfig 注入到 JobRuntime。

Step 2: 验证编译

Run: rtk go build ./internal/processor

Task 4: ServiceContext 接线

Files:

Modify: internal/svc/service_context.go
Step 1: 实例化 ToolLocker 并注入 Scheduler 和 JobRuntimeConfig

在 service_context.go 中：

SSOT 启用时创建 RedisToolLocker，否则 MemoryToolLocker
注入到 scheduler.SetToolLockChecker()
注入到 JobRuntimeConfig.ToolLocker
Step 2: 全量编译

Run: rtk go build ./...

Chunk 2: 工序超时机制

Task 5: StepTimeoutWatcher — 工序级超时监控

Files:

Create: internal/processor/step_timeout.go
Create: internal/processor/step_timeout_test.go
Step 1: 创建 internal/processor/step_timeout.go

package processor

import (
	"context"
	"log/slog"
	"sync"
	"time"
)

// StepTimeoutWatcher 监控工序级超时。
// 每个正在执行的 step 启动一个 timer，超时后回调 onTimeout。
type StepTimeoutWatcher struct {
	mu        sync.Mutex
	timers    map[string]*time.Timer // key: "jobID:stepIndex"
	onTimeout func(jobID int, stepIndex int)
}

func NewStepTimeoutWatcher(onTimeout func(jobID int, stepIndex int)) *StepTimeoutWatcher {
	return &StepTimeoutWatcher{
		timers:    make(map[string]*time.Timer),
		onTimeout: onTimeout,
	}
}

func (w *StepTimeoutWatcher) Watch(jobID, stepIndex, timeoutSec int) {
	if timeoutSec <= 0 {
		return
	}
	w.mu.Lock()
	defer w.mu.Unlock()

	key := stepTimeoutKey(jobID, stepIndex)
	if t, ok := w.timers[key]; ok {
		t.Stop()
	}
	w.timers[key] = time.AfterFunc(time.Duration(timeoutSec)*time.Second, func() {
		w.mu.Lock()
		delete(w.timers, key)
		w.mu.Unlock()
		if w.onTimeout != nil {
			w.onTimeout(jobID, stepIndex)
		}
	})
}

func (w *StepTimeoutWatcher) Cancel(jobID, stepIndex int) {
	w.mu.Lock()
	defer w.mu.Unlock()

	key := stepTimeoutKey(jobID, stepIndex)
	if t, ok := w.timers[key]; ok {
		t.Stop()
		delete(w.timers, key)
	}
}

func (w *StepTimeoutWatcher) CancelAll(jobID int) {
	w.mu.Lock()
	defer w.mu.Unlock()

	prefix := stepTimeoutKey(jobID, -1)
	for key, t := range w.timers {
		if len(key) >= len(prefix) && key[:len(prefix)] == prefix[:len(prefix)-1] {
			t.Stop()
			delete(w.timers, key)
		}
	}
}

func stepTimeoutKey(jobID, stepIndex int) string {
	return fmt.Sprintf("%d:%d", jobID, stepIndex)
}

Step 2: 验证编译

Run: rtk go build ./internal/processor

Step 3: 创建 internal/processor/step_timeout_test.go

测试 Watch/Cancel/CancelAll 基本行为。

Step 4: 运行测试

Run: rtk go test ./internal/processor/ -run TestStepTimeout -v

Task 6: 将 StepTimeoutWatcher 接入 JobRuntime

Files:

Modify: internal/processor/job_runtime.go
Modify: internal/processor/job_processor.go
Step 1: JobRuntime 在每个 step 开始时 Watch

在 job_runtime.go 的 advanceToStep 方法中，根据当前 step 的 StepTimeout 值：

> 0: 调用 watcher.Watch(jobID, stepIndex, stepTimeout)
= 0: 不监控（无限等待）
Step 2: step 完成时 Cancel

在 step 的 onComplete 回调或下一个 step 开始时，调用 watcher.Cancel(jobID, prevStepIndex)。

Step 3: 超时回调 → SuspendJob

在 JobProcessor 中实现回调：

func (jp *JobProcessor) handleStepTimeout(jobID, stepIndex int) {
    slog.Warn("step timeout, suspending job", "jobId", jobID, "stepIndex", stepIndex)
    if err := jp.SuspendJob(context.Background(), jobID, "step_timeout"); err != nil {
        slog.Error("step timeout suspend failed", "jobId", jobID, "error", err)
    }
}

Step 4: 验证编译

Run: rtk go build ./internal/processor

Chunk 3: PLC-Redis 状态校验

Task 7: 周期性校验器

Files:

Create: internal/verify/state_verifier.go
Create: internal/verify/state_verifier_test.go
Step 1: 创建 internal/verify/state_verifier.go

package verify

import (
	"context"
	"log/slog"
	"time"

	"hougai/internal/alarm"
	"hougai/internal/plc"
	"hougai/internal/state"
)

// StateVerifier 周期性对比 PLC 物理状态与 Redis 逻辑状态。
// 设计文档 §14：不一致则挂起报警。
type StateVerifier struct {
	stateMgr   state.StateManager
	plcManager *plc.Manager
	alarmSvc   *alarm.Service
	interval   time.Duration
}

func NewStateVerifier(stateMgr state.StateManager, plcManager *plc.Manager, alarmSvc *alarm.Service, interval time.Duration) *StateVerifier {
	if interval <= 0 {
		interval = 30 * time.Second
	}
	return &StateVerifier{
		stateMgr:   stateMgr,
		plcManager: plcManager,
		alarmSvc:   alarmSvc,
		interval:   interval,
	}
}

// Run 启动周期性校验循环，直到 ctx 取消。
func (v *StateVerifier) Run(ctx context.Context) {
	ticker := time.NewTicker(v.interval)
	defer ticker.Stop()

	slog.Info("state verifier: started", "interval", v.interval)
	for {
		select {
		case <-ctx.Done():
			slog.Info("state verifier: stopped")
			return
		case <-ticker.C:
			v.verifyOnce(ctx)
		}
	}
}

func (v *StateVerifier) verifyOnce(ctx context.Context) {
	// 1. 获取暂存台 Bitmap（Redis 逻辑状态）
	logicalSlots, err := v.stateMgr.GetTempSlotBitmap(ctx)
	if err != nil {
		slog.Warn("state verifier: get logical slots failed", "error", err)
		return
	}

	// 2. 读取 PLC 暂存台传感器（物理状态）
	plc := v.plcManager.GetPlc()
	if plc == nil {
		return
	}
	physicalSlots, err := plc.ReadBufferSlotStates(ctx)
	if err != nil {
		slog.Warn("state verifier: read plc slots failed", "error", err)
		return
	}

	// 3. 对比
	if logicalSlots != physicalSlots {
		slog.Warn("state verifier: mismatch detected",
			"logical", logicalSlots,
			"physical", physicalSlots,
		)
		if v.alarmSvc != nil {
			v.alarmSvc.Raise(ctx,
				"STATE_MISMATCH",
				"PLC 暂存台状态与 Redis 逻辑状态不一致",
				"WARN",
				0, 0, "state_verifier",
			)
		}
	}
}

Step 2: 检查 PLC Manager 是否有 ReadBufferSlotStates

如果不存在，在 internal/plc/manager.go 或 internal/plc/ 中添加桩方法（返回错误），Phase 4 实现。

Step 3: 验证编译

Run: rtk go build ./internal/verify

Task 8: ServiceContext 启动校验器

Files:

Modify: internal/svc/service_context.go
Step 1: 启动 StateVerifier goroutine

在 SSOT 启用时启动：

if c.SSOT.Enabled && alarmSvc != nil {
    verifier := verify.NewStateVerifier(stateMgr, plcManager, alarmSvc, 30*time.Second)
    go verifier.Run(context.Background())
    slog.Info("svc: state verifier started")
}

Step 2: 全量编译

Run: rtk go build ./...

Chunk 4: 集成验证

Task 9: 全量编译 + 测试

Step 1: 全量编译

Run: rtk go build ./...

Step 2: 运行新增测试

Run: rtk go test ./internal/state/... ./internal/scheduler/... ./internal/processor/... ./internal/verify/...

Step 3: 运行全量测试

Run: rtk go test ./...

风险点

风险	缓解
工具锁 TTL 过短导致锁提前释放	使用 30s TTL + 执行前续租，或设置足够长的 TTL 覆盖最长工具操作
StepTimeoutWatcher timer 泄漏	CancelAll 在 Job 结束时清理，添加定时巡检
PLC ReadBufferSlotStates 不存在	添加桩方法返回错误，校验器跳过本轮
Scheduler 接口不兼容 ToolLockConstraint	使用 SetToolLockChecker 方法动态注入，保持向后兼容

验证标准

rtk go build ./... 通过
rtk go test ./... 通过（排除已知失败的 preload/camera/db/robot 包）
工具锁：同一工具类型不可并发获取
工序超时：超时后自动 SuspendJob
状态校验：PLC 不一致时触发 ALARM

14 KiB Raw Blame History Unescape Escape

Phase 3: 工具锁 + 工序超时 + 状态校验 实施计划

Chunk 1: 工具锁机制

Task 1: Redis 工具锁

Task 2: 调度器工具锁约束

Task 3: JobRuntime 工具锁集成（Acquire/Release 时机）

Task 4: ServiceContext 接线

Chunk 2: 工序超时机制

Task 5: StepTimeoutWatcher — 工序级超时监控

Task 6: 将 StepTimeoutWatcher 接入 JobRuntime

Chunk 3: PLC-Redis 状态校验

Task 7: 周期性校验器

Task 8: ServiceContext 启动校验器

Chunk 4: 集成验证

Task 9: 全量编译 + 测试

风险点

验证标准

14 KiB

Raw Blame History

Phase 3: 工具锁 + 工序超时 + 状态校验实施计划