Agent Harness

class InferenceBackend(Protocol):
    """后端可切换：vLLM / SGLang / TensorRT-LLM"""
    def generate(self, prompts: list[str], stop_tokens: list[str]) -> list[str]: ...
    def get_logprobs(self, prompts: list[str], completions: list[str]) -> list[float]: ...
    def update_weights(self, state_dict: dict) -> None: ...

class AgentEnv:
    """reset/step/close — 与 Gym 同构的 Agent 环境接口"""
    def reset(self, seed=None):
        # 加载任务样本 → skill 路由 → 构造 system prompt
        return messages_list, {"tools": schema, "sample_id": id}

    def step(self, action: str):
        # 解析 action → 执行工具 → 终止判断 → 计算 reward
        return observation, reward, terminated, truncated, info

class AsyncRolloutEngine:
    async def collect_batch(self, prompts, group_size=8):
        # 每个 prompt 展开 group_size 条轨迹
        for prompt in prompts:
            for _ in range(group_size):
                traj = Trajectory(prompt=prompt)
                await self.pending_inference.put(traj)
        # 双 worker 并行：推理 + 工具执行
        await asyncio.gather(
            self._inference_worker(),
            self._tool_worker()
        )

    async def _inference_worker(self):
        """批量合并推理请求，最大化 GPU throughput"""
        while self.active_trajectories:
            batch = self._collect_batch(max_size=32)
            responses = self.inference.generate(
                [t.current_prompt for t in batch],
                stop_tokens=["Observation:"]
            )
            for traj, resp in zip(batch, responses):
                action = parse_action(resp)
                if action is None or traj.is_terminal():
                    traj.finalize()
                else:
                    traj.add_step(resp, action)
                    await self.pending_tool.put(traj)

    async def _tool_worker(self):
        """并发执行工具调用，完成后送回推理队列"""
        while self.active_trajectories:
            traj = await self.pending_tool.get()
            observation = await self.env.execute(traj.last_action)
            traj.add_observation(observation)
            await self.pending_inference.put(traj)

class BaseTool:
    def get_schema(self) -> dict:      # JSON Schema 给 LLM 理解工具能力
    def validate(self, params) -> bool: # 参数校验（constrained decoding 兜底）
    def execute(self, params) -> str:   # 实际执行（API / CLI / Python 各自实现）
    def format_output(self, raw) -> str: # 结果标准化（截断 + 提取关键字段）

class SandboxPool:
    """预创建 N 个 sandbox 实例，按需分配/回收"""
    def __init__(self, pool_size=16, image='agent-tools:v1'):
        self.available = asyncio.Queue()
        for _ in range(pool_size):
            sb = Sandbox(image=image, timeout=90)
            self.available.put_nowait(sb)

    async def acquire(self) -> Sandbox:
        return await asyncio.wait_for(
            self.available.get(), timeout=30
        )

    async def release(self, sandbox: Sandbox):
        await sandbox.reset()      # 清理状态
        self.available.put_nowait(sandbox)

# Step 1: 按步归一化（同一步位置 G 条轨迹互比）
A_{i,t} = (r_{i,t} - μ_t) / (σ_t + 1e-8)

# Step 2: Generation Mask
mask_t[k] = 1  if token k ∈ {Thought, Action}    # LLM 生成
mask_t[k] = 0  if token k ∈ {Observation, System} # 环境/系统

# Step 3: Clipped Policy Ratio
ratio = π_θ(token|ctx) / π_ref(token|ctx)
clipped = clip(ratio, 1-ε, 1+ε)

# Step 4: Final Loss
L = -(1/N) Σᵢ Σ_t Σ_k [mask · min(ratio·A_{i,t}, clipped·A_{i,t})]
    + β · D_KL(π_θ ∥ π_ref)

Token:  [Thought: 我需要查询...] [Action: {"tool":"search"...}] [Observation: 找到3条结果...] [Thought: 根据结果...]
Mask:   [1  1  1  1  1  1  1 ]  [1  1  1  1  1  1  1  1  ] [0  0  0  0  0  0  0  0 ]  [1  1  1  1  1 ]
                                                              ↑ 环境返回，不计入 loss

Loss = -Σ(generation_mask × log_prob × advantage) / Σ(generation_mask)

skill_name: enzyme_catalysis
available_tools: [CLEAN, mmseqs2, UniProt_API]
max_rounds: 8
answer_format:
  required_fields: ["ec_number", "substrate", "product"]
audit_rules:
  - field: "ec_number"
    pattern: "^\\d+\\.\\d+\\.\\d+\\.\\d+$"
reward_weights:
  tool_accuracy: 0.3
  answer_f1: 0.4
  reasoning_quality: 0.3

score = α × semantic_similarity   # cosine(query_emb, memory_emb), α=0.65
      + β × lexical_overlap        # |query_words ∩ memory_words| / |query_words|, β=0.25
      + γ × importance             # 预设的记忆重要性分数, γ=0.10