服务层

@app.post("/v1/chat/completions")
async def chat(req: ChatRequest, raw: Request):
    prompt = apply_chat_template(req.messages, tokenizer)

    async def stream():
        try:
            async for token_text in engine.generate(prompt, req.sampling):
                if await raw.is_disconnected():
                    await engine.abort(req.id)      # free the KV blocks
                    return
                chunk = {"choices": [{"delta": {"content": token_text}}]}
                yield f"data: {json.dumps(chunk)}\n\n"

            yield "data: [DONE]\n\n"
        finally:
            # Runs on client disconnect, server shutdown and exceptions
            # alike. This is the line that stops the memory leak.
            await engine.abort(req.id)

    return StreamingResponse(stream(), media_type="text/event-stream")

class StopChecker:
    """Stop strings are text; generation is tokens. Buffer accordingly."""

    def __init__(self, stops: list[str]):
        self.stops = stops
        self.window = ""
        self.keep = max((len(s) for s in stops), default=0)

    def push(self, text: str) -> tuple[str, bool]:
        self.window += text
        for s in self.stops:
            if (i := self.window.find(s)) != -1:
                # Emit only what precedes the stop string, then finish.
                return self.window[:i], True

        if self.keep <= 1:                    # nothing can straddle a boundary
            safe, self.window = self.window, ""
            return safe, False

        # Hold back the last keep-1 chars: a stop string may span chunks.
        split = max(0, len(self.window) - (self.keep - 1))
        safe, self.window = self.window[:split], self.window[split:]
        return safe, False