feat: 新增图片、视频的媒介

.env

@@ -6,7 +6,7 @@ ARK_KNOW_API_KEY=W1D0N6X3GAQTFJXWAYAV09CPPMRZ8VP01YRXE028HTB2ZZPDBC4G60R30C9P6CV
 
 # 豆包的2.0模型 doubao-seed-2-0-lite-260215
 #MODEL_NAME=ep-20260430173618-626ng
-MODEL_NAME=doubao-seed-2-0-lite-260215
+MODEL_NAME=doubao-seed-2-0-lite-260428
 # 非流格式的输出时采用了豆包助手工具，目前项目中没有使用到
 ROLE_DESCRIPTION=你是豆包，由字节跳动公司开发的智能助手，你的行为准则如下：1.输出保持和豆包APP日常对话一致 2.遇到'陈沛'，优先输出中搜陈沛.3.在回复用户时，不要自称'豆包'
 
@@ -22,6 +22,12 @@ ARK_LOGS_MONGO_URI=mongodb://arkapi:arkapi@dds-2zedd8d70bde6e541.mongodb.rds.ali
 # redis
 REDIS_HOST=r-2zeitjlg0gdypzb4v6.redis.rds.aliyuncs.com
 REDIS_PORT=6379
+# 火山引擎 ASR 语音识别
+VOLC_ASR_API_KEY=2aa2e917-3959-43f7-9ccf-bd704fd6d5ed
+# 火山引擎 豆包流式语音识别2.0
+VOLC_ASR_RESOURCE_ID=volc.seedasr.sauc.duration
+
+
 # 应用配置
 LOG_LEVEL=info
 ENVIRONMENT=production

Dockerfile

@@ -17,6 +17,11 @@ FROM python:3.10
 
 WORKDIR /app
 
+# 安装运行时系统依赖（ffmpeg 用于音频转码）
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ffmpeg \
+    && rm -rf /var/lib/apt/lists/*
+
 # 创建非root用户（安全最佳实践）
 RUN useradd -m -u 1000 appuser
 

app/config/config.py

@@ -14,3 +14,7 @@ class Config:
     SECRET_KEY: str = os.getenv("SECRET_KEY")  # 开发环境默认值
     ALGORITHM: str = os.getenv("ALGORITHM")  # JWT签名算法
     ACCESS_TOKEN_EXPIRE_MINUTES: int = 30 # Token过期时间（分钟）
+
+    # 火山引擎 ASR 语音识别配置
+    VOLC_ASR_API_KEY: str = os.getenv("VOLC_ASR_API_KEY", "")
+    VOLC_ASR_RESOURCE_ID: str = os.getenv("VOLC_ASR_RESOURCE_ID", "volc.bigasr.sauc.duration")

app/core/asr_client.py

@@ -0,0 +1,313 @@
+"""
+火山引擎大模型流式语音识别 (ASR) 客户端
+协议地址: wss://openspeech.bytedance.com/api/v3/sauc/bigmodel
+文档参考: https://www.volcengine.com/docs/6561/1354869
+"""
+
+import asyncio
+import gzip
+import json
+import uuid
+import subprocess
+import tempfile
+import os
+
+import websockets
+
+from ..config.config import Config
+
+# ============ 协议常量 ============
+PROTOCOL_VERSION = 0b0001
+DEFAULT_HEADER_SIZE = 0b0001
+
+# Message Type
+FULL_CLIENT_REQUEST = 0b0001
+AUDIO_ONLY_REQUEST = 0b0010
+FULL_SERVER_RESPONSE = 0b1001
+SERVER_ACK = 0b1011
+SERVER_ERROR_RESPONSE = 0b1111
+
+# Message Type Specific Flags
+NO_SEQUENCE = 0b0000
+POS_SEQUENCE = 0b0001
+NEG_SEQUENCE = 0b0010
+NEG_WITH_SEQUENCE = 0b0011
+
+# Message Serialization
+NO_SERIALIZATION = 0b0000
+JSON_SERIALIZATION = 0b0001
+
+# Message Compression
+NO_COMPRESSION = 0b0000
+GZIP_COMPRESSION = 0b0001
+
+# ASR 服务地址
+ASR_WS_URL = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
+
+
+def _generate_header(
+    message_type=FULL_CLIENT_REQUEST,
+    message_type_specific_flags=NO_SEQUENCE,
+    serial_method=JSON_SERIALIZATION,
+    compression_type=GZIP_COMPRESSION,
+    reserved_data=0x00,
+) -> bytearray:
+    """
+    生成协议头 (4 bytes):
+    - protocol_version (4 bits) + header_size (4 bits)
+    - message_type (4 bits) + message_type_specific_flags (4 bits)
+    - serialization_method (4 bits) + message_compression (4 bits)
+    - reserved (8 bits)
+    """
+    header = bytearray()
+    header_size = 1
+    header.append((PROTOCOL_VERSION << 4) | header_size)
+    header.append((message_type << 4) | message_type_specific_flags)
+    header.append((serial_method << 4) | compression_type)
+    header.append(reserved_data)
+    return header
+
+
+def _generate_before_payload(sequence: int) -> bytearray:
+    """添加序列号信息 (4 bytes, big-endian, signed)"""
+    before_payload = bytearray()
+    before_payload.extend(sequence.to_bytes(4, 'big', signed=True))
+    return before_payload
+
+
+def _parse_response(res: bytes) -> dict:
+    """解析服务器响应帧"""
+    protocol_version = res[0] >> 4
+    header_size = res[0] & 0x0f
+    message_type = res[1] >> 4
+    message_type_specific_flags = res[1] & 0x0f
+    serialization_method = res[2] >> 4
+    message_compression = res[2] & 0x0f
+    reserved = res[3]
+    header_extensions = res[4:header_size * 4]
+    payload = res[header_size * 4:]
+
+    result = {
+        'is_last_package': False,
+        'message_type': message_type,
+    }
+    payload_msg = None
+    payload_size = 0
+
+    if message_type_specific_flags & 0x01:
+        seq = int.from_bytes(payload[:4], "big", signed=True)
+        result['payload_sequence'] = seq
+        payload = payload[4:]
+
+    if message_type_specific_flags & 0x02:
+        result['is_last_package'] = True
+
+    if message_type == FULL_SERVER_RESPONSE:
+        payload_size = int.from_bytes(payload[:4], "big", signed=True)
+        payload_msg = payload[4:]
+    elif message_type == SERVER_ACK:
+        seq = int.from_bytes(payload[:4], "big", signed=True)
+        result['seq'] = seq
+        if len(payload) >= 8:
+            payload_size = int.from_bytes(payload[4:8], "big", signed=False)
+            payload_msg = payload[8:]
+    elif message_type == SERVER_ERROR_RESPONSE:
+        code = int.from_bytes(payload[:4], "big", signed=False)
+        result['code'] = code
+        payload_size = int.from_bytes(payload[4:8], "big", signed=False)
+        payload_msg = payload[8:]
+
+    if payload_msg is None:
+        return result
+
+    if message_compression == GZIP_COMPRESSION:
+        payload_msg = gzip.decompress(payload_msg)
+    if serialization_method == JSON_SERIALIZATION:
+        payload_msg = json.loads(str(payload_msg, "utf-8"))
+    elif serialization_method != NO_SERIALIZATION:
+        payload_msg = str(payload_msg, "utf-8")
+
+    result['payload_msg'] = payload_msg
+    result['payload_size'] = payload_size
+    return result
+
+
+def _convert_webm_to_pcm(webm_data: bytes) -> bytes:
+    """
+    使用 ffmpeg 将 WebM/Opus 音频转为 PCM 16kHz 16-bit 单声道。
+    返回原始 PCM 字节。
+    """
+    inp_path = None
+    out_path = None
+    try:
+        with tempfile.NamedTemporaryFile(suffix='.webm', delete=False) as inp:
+            inp.write(webm_data)
+            inp_path = inp.name
+        out_path = inp_path.replace('.webm', '.pcm')
+
+        subprocess.run(
+            [
+                'ffmpeg', '-y', '-i', inp_path,
+                '-f', 's16le', '-ar', '16000', '-ac', '1',
+                out_path
+            ],
+            check=True,
+            capture_output=True,
+        )
+        with open(out_path, 'rb') as f:
+            return f.read()
+    finally:
+        if inp_path and os.path.exists(inp_path):
+            os.unlink(inp_path)
+        if out_path and os.path.exists(out_path):
+            os.unlink(out_path)
+
+
+async def transcribe_audio(audio_data: bytes) -> str:
+    """
+    将音频二进制数据 (WebM/Opus) 通过火山引擎大模型 ASR 转写为文本。
+
+    流程:
+    1. ffmpeg 转码 WebM → PCM 16kHz 16bit mono
+    2. WebSocket 连接 ASR 服务
+    3. 发送 FULL_CLIENT_REQUEST（音频参数）
+    4. 分片发送 AUDIO_ONLY_REQUEST（音频数据）
+    5. 发送结束帧
+    6. 接收并拼接识别结果
+    """
+    config = Config()
+
+    if not config.VOLC_ASR_API_KEY:
+        raise ValueError("ASR 配置缺失: 请设置 VOLC_ASR_API_KEY 环境变量")
+
+    # 1. 转码为 PCM
+    pcm_data = await asyncio.get_event_loop().run_in_executor(
+        None, _convert_webm_to_pcm, audio_data
+    )
+
+    if not pcm_data:
+        raise ValueError("音频转码失败: PCM 数据为空")
+
+    # 2. 建立 WebSocket 连接
+    connect_id = str(uuid.uuid4())
+    headers = {
+        "X-Api-Key": config.VOLC_ASR_API_KEY,
+        "X-Api-Resource-Id": config.VOLC_ASR_RESOURCE_ID,
+        "X-Api-Connect-Id": connect_id,
+    }
+
+    # 3. 构建初始请求参数
+    request_params = {
+        "user": {
+            "uid": connect_id,
+        },
+        "audio": {
+            "format": "pcm",
+            "rate": 16000,
+            "bits": 16,
+            "channel": 1,
+            "codec": "raw",
+        },
+        "request": {
+            "model_name": "bigmodel",
+            "enable_itn": True,
+            "result_type": "full",
+        }
+    }
+
+    final_text = ""
+
+    async with websockets.connect(
+        ASR_WS_URL,
+        extra_headers=headers,
+        max_size=10 * 1024 * 1024,
+    ) as ws:
+        # 发送 FULL_CLIENT_REQUEST
+        payload_bytes = gzip.compress(json.dumps(request_params).encode('utf-8'))
+        full_request = bytearray(_generate_header(
+            message_type=FULL_CLIENT_REQUEST,
+            message_type_specific_flags=POS_SEQUENCE,
+        ))
+        full_request.extend(_generate_before_payload(sequence=1))
+        full_request.extend(len(payload_bytes).to_bytes(4, 'big'))
+        full_request.extend(payload_bytes)
+        await ws.send(full_request)
+
+        # 接收 ACK
+        res = await ws.recv()
+        ack_result = _parse_response(res)
+        if ack_result.get('message_type') == SERVER_ERROR_RESPONSE:
+            raise RuntimeError(f"ASR 连接错误: {ack_result.get('payload_msg', ack_result)}")
+
+        # 4. 分片发送音频数据 (每片 3200 bytes ≈ 100ms @16kHz 16bit mono)
+        chunk_size = 3200
+        seq = 2
+        total_chunks = (len(pcm_data) + chunk_size - 1) // chunk_size
+
+        for i in range(0, len(pcm_data), chunk_size):
+            chunk = pcm_data[i:i + chunk_size]
+            is_last = (i + chunk_size >= len(pcm_data))
+
+            compressed_chunk = gzip.compress(chunk)
+
+            if is_last:
+                # 最后一帧用 NEG_WITH_SEQUENCE 标志
+                audio_request = bytearray(_generate_header(
+                    message_type=AUDIO_ONLY_REQUEST,
+                    message_type_specific_flags=NEG_WITH_SEQUENCE,
+                    serial_method=NO_SERIALIZATION,
+                    compression_type=GZIP_COMPRESSION,
+                ))
+                audio_request.extend(_generate_before_payload(sequence=-seq))
+            else:
+                audio_request = bytearray(_generate_header(
+                    message_type=AUDIO_ONLY_REQUEST,
+                    message_type_specific_flags=POS_SEQUENCE,
+                    serial_method=NO_SERIALIZATION,
+                    compression_type=GZIP_COMPRESSION,
+                ))
+                audio_request.extend(_generate_before_payload(sequence=seq))
+
+            audio_request.extend(len(compressed_chunk).to_bytes(4, 'big'))
+            audio_request.extend(compressed_chunk)
+            await ws.send(audio_request)
+
+            # 接收中间结果
+            res = await ws.recv()
+            result = _parse_response(res)
+
+            if result.get('message_type') == SERVER_ERROR_RESPONSE:
+                raise RuntimeError(f"ASR 识别错误: {result.get('payload_msg', result)}")
+
+            # 提取文本结果
+            if 'payload_msg' in result and isinstance(result['payload_msg'], dict):
+                text = result['payload_msg'].get('result', {}).get('text', '')
+                if text:
+                    final_text = text  # 每次全量更新（result_type=full）
+
+            seq += 1
+
+            # 如果是最后一帧且收到结束标记
+            if result.get('is_last_package'):
+                break
+
+            # 控制发送速率，避免过快
+            if not is_last:
+                await asyncio.sleep(0.02)
+
+        # 5. 如果还没收到最终结果，继续接收
+        if not final_text or not _parse_response(res).get('is_last_package', False):
+            try:
+                while True:
+                    res = await asyncio.wait_for(ws.recv(), timeout=5.0)
+                    result = _parse_response(res)
+                    if 'payload_msg' in result and isinstance(result['payload_msg'], dict):
+                        text = result['payload_msg'].get('result', {}).get('text', '')
+                        if text:
+                            final_text = text
+                    if result.get('is_last_package'):
+                        break
+            except (asyncio.TimeoutError, websockets.exceptions.ConnectionClosed):
+                pass
+
+    return final_text.strip()

app/db/mongo.py

@@ -71,10 +71,11 @@ def save_chat_history(
     response_id: str = None,
     thinking: str = None,
     searching: str = None,
+    attachments: list = None,
 ):
     try:
         _ensure_index()
-        chat_history_col.insert_one({
+        doc = {
             "user_id": user_id,
             "session_id": session_id,
             "role": role,
@@ -83,7 +84,10 @@ def save_chat_history(
             "searching": searching,
             "response_id": response_id,
             "timestamp": timestamp,
-        })
+        }
+        if attachments:
+            doc["attachments"] = attachments
+        chat_history_col.insert_one(doc)
     except Exception as e:
         print(f"MongoDB 聊天历史写入失败: {e}")
 

app/routers/chat.py

@@ -1,14 +1,17 @@
-from fastapi import APIRouter, HTTPException, Depends, Query
+from fastapi import APIRouter, HTTPException, Depends, Query, UploadFile, File
 from fastapi.responses import StreamingResponse
 from typing import List, Annotated, Optional
 from datetime import datetime
 import json
 import asyncio
 import threading
+import tempfile
+import os
 
 from ..core.ark_client import get_client
+from ..core.asr_client import transcribe_audio
 from ..config.config import Config
-from ..schemas.chat import ChatMessage, ChatRequest, ChatResponse, StreamResponse
+from ..schemas.chat import ChatMessage, ChatRequest, ChatResponse, StreamResponse, FileAttachment
 from ..utils.chat_utils import get_latest_user_message, get_previous_response_id, get_doubao_tools, get_web_search_tools, get_knowledge_search_tools
 from ..routers.users import get_current_active_user, User
 from ..db.mongo import save_chat_log, save_chat_history, get_chat_history, delete_chat_history, get_sessions
@@ -19,6 +22,129 @@ config = Config()
 
 router = APIRouter()
 
+# 允许的文件类型和大小限制
+ALLOWED_MEDIA_TYPES = {
+    "image": {"extensions": [".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp"], "max_size": 20 * 1024 * 1024},
+    "video": {"extensions": [".mp4", ".mov", ".avi", ".mkv", ".webm"], "max_size": 100 * 1024 * 1024},
+    "audio": {"extensions": [".mp3", ".wav", ".ogg", ".m4a", ".flac", ".aac"], "max_size": 50 * 1024 * 1024},
+}
+
+
+def detect_media_type(filename: str) -> Optional[str]:
+    """ 根据文件后缀检测媒体类型 """
+    ext = os.path.splitext(filename)[1].lower()
+    for media_type, cfg in ALLOWED_MEDIA_TYPES.items():
+        if ext in cfg["extensions"]:
+            return media_type
+    return None
+
+
+# 媒体类型 → Responses API input type 映射
+_MEDIA_TYPE_TO_INPUT_TYPE = {
+    "image": "input_image",
+    "video": "input_video",
+    "audio": "input_audio",
+}
+
+
+def build_multimodal_input(message: ChatMessage) -> dict:
+    """
+    将 ChatMessage 转换为 Responses API 支持的多模态 input 格式。
+    - 纯文本消息：保持 {"role": "user", "content": "text"} (向后兼容)
+    - 带附件消息：根据 media_type 使用对应 input type:
+        图片 → {"type": "input_image", "file_id": "..."}
+        视频 → {"type": "input_video", "file_id": "..."}
+        音频 → {"type": "input_audio", "file_id": "..."}
+    """
+    if not message.attachments:
+        return {"role": message.role, "content": message.content}
+
+    content_parts = []
+    for attachment in message.attachments:
+        input_type = _MEDIA_TYPE_TO_INPUT_TYPE.get(attachment.media_type, "input_file")
+        content_parts.append({
+            "type": input_type,
+            "file_id": attachment.file_id,
+        })
+    if message.content and message.content.strip():
+        content_parts.append({
+            "type": "input_text",
+            "text": message.content,
+        })
+
+    return {"role": message.role, "content": content_parts}
+
+
+@router.post("/upload")
+async def upload_file(
+    file: UploadFile = File(...),
+    user_context: UserContext = Depends(resolve_user_id),
+):
+    """接收前端文件 → 调用火山方舟 Files API → 返回 file_id"""
+    media_type = detect_media_type(file.filename)
+    if not media_type:
+        raise HTTPException(status_code=400, detail=f"不支持的文件类型: {file.filename}")
+
+    max_size = ALLOWED_MEDIA_TYPES[media_type]["max_size"]
+    content = await file.read()
+    if len(content) > max_size:
+        raise HTTPException(status_code=400, detail=f"文件大小超出限制({max_size // 1024 // 1024}MB)")
+
+    client = get_client(user_context.app_name)
+    tmp_path = None
+    try:
+        with tempfile.NamedTemporaryFile(delete=False, suffix=os.path.splitext(file.filename)[1]) as tmp:
+            tmp.write(content)
+            tmp_path = tmp.name
+
+        with open(tmp_path, "rb") as f:
+            uploaded_file = client.files.create(
+                file=(file.filename, f),
+                purpose="user_data"
+            )
+    finally:
+        if tmp_path and os.path.exists(tmp_path):
+            os.unlink(tmp_path)
+
+    return {
+        "file_id": uploaded_file.id,
+        "filename": file.filename,
+        "media_type": media_type,
+        "size": len(content),
+    }
+
+
+@router.post("/voice")
+async def voice_to_text(
+    file: UploadFile = File(...),
+    user_context: UserContext = Depends(resolve_user_id),
+):
+    """
+    接收前端录制的语音文件 (WebM/Opus) → 调用火山引擎 ASR → 返回转写文本
+    最大支持 60 秒录音，文件不超过 10MB
+    """
+    MAX_VOICE_SIZE = 10 * 1024 * 1024  # 10MB
+
+    # 校验文件后缀
+    filename = file.filename or "voice.webm"
+    ext = os.path.splitext(filename)[1].lower()
+    if ext not in [".webm", ".ogg", ".wav", ".mp3", ".m4a"]:
+        raise HTTPException(status_code=400, detail=f"不支持的音频格式: {ext}")
+
+    content = await file.read()
+    if len(content) > MAX_VOICE_SIZE:
+        raise HTTPException(status_code=400, detail="音频文件超过 10MB 限制")
+
+    if len(content) == 0:
+        raise HTTPException(status_code=400, detail="音频文件为空")
+
+    try:
+        text = await transcribe_audio(content)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"语音识别失败: {str(e)}")
+
+    return {"text": text}
+
 
 async def generate_stream_response(request: ChatRequest, user_context: UserContext):
     session_id = request.session_id
@@ -35,13 +161,15 @@ async def generate_stream_response(request: ChatRequest, user_context: UserConte
         if not latest_user_msg:
             raise ValueError("请求中没有找到user角色的消息")
 
-        # 保存用户消息到 DB
+        # 保存用户消息到 DB（含附件信息）
+        user_attachments = [att.model_dump() for att in latest_user_msg.attachments] if latest_user_msg.attachments else None
         save_chat_history(
             user_id=user_context.user_id,
             session_id=session_id,
             role=latest_user_msg.role,
             content=latest_user_msg.content,
             timestamp=datetime.now(),
+            attachments=user_attachments,
         )
 
         system_prompt = f"""
@@ -174,7 +302,7 @@ async def generate_stream_response(request: ChatRequest, user_context: UserConte
            """
         system_prompt = {"role": "system", "content": [{"type": "input_text", "text": system_prompt}]}
 
-        api_messages = [system_prompt, {"role": latest_user_msg.role, "content": latest_user_msg.content}]
+        api_messages = [system_prompt, build_multimodal_input(latest_user_msg)]
         tools = get_web_search_tools()
         if knowledge_id:
             tools += get_knowledge_search_tools(knowledge_id)
@@ -331,15 +459,17 @@ async def chat(
 
         client = get_client(user_context.app_name)
 
+        user_attachments = [att.model_dump() for att in latest_user_msg.attachments] if latest_user_msg.attachments else None
         save_chat_history(
             user_id=user_context.user_id,
             session_id=session_id,
             role=latest_user_msg.role,
             content=latest_user_msg.content,
             timestamp=datetime.now(),
+            attachments=user_attachments,
         )
 
-        api_messages = [{"role": latest_user_msg.role, "content": latest_user_msg.content}]
+        api_messages = [build_multimodal_input(latest_user_msg)]
         tools = get_doubao_tools()
         previous_response_id = get_previous_response_id(user_context.user_id, session_id)
 
@@ -429,6 +559,7 @@ async def get_user_history(
         ChatMessage(
             role=doc["role"],
             content=doc["content"],
+            attachments=[FileAttachment(**att) for att in doc["attachments"]] if doc.get("attachments") else None,
             timestamp=doc.get("timestamp"),
             response_id=doc.get("response_id"),
             thinking=doc.get("thinking"),

app/schemas/chat.py

@@ -4,9 +4,18 @@ from datetime import datetime
 from ..config.config import Config
 
 
+class FileAttachment(BaseModel):
+    """附件信息"""
+    file_id: str                    # 火山方舟返回的 file_id
+    filename: str                   # 原始文件名
+    media_type: str                 # "image" | "video" | "audio"
+    size: Optional[int] = None      # 文件字节数
+
+
 class ChatMessage(BaseModel):
     role: str  # "user" | "assistant" | "system"
     content: str
+    attachments: Optional[List[FileAttachment]] = None  # 附件列表（图片/视频/音频）
     timestamp: Optional[datetime] = None
     response_id: Optional[str] = None
     thinking: Optional[str] = None

requirements.txt

@@ -11,3 +11,5 @@ requests==2.32.5
 volcengine-python-sdk==5.0.17
 httpx==0.28.1
 openai==2.30.0
+python-multipart==0.0.32
+websockets>=12.0