|
|
@@ -5,7 +5,7 @@ from urllib.parse import urlparse
|
|
|
import requests
|
|
|
from fastapi import APIRouter, HTTPException
|
|
|
|
|
|
-from ..core.ark_client import get_client
|
|
|
+from ..core.ark_client import get_ark_client
|
|
|
from ..config.config import Config
|
|
|
from ..schemas.chat import ModerationRequest, ModerationResponse
|
|
|
from ..db.mongo import save_chat_log
|
|
|
@@ -21,39 +21,10 @@ MAX_IMAGE_SIZE = 20 * 1024 * 1024
|
|
|
_DOWNLOAD_TIMEOUT = (10, 60)
|
|
|
|
|
|
# 内容审核系统提示词:约束模型仅输出 true / false,降低解析歧义并提升响应速度
|
|
|
-_MODERATION_PROMPT = """你是一个严格、专业的内容安全审核员。请判断用户提供的内容(文本、图片或视频画面)是否包含违规信息。
|
|
|
-
|
|
|
-违规内容包括以下任一类别:
|
|
|
-1. 涉黄内容:色情、低俗、性暗示、裸露等;
|
|
|
-2. 暴力恐怖内容:血腥、暴力、恐怖主义、极端主义等;
|
|
|
-3. 涉政敏感内容:政治敏感、反动言论、危害国家安全等;
|
|
|
-4. 其他不良信息:违法犯罪、赌博、诈骗、毒品、辱骂攻击、封建迷信等;
|
|
|
-5. 广告推广内容:商业营销、引流推广、联系方式诱导、二维码等。
|
|
|
-
|
|
|
-判断规则:
|
|
|
-- 只要内容命中上述任意一类,即判定为违规;
|
|
|
-- 对图片/视频,需结合画面主体、文字、场景等综合判断;
|
|
|
-- 内容健康、正常、合规的判定为合规;
|
|
|
-- 仅依据所提供的内容判断,不做额外联想或补全。
|
|
|
-
|
|
|
-输出要求(务必严格遵守):
|
|
|
-- 只输出一个单词:true 或 false;
|
|
|
-- true 表示检测到违规内容,false 表示合规内容;
|
|
|
-- 禁止输出任何解释、标点、空格或其他多余字符。"""
|
|
|
-
|
|
|
-
|
|
|
-def _extract_text(response) -> str:
|
|
|
- """从 responses API 返回结果中提取文本内容"""
|
|
|
- result = ""
|
|
|
- for item in response.output:
|
|
|
- if hasattr(item, 'type') and item.type == 'message' and hasattr(item, 'content'):
|
|
|
- if isinstance(item.content, list):
|
|
|
- for content_item in item.content:
|
|
|
- if hasattr(content_item, 'text'):
|
|
|
- result += content_item.text
|
|
|
- else:
|
|
|
- result += str(item.content)
|
|
|
- return result
|
|
|
+_MODERATION_PROMPT = """你是严格的内容安全审核员。判断用户提供的内容(文本/图片/视频画面)是否违规。
|
|
|
+违规类别(命中任一即违规):1.涉黄、低俗、性暗示、裸露;2.暴力、恐怖、血腥、极端主义;3.涉政敏感、反动、危害国家安全;4.违法犯罪、赌博、诈骗、毒品、辱骂、封建迷信;5.广告营销、引流、联系方式、二维码。
|
|
|
+图片/视频结合画面主体、文字、场景综合判断;仅依据所给内容判断,不做联想或补全。
|
|
|
+只输出一个单词:true(违规)或 false(合规),禁止输出任何其他字符。"""
|
|
|
|
|
|
|
|
|
def _parse_moderation_result(text: str) -> bool:
|
|
|
@@ -74,6 +45,21 @@ def _parse_moderation_result(text: str) -> bool:
|
|
|
raise ValueError(f"无法解析审核结果: {text[:50]}")
|
|
|
|
|
|
|
|
|
+def _try_early_decide(accumulated: str) -> bool | None:
|
|
|
+ """流式场景下的提前判定:true/false 首字母不同(t vs f),
|
|
|
+ 读到第一个非空字符即可判定,无需等模型吐完整个单词。
|
|
|
+ 返回 None 表示暂无法判定(如首字符是空白/不明字符)。"""
|
|
|
+ stripped = accumulated.strip().lower()
|
|
|
+ if not stripped:
|
|
|
+ return None
|
|
|
+ first = stripped[0]
|
|
|
+ if first == "t":
|
|
|
+ return True
|
|
|
+ if first == "f":
|
|
|
+ return False
|
|
|
+ return None
|
|
|
+
|
|
|
+
|
|
|
def _resolve_ext(url: str, content: bytes, default: str) -> str:
|
|
|
"""推断文件扩展名:优先取 URL 路径后缀,其次按文件魔数嗅探,最后用默认值。
|
|
|
Files API 依据扩展名识别媒体类型,因此需给出正确后缀。"""
|
|
|
@@ -160,9 +146,9 @@ def _resolve_type(request: ModerationRequest) -> str:
|
|
|
raise HTTPException(status_code=400, detail="请提供待审核的内容(content / image_url / video_url)")
|
|
|
|
|
|
|
|
|
-def _build_input_content(mod_type: str, request: ModerationRequest, client) -> list:
|
|
|
- """根据审核类型构造 Responses API 多模态 input content,并做参数校验。
|
|
|
- 图片/视频统一走 Files API(file_id),避免第三方 URL 的 mimetype 校验问题。"""
|
|
|
+def _build_input_content(mod_type: str, request: ModerationRequest, client) -> str | list:
|
|
|
+ """根据审核类型构造 Chat Completions API 的 content 参数,并做参数校验。
|
|
|
+ 文本审核返回字符串,图片/视频审核返回多模态 content 数组。"""
|
|
|
if mod_type == "text":
|
|
|
content = (request.content or "").strip()
|
|
|
if not content:
|
|
|
@@ -172,16 +158,15 @@ def _build_input_content(mod_type: str, request: ModerationRequest, client) -> l
|
|
|
status_code=400,
|
|
|
detail=f"待审核文本超过长度限制({MAX_CONTENT_LENGTH} 字符),请分段提交",
|
|
|
)
|
|
|
- return [{"type": "input_text", "text": content}]
|
|
|
+ return content
|
|
|
|
|
|
if mod_type == "image":
|
|
|
image_url = (request.image_url or "").strip()
|
|
|
if not image_url:
|
|
|
raise HTTPException(status_code=400, detail="图片审核需提供 image_url")
|
|
|
- file_id = _upload_remote_to_ark(client, image_url, "image")
|
|
|
return [
|
|
|
- {"type": "input_image", "file_id": file_id},
|
|
|
- {"type": "input_text", "text": "请审核这张图片的画面内容是否违规。"},
|
|
|
+ {"type": "image_url", "image_url": {"url": image_url}},
|
|
|
+ {"type": "text", "text": "请审核这张图片的画面内容是否违规。"},
|
|
|
]
|
|
|
|
|
|
# video:对视频画面 + 封面图一起审核
|
|
|
@@ -194,9 +179,9 @@ def _build_input_content(mod_type: str, request: ModerationRequest, client) -> l
|
|
|
video_file_id = _upload_remote_to_ark(client, video_url, "video")
|
|
|
thumb_file_id = _upload_remote_to_ark(client, thumb_url, "image")
|
|
|
return [
|
|
|
- {"type": "input_video", "file_id": video_file_id},
|
|
|
- {"type": "input_image", "file_id": thumb_file_id},
|
|
|
- {"type": "input_text", "text": "请综合审核这段视频的画面及封面图内容是否违规。"},
|
|
|
+ {"type": "file", "file_id": video_file_id},
|
|
|
+ {"type": "file", "file_id": thumb_file_id},
|
|
|
+ {"type": "text", "text": "请综合审核这段视频的画面及封面图内容是否违规。"},
|
|
|
]
|
|
|
|
|
|
|
|
|
@@ -243,32 +228,55 @@ async def moderate_content(request: ModerationRequest):
|
|
|
log_target = _log_target(mod_type, request)
|
|
|
|
|
|
try:
|
|
|
- client = get_client()
|
|
|
+ client = get_ark_client()
|
|
|
except Exception as e:
|
|
|
raise HTTPException(status_code=503, detail=f"审核服务初始化失败: {str(e)}")
|
|
|
|
|
|
# 图片/视频会在此处下载远程资源并上传 Files API(可能抛 400/502)
|
|
|
input_content = _build_input_content(mod_type, request, client)
|
|
|
|
|
|
- system_msg = {"role": "system", "content": [{"type": "input_text", "text": _MODERATION_PROMPT}]}
|
|
|
+ system_msg = {"role": "system", "content": _MODERATION_PROMPT}
|
|
|
user_msg = {"role": "user", "content": input_content}
|
|
|
|
|
|
+ accumulated = ""
|
|
|
+ is_sensitive = None
|
|
|
try:
|
|
|
- response = client.responses.create(
|
|
|
+ stream = client.chat.completions.create(
|
|
|
model="doubao-seed-2-0-mini-260428",
|
|
|
- input=[system_msg, user_msg],
|
|
|
- stream=False,
|
|
|
- store=False, # 无状态审核,不保存上下文,提升性能
|
|
|
- thinking={"type": "disabled"}, # 关闭深度思考,降低简单审核任务耗时
|
|
|
- text={"format": {"type": "text"}},
|
|
|
+ messages=[system_msg, user_msg],
|
|
|
+ # service_tier="fast",
|
|
|
+ stream=True,
|
|
|
+ thinking={"type": "disabled"}, # 通过 extra_body 透传给火山方舟
|
|
|
+ response_format={"type": "text"}
|
|
|
)
|
|
|
+ try:
|
|
|
+ for event in stream:
|
|
|
+ # 豆包API使用标准的OpenAI兼容格式:event.choices[0].delta.content
|
|
|
+ # 检查是否命中内容过滤器(护栏拦截)
|
|
|
+ if hasattr(event, 'choices') and event.choices:
|
|
|
+ choice = event.choices[0]
|
|
|
+ # 如果 finish_reason 是 content_filter,说明内容被护栏拦截
|
|
|
+ if choice.finish_reason == 'content_filter':
|
|
|
+ is_sensitive = True
|
|
|
+ print("检测到内容过滤器拦截 (finish_reason=content_filter),判定为违规")
|
|
|
+ break
|
|
|
+
|
|
|
+ delta = choice.delta.content
|
|
|
+ if delta:
|
|
|
+ accumulated += delta
|
|
|
+ # true/false 首字母不同,读到第一个非空字符即可判定
|
|
|
+ is_sensitive = _try_early_decide(accumulated)
|
|
|
+ if is_sensitive is not None:
|
|
|
+ break
|
|
|
+ finally:
|
|
|
+ stream.close() # 提前判定后主动关闭连接,释放资源
|
|
|
except Exception as e:
|
|
|
# 护栏拦截即视为命中违规内容,直接返回 sensitive=true
|
|
|
if _is_sensitive_block(e):
|
|
|
save_chat_log(
|
|
|
user_id="moderation",
|
|
|
question=log_target,
|
|
|
- stream_mode=False,
|
|
|
+ stream_mode=True,
|
|
|
status="blocked",
|
|
|
error=str(e),
|
|
|
)
|
|
|
@@ -276,25 +284,26 @@ async def moderate_content(request: ModerationRequest):
|
|
|
save_chat_log(
|
|
|
user_id="moderation",
|
|
|
question=log_target,
|
|
|
- stream_mode=False,
|
|
|
+ stream_mode=True,
|
|
|
status="error",
|
|
|
error=str(e),
|
|
|
)
|
|
|
raise HTTPException(status_code=500, detail=f"内容审核服务异常: {str(e)}")
|
|
|
|
|
|
- raw_text = _extract_text(response)
|
|
|
- try:
|
|
|
- is_sensitive = _parse_moderation_result(raw_text)
|
|
|
- except ValueError as e:
|
|
|
- save_chat_log(
|
|
|
- user_id="moderation",
|
|
|
- question=log_target,
|
|
|
- stream_mode=False,
|
|
|
- raw_response=repr(response),
|
|
|
- status="error",
|
|
|
- error=str(e),
|
|
|
- )
|
|
|
- raise HTTPException(status_code=500, detail="内容审核结果解析失败,请重试")
|
|
|
+ if is_sensitive is None:
|
|
|
+ # 未能提前判定(如首字符异常),兜底解析已累积到的文本
|
|
|
+ try:
|
|
|
+ is_sensitive = _parse_moderation_result(accumulated)
|
|
|
+ except ValueError as e:
|
|
|
+ save_chat_log(
|
|
|
+ user_id="moderation",
|
|
|
+ question=log_target,
|
|
|
+ stream_mode=True,
|
|
|
+ raw_response=accumulated,
|
|
|
+ status="error",
|
|
|
+ error=str(e),
|
|
|
+ )
|
|
|
+ raise HTTPException(status_code=500, detail="内容审核结果解析失败,请重试")
|
|
|
|
|
|
return ModerationResponse(sensitive=is_sensitive)
|
|
|
|