Browse Source

feat: 火山方舟有自带的输入安全护栏:当待审文本本身命中敏感信息(如涉政),它会在模型处理前直接返回 InputTextSensitiveContentDetected 的 400 错误。证明内容违规

zhangwl 1 month ago
parent
commit
16c32f1ac4
1 changed files with 27 additions and 0 deletions
  1. 27 0
      app/routers/moderation.py

+ 27 - 0
app/routers/moderation.py

@@ -200,6 +200,23 @@ def _build_input_content(mod_type: str, request: ModerationRequest, client) -> l
     ]
     ]
 
 
 
 
+# 火山方舟输入/输出安全护栏拦截时返回的错误码统一含该关键字
+_SENSITIVE_ERROR_KEYWORD = "SensitiveContentDetected"
+
+
+def _is_sensitive_block(error: Exception) -> bool:
+    """判断异常是否为方舟安全护栏的敏感内容拦截。
+
+    当输入(文本/图片/视频)命中敏感信息时,方舟会在模型处理前直接返回
+    形如 InputTextSensitiveContentDetected 的 400 错误。对审核接口而言,
+    这本身即表明内容违规,应判定为 sensitive=true,而非当作服务异常报错。
+    """
+    code = getattr(error, "code", None)
+    if code and _SENSITIVE_ERROR_KEYWORD in str(code):
+        return True
+    return _SENSITIVE_ERROR_KEYWORD in str(error)
+
+
 def _log_target(mod_type: str, request: ModerationRequest) -> str:
 def _log_target(mod_type: str, request: ModerationRequest) -> str:
     """生成审核日志的目标标识(文本截断 / 媒体 URL)"""
     """生成审核日志的目标标识(文本截断 / 媒体 URL)"""
     if mod_type == "text":
     if mod_type == "text":
@@ -245,6 +262,16 @@ async def moderate_content(request: ModerationRequest):
             text={"format": {"type": "text"}},
             text={"format": {"type": "text"}},
         )
         )
     except Exception as e:
     except Exception as e:
+        # 护栏拦截即视为命中违规内容,直接返回 sensitive=true
+        if _is_sensitive_block(e):
+            save_chat_log(
+                user_id="moderation",
+                question=log_target,
+                stream_mode=False,
+                status="blocked",
+                error=str(e),
+            )
+            return ModerationResponse(sensitive=True)
         save_chat_log(
         save_chat_log(
             user_id="moderation",
             user_id="moderation",
             question=log_target,
             question=log_target,