init

mildoc_admin/.env

@@ -0,0 +1,40 @@
+# mildoc admin
+
+# # minio local
+MINIO_ENDPOINT=127.0.0.1:9000
+MINIO_ACCESS_KEY=minioadmin
+MINIO_SECRET_KEY=minioadmin
+MINIO_BUCKET=mildoc
+MINIO_REGION=minio
+MINIO_USE_VIRTUAL_HOST=false
+MINIO_USE_SSL=false
+
+
+# minio oss
+# MINIO_ENDPOINT=oss-cn-hangzhou.aliyuncs.com:443
+# MINIO_ACCESS_KEY=LTAI5tHYKMyo8fNScFHkaNUS
+# MINIO_SECRET_KEY=
+# MINIO_BUCKET=mildoc
+# MINIO_REGION=cn-hangzhou
+# MINIO_USE_VIRTUAL_HOST=true
+# MINIO_USE_SSL=true
+
+# milvus
+MILVUS_HOST=127.0.0.1
+MILVUS_PORT=19530
+MILVUS_USER=root
+MILVUS_PASSWORD=admin123
+MILVUS_DATABASE=mildoc
+MILVUS_COLLECTION=mildoc_collection
+MILVUS_INDEX_NAME=mildoc_index
+MILVUS_VECTOR_DIM=768
+
+# 管理后台配置
+ADMIN_USERNAME=admin
+ADMIN_PASSWORD=admin123
+
+# flask
+FLASK_DEBUG=True
+FLASK_SECRET_KEY=tXTCjeWx50yqcUERo6x1qCU
+FLASK_HOST=0.0.0.0
+FLASK_PORT=8870

mildoc_admin/.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+mildoc_admin.log
+
+

mildoc_admin/admin_app.py

@@ -0,0 +1,682 @@
+# -*- coding: utf-8 -*-
+from flask import Flask, render_template, request, redirect, url_for, flash, session, jsonify, abort
+from minio import Minio
+from pymilvus import MilvusClient
+import os
+from dotenv import load_dotenv
+from datetime import timezone
+from functools import wraps
+import pytz
+
+
+# 加载环境变量
+load_dotenv()
+
+
+
+# 管理员账号
+ADMIN_USERNAME = os.getenv('ADMIN_USERNAME')
+ADMIN_PASSWORD = os.getenv('ADMIN_PASSWORD')
+
+# Minio 配置
+MINIO_BUCKET = os.getenv('MINIO_BUCKET')
+MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT')
+MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY')
+MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY')
+MINIO_REGION = os.getenv('MINIO_REGION')
+MINIO_USE_VIRTUAL_HOST = os.getenv('MINIO_USE_VIRTUAL_HOST', 'false').lower() == 'true'
+MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true'
+
+# Milvus 配置
+MILVUS_HOST = os.getenv("MILVUS_HOST")
+MILVUS_PORT = os.getenv("MILVUS_PORT")
+MILVUS_USER = os.getenv("MILVUS_USER")
+MILVUS_PASSWORD = os.getenv("MILVUS_PASSWORD")
+MILVUS_DATABASE = os.getenv("MILVUS_DATABASE")
+MILVUS_COLLECTION = os.getenv("MILVUS_COLLECTION")
+MILVUS_INDEX_NAME = os.getenv("MILVUS_INDEX_NAME")
+
+
+
+
+app = Flask(__name__)
+app.secret_key = os.getenv('FLASK_SECRET_KEY', 'default-secret-key')
+
+
+@app.before_request
+def before_request():
+    """在每个请求之前执行域名白名单检查"""
+    app.logger.info(f"before_request: {request.url}")
+
+
+def _get_minio_client() -> Minio:
+    client = Minio(
+        endpoint=MINIO_ENDPOINT,
+        access_key=MINIO_ACCESS_KEY,
+        secret_key=MINIO_SECRET_KEY,
+        secure=MINIO_USE_SSL,
+        region=MINIO_REGION,
+    )
+
+    if MINIO_USE_VIRTUAL_HOST:
+        client.enable_virtual_style_endpoint()
+        
+    return client
+
+# 初始化 Minio 客户端
+minio_client = _get_minio_client()
+
+
+# 初始化 Milvus API
+milvus_client = MilvusClient(
+    uri=f"http://{MILVUS_HOST}:{MILVUS_PORT}",
+    token=f"{MILVUS_USER}:{MILVUS_PASSWORD}",
+    db_name=MILVUS_DATABASE
+)
+
+
+def convert_utc_to_local(utc_datetime, timezone_name='Asia/Shanghai'):
+    """
+    将UTC时间转换为本地时间
+    
+    Args:
+        utc_datetime: UTC时间对象（datetime）
+        timezone_name: 目标时区名称，默认为'Asia/Shanghai'（中国标准时间）
+    
+    Returns:
+        str: 格式化后的本地时间字符串（YYYY-MM-DD HH:MM:SS）
+    """
+    if not utc_datetime:
+        return ''
+    
+    try:
+        # 确保UTC时间有时区信息
+        if utc_datetime.tzinfo is None:
+            utc_time = utc_datetime.replace(tzinfo=timezone.utc)
+        else:
+            utc_time = utc_datetime
+        
+        # 转换为目标时区
+        local_tz = pytz.timezone(timezone_name)
+        local_time = utc_time.astimezone(local_tz)
+        
+        # 格式化为字符串
+        return local_time.strftime('%Y-%m-%d %H:%M:%S')
+    
+    except Exception as e:
+        app.logger.error(f"时区转换失败: {str(e)}")
+        # 如果转换失败，返回原始时间的字符串格式
+        return utc_datetime.strftime('%Y-%m-%d %H:%M:%S') if utc_datetime else ''
+
+def login_required(f):
+    """登录验证装饰器"""
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        if 'username' not in session:
+            return redirect(url_for('login'))
+        return f(*args, **kwargs)
+    return decorated_function
+
+@app.route('/')
+def index():
+    """首页，重定向到登录页面或文件浏览页面"""
+    if 'username' in session:
+        return redirect(url_for('file_browser'))
+    return redirect(url_for('login'))
+
+@app.route('/login', methods=['GET', 'POST'])
+def login():
+    """登录页面"""
+    if request.method == 'POST':
+        username = request.form['username']
+        password = request.form['password']
+        
+        if username == ADMIN_USERNAME and password == ADMIN_PASSWORD:
+            session['username'] = username
+            flash('登录成功！', 'success')
+            return redirect(url_for('file_browser'))
+        else:
+            flash('用户名或密码错误', 'error')
+    
+    return render_template('login.html')
+
+@app.route('/logout')
+def logout():
+    """退出登录"""
+    session.pop('username', None)
+    flash('已退出登录', 'info')
+    return redirect(url_for('login'))
+
+
+# 企业微信登录相关接口，暂时不使用
+#app.route('/wxwork/callback')
+
+#app.route('/wxwork/login')
+
+#app.route('/wxwork/verify', methods=['GET', 'POST'])
+
+#app.route('/api/wxwork/user')
+
+# 文件解析测试 API，暂时不使用
+#app.route('/api/file/<path:file_path>/parse-test', methods=['POST'])
+
+# 文件浏览页面
+@app.route('/files')
+@login_required
+def file_browser():
+    """文件浏览页面"""
+    path = request.args.get('path', '')
+    return render_template('file_browser.html', current_path=path)
+
+# 获取文件列表 API
+@app.route('/api/files')
+@login_required
+def api_files():
+    """获取文件列表 API"""
+    path = request.args.get('path', '')
+    
+    try:
+        # 测试 MinIO 连接
+        if not minio_client.bucket_exists(MINIO_BUCKET):
+            return jsonify({'error': f'桶 "{MINIO_BUCKET}" 不存在，请检查配置'}), 500
+        
+        # 获取 Minio 中的文件列表
+        # 构建搜索前缀
+        search_prefix = path if path == '' else (path if path.endswith('/') else path + '/')
+        
+        objects = minio_client.list_objects(MINIO_BUCKET, prefix=search_prefix, recursive=False)
+        
+        files = []
+        folders = set()
+        
+        for obj in objects:
+            # 跳过当前目录的标记对象
+            if obj.object_name == search_prefix.rstrip('/'):
+                continue
+            
+            # 计算相对路径
+            if search_prefix:
+                if not obj.object_name.startswith(search_prefix):
+                    continue
+                relative_path = obj.object_name[len(search_prefix):]
+            else:
+                relative_path = obj.object_name
+            
+            # 跳过空路径
+            if not relative_path:
+                continue
+            
+            if '/' in relative_path:
+                # 这是一个子目录中的对象，我们只需要目录名
+                folder_name = relative_path.split('/')[0]
+                if folder_name and folder_name not in folders:
+                    folders.add(folder_name)
+                    folder_path = path + '/' + folder_name if path else folder_name
+                    files.append({
+                        'name': folder_name,
+                        'path': folder_path,
+                        'type': 'folder',
+                        'size': 0,
+                        'modified': ''
+                    })
+            else:
+                # 这是当前目录下的文件
+                files.append({
+                    'name': relative_path,
+                    'path': obj.object_name,
+                    'type': 'file',
+                    'size': obj.size,
+                    'modified': convert_utc_to_local(obj.last_modified)
+                })
+        
+        return jsonify({'files': files})
+    except Exception as e:
+        app.logger.error(f"获取文件列表失败: {str(e)}")
+        if "SignatureDoesNotMatch" in str(e):
+            return jsonify({'error': 'MinIO 认证失败，请检查 ACCESS_KEY 和 SECRET_KEY 配置'}), 500
+        elif "Connection" in str(e):
+            return jsonify({'error': f'无法连接到 MinIO 服务器 ({os.getenv("ENDPOINT")})，请检查网络和地址配置'}), 500
+        else:
+            return jsonify({'error': f'获取文件列表失败: {str(e)}'}), 500
+
+# 文件详情页面
+@app.route('/file/<path:file_path>')
+@login_required
+def file_detail(file_path):
+    """文件详情页面"""
+    return render_template('file_detail.html', file_path=file_path)
+
+# 获取文件详情 API
+@app.route('/api/file/<path:file_path>')
+@login_required
+def api_file_detail(file_path):
+    """获取文件详情 API"""
+    try:
+        # URL 解码文件路径（处理可能的双重编码）
+        from urllib.parse import unquote
+        # 先尝试解码一次，如果结果仍然是编码格式，再解码一次
+        decoded_path = unquote(file_path)
+        if '%' in decoded_path:
+            decoded_path = unquote(decoded_path)
+        file_path = decoded_path
+        
+        app.logger.debug(f"原始路径: {file_path}")
+        app.logger.debug(f"解码后路径: {decoded_path}")
+        # 从 Minio 获取文件基本信息
+        try:
+            file_stat = minio_client.stat_object(MINIO_BUCKET, file_path)
+            
+            file_info = {
+                'doc_name': os.path.basename(file_path),
+                'doc_path_name': file_path,
+                'file_size': file_stat.size,
+                'file_md5': file_stat.etag.strip('"'),
+                'last_modified': convert_utc_to_local(file_stat.last_modified),
+                'indexed': False,
+                'index_time': '',
+                'chunks': []
+            }
+        except Exception as e:
+            app.logger.error(f"获取文件信息失败: {str(e)}")
+            if "SignatureDoesNotMatch" in str(e):
+                return jsonify({'error': 'MinIO 认证失败，请检查 ACCESS_KEY 和 SECRET_KEY 配置'}), 500
+            elif "NoSuchKey" in str(e) or "not found" in str(e).lower():
+                return jsonify({'error': f'文件不存在: {file_path}'}), 404
+            else:
+                return jsonify({'error': f'获取文件信息失败: {str(e)}'}), 500
+        
+        # 从 Milvus 查询文件索引信息
+        try:
+            milvus_client.load_collection(collection_name=MILVUS_COLLECTION)
+            filter_expr = f'doc_path_name == "{file_path}"'
+            
+            results = milvus_client.query(
+                collection_name=MILVUS_COLLECTION,
+                filter=filter_expr,
+                output_fields=["doc_name", "doc_path_name", "doc_type", "doc_md5", "doc_length", "content", "embedding_model"],
+                limit=100
+            )
+            
+            if results:
+                file_info['indexed'] = True
+                # 取第一条记录的基本信息
+                first_record = results[0]
+                file_info['doc_type'] = first_record.get('doc_type', '')
+                file_info['indexed_md5'] = first_record.get('doc_md5', '')
+                file_info['doc_length'] = first_record.get('doc_length', 0)
+                file_info['embedding_model'] = first_record.get('embedding_model', '')
+                
+                # 处理所有分片
+                for result in results:
+                    file_info['chunks'].append({
+                        'content': result.get('content', ''),
+                        'length': len(result.get('content', ''))
+                    })
+        except Exception as e:
+            print(f"查询 Milvus 失败: {e}")
+        
+        return jsonify(file_info)
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
+
+# 文件下载 API
+@app.route('/api/file/<path:file_path>/download')
+@login_required
+def api_download_file(file_path):
+    """文件下载 API"""
+    try:
+        # URL 解码文件路径（处理可能的双重编码）
+        from urllib.parse import unquote
+        decoded_path = unquote(file_path)
+        if '%' in decoded_path:
+            decoded_path = unquote(decoded_path)
+        file_path = decoded_path
+        
+        app.logger.info(f"下载文件: {file_path}")
+        
+        # 检查文件是否存在
+        try:
+            file_stat = minio_client.stat_object(MINIO_BUCKET, file_path)
+        except Exception as e:
+            if "NoSuchKey" in str(e) or "not found" in str(e).lower():
+                return jsonify({'error': f'文件不存在: {file_path}'}), 404
+            else:
+                return jsonify({'error': f'获取文件信息失败: {str(e)}'}), 500
+        
+        # 从 MinIO 获取文件
+        from flask import Response
+        import io
+        
+        try:
+            response = minio_client.get_object(MINIO_BUCKET, file_path)
+            file_data = response.data
+            
+            # 获取文件名
+            filename = os.path.basename(file_path)
+            
+            # 设置正确的 Content-Type
+            content_type = file_stat.content_type or 'application/octet-stream'
+            
+            # 对文件名进行URL编码以支持中文字符
+            from urllib.parse import quote
+            encoded_filename = quote(filename, safe='')
+            
+            # 创建响应
+            return Response(
+                file_data,
+                mimetype=content_type,
+                headers={
+                    'Content-Disposition': f'attachment; filename*=UTF-8\'\'{encoded_filename}',
+                    'Content-Length': str(len(file_data)),
+                    'Cache-Control': 'no-cache'
+                }
+            )
+            
+        except Exception as e:
+            app.logger.error(f"下载文件失败: {str(e)}")
+            return jsonify({'error': f'下载文件失败: {str(e)}'}), 500
+        finally:
+            if 'response' in locals():
+                response.close()
+                response.release_conn()
+        
+    except Exception as e:
+        app.logger.error(f"下载请求处理失败: {str(e)}")
+        return jsonify({'error': f'下载请求处理失败: {str(e)}'}), 500
+
+# 文件删除 API
+@app.route('/api/file/<path:file_path>/delete', methods=['DELETE'])
+@login_required
+def api_delete_file(file_path):
+    """文件删除 API"""
+    try:
+        # URL 解码文件路径（处理可能的双重编码）
+        from urllib.parse import unquote
+        decoded_path = unquote(file_path)
+        if '%' in decoded_path:
+            decoded_path = unquote(decoded_path)
+        file_path = decoded_path
+        
+        app.logger.info(f"删除文件: {file_path}")
+        
+        # 检查文件是否存在
+        try:
+            file_stat = minio_client.stat_object(MINIO_BUCKET, file_path)
+            file_name = os.path.basename(file_path)
+        except Exception as e:
+            if "NoSuchKey" in str(e) or "not found" in str(e).lower():
+                return jsonify({'error': f'文件不存在: {file_path}'}), 404
+            else:
+                return jsonify({'error': f'获取文件信息失败: {str(e)}'}), 500
+        
+        # 从 MinIO 删除文件
+        try:
+            minio_client.remove_object(MINIO_BUCKET, file_path)
+            app.logger.info(f"文件删除成功: {file_path}")
+            
+            # 如果文件在 Milvus 中有索引，也删除索引记录
+            try:
+                milvus_client.load_collection(collection_name=MILVUS_COLLECTION)
+                filter_expr = f'doc_path_name == "{file_path}"'
+                
+                # 查询是否存在索引记录
+                results = milvus_client.query(
+                    collection_name=MILVUS_COLLECTION,
+                    filter=filter_expr,
+                    output_fields=["id"],
+                    limit=1000
+                )
+                
+                if results:
+                    # 删除 Milvus 中的记录
+                    ids_to_delete = [str(result['id']) for result in results]
+                    milvus_client.delete(
+                        collection_name=MILVUS_COLLECTION,
+                        filter=f'id in {ids_to_delete}'
+                    )
+                    app.logger.info(f"已删除 Milvus 中的 {len(ids_to_delete)} 条记录")
+                
+            except Exception as e:
+                app.logger.warning(f"删除 Milvus 索引时出错 (文件已删除): {str(e)}")
+            
+            return jsonify({
+                'success': True,
+                'message': f'文件 "{file_name}" 删除成功',
+                'file_name': file_name,
+                'file_path': file_path
+            })
+            
+        except Exception as e:
+            app.logger.error(f"删除文件失败: {str(e)}")
+            return jsonify({'error': f'删除文件失败: {str(e)}'}), 500
+        
+    except Exception as e:
+        app.logger.error(f"删除请求处理失败: {str(e)}")
+        return jsonify({'error': f'删除请求处理失败: {str(e)}'}), 500
+
+# 创建目录 API
+@app.route('/api/create-directory', methods=['POST'])
+@login_required
+def api_create_directory():
+    """创建目录 API"""
+    try:
+        data = request.get_json()
+        if not data:
+            return jsonify({'success': False, 'error': '无效的请求数据'}), 400
+        
+        current_path = data.get('path', '')
+        dir_name = data.get('name', '').strip()
+        
+        # 验证目录名称
+        if not dir_name:
+            return jsonify({'success': False, 'error': '目录名称不能为空'}), 400
+        
+        # 检查非法字符
+        import re
+        if re.search(r'[/\\:*?"<>|]', dir_name):
+            return jsonify({'success': False, 'error': '目录名称不能包含以下字符: / \\ : * ? " < > |'}), 400
+        
+        # 构建完整路径
+        if current_path:
+            full_path = current_path.rstrip('/') + '/' + dir_name + '/'
+        else:
+            full_path = dir_name + '/'
+        
+        app.logger.info(f"创建目录: {full_path}")
+        
+        # 检查目录是否已存在
+        try:
+            minio_client.stat_object(MINIO_BUCKET, full_path)
+            return jsonify({'success': False, 'error': '目录已存在'}), 400
+        except Exception:
+            # 目录不存在，可以创建
+            pass
+        
+        # 创建目录（通过上传一个空对象）
+        from io import BytesIO
+        minio_client.put_object(
+            MINIO_BUCKET,
+            full_path,
+            BytesIO(b''),
+            0,
+            content_type='application/x-directory'
+        )
+        
+        app.logger.info(f"目录创建成功: {full_path}")
+        return jsonify({'success': True, 'message': '目录创建成功'})
+        
+    except Exception as e:
+        app.logger.error(f"创建目录失败: {str(e)}")
+        return jsonify({'success': False, 'error': f'创建目录失败: {str(e)}'}), 500
+
+# 删除目录 API
+@app.route('/api/delete-directory', methods=['DELETE'])
+@login_required
+def api_delete_directory():
+    """删除目录 API"""
+    try:
+        data = request.get_json()
+        if not data:
+            return jsonify({'success': False, 'error': '无效的请求数据'}), 400
+        
+        dir_path = data.get('path', '').strip()
+        
+        # 验证路径
+        if not dir_path:
+            return jsonify({'success': False, 'error': '目录路径不能为空'}), 400
+        
+        # 确保路径以 / 结尾
+        if not dir_path.endswith('/'):
+            dir_path += '/'
+        
+        app.logger.info(f"删除目录: {dir_path}")
+        
+        # 检查目录是否存在
+        try:
+            minio_client.stat_object(MINIO_BUCKET, dir_path)
+        except Exception:
+            return jsonify({'success': False, 'error': '目录不存在'}), 404
+        
+        # 检查目录是否为空（不包含任何文件或子目录）
+        objects = list(minio_client.list_objects(MINIO_BUCKET, prefix=dir_path, recursive=True))
+        
+        # 过滤掉目录本身
+        content_objects = [obj for obj in objects if obj.object_name != dir_path]
+        
+        if content_objects:
+            return jsonify({'success': False, 'error': '目录不为空，无法删除'}), 400
+        
+        # 删除目录
+        minio_client.remove_object(MINIO_BUCKET, dir_path)
+        
+        app.logger.info(f"目录删除成功: {dir_path}")
+        return jsonify({'success': True, 'message': '目录删除成功'})
+        
+    except Exception as e:
+        app.logger.error(f"删除目录失败: {str(e)}")
+        return jsonify({'success': False, 'error': f'删除目录失败: {str(e)}'}), 500
+
+# 上传文件 API
+@app.route('/api/upload', methods=['POST'])
+@login_required
+def api_upload_files():
+    """文件上传 API"""
+    try:
+        # 获取上传路径
+        upload_path = request.form.get('path', '')
+        
+        # 检查是否有文件
+        if 'files' not in request.files:
+            return jsonify({'error': '没有选择文件'}), 400
+        
+        files = request.files.getlist('files')
+        if not files or all(f.filename == '' for f in files):
+            return jsonify({'error': '没有选择有效的文件'}), 400
+        
+        app.logger.info(f"开始上传 {len(files)} 个文件到路径: {upload_path}")
+        
+        uploaded_files = []
+        failed_files = []
+        
+        for file in files:
+            if file.filename == '':
+                continue
+                
+            try:
+                # 构建完整的对象路径
+                if upload_path:
+                    object_name = f"{upload_path}/{file.filename}"
+                else:
+                    object_name = file.filename
+                
+                # 检查文件是否已存在
+                try:
+                    minio_client.stat_object(MINIO_BUCKET, object_name)
+                    failed_files.append({
+                        'filename': file.filename,
+                        'error': '文件已存在'
+                    })
+                    continue
+                except:
+                    # 文件不存在，可以上传
+                    pass
+                
+                # 上传文件到 MinIO
+                file.seek(0)  # 重置文件指针
+                file_data = file.read()
+                file_size = len(file_data)
+                
+                # 检查文件大小限制 (500MB)
+                max_size = 500 * 1024 * 1024
+                if file_size > max_size:
+                    failed_files.append({
+                        'filename': file.filename,
+                        'error': f'文件过大 ({file_size / 1024 / 1024:.2f} MB > 500 MB)'
+                    })
+                    continue
+                
+                # 重置文件指针并上传
+                file.seek(0)
+                minio_client.put_object(
+                    MINIO_BUCKET,
+                    object_name,
+                    file,
+                    file_size,
+                    content_type=file.content_type or 'application/octet-stream'
+                )
+                
+                uploaded_files.append({
+                    'filename': file.filename,
+                    'object_name': object_name,
+                    'size': file_size
+                })
+                
+                app.logger.info(f"文件上传成功: {object_name}")
+                
+            except Exception as e:
+                app.logger.error(f"上传文件 {file.filename} 失败: {str(e)}")
+                failed_files.append({
+                    'filename': file.filename,
+                    'error': str(e)
+                })
+        
+        # 构造返回结果
+        result = {
+            'success': len(uploaded_files) > 0,
+            'uploaded_count': len(uploaded_files),
+            'failed_count': len(failed_files),
+            'uploaded_files': uploaded_files,
+            'failed_files': failed_files,
+            'message': f'成功上传 {len(uploaded_files)} 个文件'
+        }
+        
+        if failed_files:
+            result['message'] += f'，{len(failed_files)} 个文件上传失败'
+        
+        return jsonify(result)
+        
+    except Exception as e:
+        app.logger.error(f"文件上传处理失败: {str(e)}")
+        return jsonify({'error': f'文件上传处理失败: {str(e)}'}), 500
+
+
+
+if __name__ == '__main__':
+    
+    # 获取主机和端口配置
+    host = os.getenv('FLASK_HOST')
+    port = int(os.getenv('FLASK_PORT'))
+
+
+    print("=" * 60)
+    print("Mildoc 管理后台启动中...")
+    print(f"访问端口: {port}")
+    print("=" * 60)
+    
+    app.run(
+        host=host,
+        port=port,
+        debug=os.getenv('FLASK_DEBUG', 'False').lower() == 'true',
+        threaded=True,
+        processes=1
+    ) 

mildoc_admin/requirements.txt

@@ -0,0 +1,15 @@
+# Minio对象存储客户端
+minio==7.2.20
+
+# Milvus向量数据库客户端
+pymilvus==2.6.5
+
+# 环境变量管理
+python-dotenv==1.2.1
+
+# 日期处理
+pytz==2025.2
+
+# Flask Web框架
+Flask==3.1.2
+Flask-Login==0.6.3

mildoc_admin/run_mildoc_admin

@@ -0,0 +1,9 @@
+#!/bin/bash
+echo "Mildoc Admin Starting... "
+
+uv sync
+
+nohup uv run gunicorn --workers 1 --bind 0.0.0.0:8870 admin_app:app >> mildoc_admin.log 2>&1 &
+
+# 调试运行
+# uv run gunicorn --workers 1 --bind 0.0.0.0:8870 admin_app:app

mildoc_admin/static/js/file_browser.js

@@ -0,0 +1,585 @@
+// 文件浏览器功能
+class FileBrowser {
+    constructor() {
+        this.currentPath = new URLSearchParams(window.location.search).get('path') || '';
+        this.uploadingFiles = new Map(); // 存储正在上传的文件
+        this.init();
+    }
+
+    init() {
+        this.loadFiles();
+        this.updateBreadcrumb();
+        this.bindEvents();
+    }
+
+    // 绑定事件
+    bindEvents() {
+        // 上传按钮事件
+        document.getElementById('upload-btn').addEventListener('click', () => {
+            this.showUploadDialog();
+        });
+
+        // 文件选择事件
+        document.getElementById('file-input').addEventListener('change', (e) => {
+            this.handleFileSelect(e);
+        });
+
+        // 创建目录按钮事件
+        document.getElementById('create-dir-btn').addEventListener('click', () => {
+            this.showCreateDirectoryDialog();
+        });
+
+        // 删除目录按钮事件
+        document.getElementById('delete-dir-btn').addEventListener('click', () => {
+            this.showDeleteDirectoryDialog();
+        });
+    }
+
+    // 加载文件列表
+    loadFiles() {
+        const contentElement = document.getElementById('file-list-content');
+        Utils.showLoading(contentElement);
+
+        const url = `/api/files?path=${encodeURIComponent(this.currentPath)}`;
+        
+        Utils.ajax(url)
+            .then(data => {
+                this.renderFiles(data.files);
+            })
+            .catch(error => {
+                Utils.showError(contentElement, error.message);
+            });
+    }
+
+    // 渲染文件列表
+    renderFiles(files) {
+        const contentElement = document.getElementById('file-list-content');
+        
+        // 检查是否显示删除目录按钮
+        this.updateDeleteDirectoryButton(files);
+        
+        if (files.length === 0) {
+            contentElement.innerHTML = '<div class="loading">此目录为空</div>';
+            return;
+        }
+
+        let html = '';
+        
+        // 如果不在根目录，显示返回上级目录
+        if (this.currentPath) {
+            const parentPath = this.currentPath.substring(0, this.currentPath.lastIndexOf('/'));
+            html += `
+                <div class="file-item" onclick="fileBrowser.navigateToPath('${parentPath}')">
+                    <div class="file-item-name">
+                        <span class="file-icon folder">📁</span>
+                        <span>..</span>
+                    </div>
+                    <div class="file-item-size">-</div>
+                    <div class="file-item-modified">-</div>
+                </div>
+            `;
+        }
+
+        files.forEach(file => {
+            const icon = file.type === 'folder' ? '📁' : '📄';
+            const size = file.type === 'folder' ? '-' : Utils.formatFileSize(file.size);
+            const onclick = file.type === 'folder' 
+                ? `fileBrowser.navigateToPath('${file.path}')` 
+                : `fileBrowser.viewFile('${file.path}')`;
+
+            html += `
+                <div class="file-item" onclick="${onclick}">
+                    <div class="file-item-name">
+                        <span class="file-icon ${file.type}">${icon}</span>
+                        <span>${file.name}</span>
+                    </div>
+                    <div class="file-item-size">${size}</div>
+                    <div class="file-item-modified">${file.modified}</div>
+                </div>
+            `;
+        });
+
+        contentElement.innerHTML = html;
+    }
+
+    // 导航到指定路径
+    navigateToPath(path) {
+        const url = new URL(window.location);
+        if (path) {
+            url.searchParams.set('path', path);
+        } else {
+            url.searchParams.delete('path');
+        }
+        window.location.href = url.toString();
+    }
+
+    // 查看文件详情
+    viewFile(filePath) {
+        // 对路径进行正确的编码
+        const encodedPath = filePath.split('/').map(part => encodeURIComponent(part)).join('/');
+        window.location.href = `/file/${encodedPath}`;
+    }
+
+    // 更新面包屑导航
+    updateBreadcrumb() {
+        const breadcrumbPath = document.getElementById('breadcrumb-path');
+        
+        if (!this.currentPath) {
+            breadcrumbPath.innerHTML = '';
+            return;
+        }
+
+        const parts = this.currentPath.split('/').filter(part => part);
+        let html = '';
+        let currentPath = '';
+
+        parts.forEach((part, index) => {
+            currentPath += (currentPath ? '/' : '') + part;
+            
+            if (index === parts.length - 1) {
+                html += ` / <span class="breadcrumb-item">${part}</span>`;
+            } else {
+                html += ` / <span class="breadcrumb-item">
+                    <a href="?path=${encodeURIComponent(currentPath)}" class="breadcrumb-link">${part}</a>
+                </span>`;
+            }
+        });
+
+        breadcrumbPath.innerHTML = html;
+    }
+
+    // 显示上传对话框
+    showUploadDialog() {
+        document.getElementById('file-input').click();
+    }
+
+    // 处理文件选择
+    handleFileSelect(event) {
+        const files = Array.from(event.target.files);
+        if (files.length === 0) return;
+
+        // 重置文件选择器
+        event.target.value = '';
+
+        // 开始上传文件
+        this.uploadFiles(files);
+    }
+
+    // 上传文件
+    async uploadFiles(files) {
+        const formData = new FormData();
+        
+        // 添加当前路径
+        formData.append('path', this.currentPath);
+        
+        // 添加所有文件
+        files.forEach(file => {
+            formData.append('files', file);
+        });
+
+        // 在列表中显示上传中的文件
+        this.showUploadingFiles(files);
+
+        try {
+            // 发送上传请求
+            const response = await fetch('/api/upload', {
+                method: 'POST',
+                body: formData
+            });
+
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+
+            const result = await response.json();
+            
+            // 处理上传结果
+            this.handleUploadResult(result);
+
+        } catch (error) {
+            console.error('上传失败:', error);
+            this.handleUploadError(error.message);
+        }
+    }
+
+    // 在列表中显示上传中的文件
+    showUploadingFiles(files) {
+        const contentElement = document.getElementById('file-list-content');
+        
+        files.forEach(file => {
+            const fileId = 'upload-' + Date.now() + '-' + Math.random().toString(36).substr(2, 9);
+            this.uploadingFiles.set(fileId, file);
+
+            const uploadItem = document.createElement('div');
+            uploadItem.className = 'file-item uploading';
+            uploadItem.id = fileId;
+            uploadItem.innerHTML = `
+                <div class="file-item-name">
+                    <span class="file-icon file">📄</span>
+                    <span>${file.name}</span>
+                    <span class="upload-status">上传中...</span>
+                </div>
+                <div class="file-item-size">${Utils.formatFileSize(file.size)}</div>
+                <div class="file-item-modified">
+                    <div class="upload-progress">
+                        <div class="progress-bar">
+                            <div class="progress-fill"></div>
+                        </div>
+                    </div>
+                </div>
+            `;
+
+            // 添加到列表顶部（在返回上级目录项之后）
+            const firstFileItem = contentElement.querySelector('.file-item:not(.uploading)');
+            if (firstFileItem && firstFileItem.textContent.includes('..')) {
+                firstFileItem.insertAdjacentElement('afterend', uploadItem);
+            } else {
+                contentElement.insertBefore(uploadItem, contentElement.firstChild);
+            }
+
+            // 模拟进度条动画
+            setTimeout(() => {
+                const progressFill = uploadItem.querySelector('.progress-fill');
+                if (progressFill) {
+                    progressFill.style.width = '90%';
+                }
+            }, 100);
+        });
+    }
+
+    // 处理上传结果
+    handleUploadResult(result) {
+        // 移除上传中的文件项
+        this.uploadingFiles.forEach((file, fileId) => {
+            const uploadItem = document.getElementById(fileId);
+            if (uploadItem) {
+                uploadItem.remove();
+            }
+        });
+        this.uploadingFiles.clear();
+
+        // 显示上传结果
+        this.showUploadResultDialog(result);
+    }
+
+    // 处理上传错误
+    handleUploadError(errorMessage) {
+        // 移除上传中的文件项并显示错误
+        this.uploadingFiles.forEach((file, fileId) => {
+            const uploadItem = document.getElementById(fileId);
+            if (uploadItem) {
+                const statusElement = uploadItem.querySelector('.upload-status');
+                const progressElement = uploadItem.querySelector('.upload-progress');
+                
+                if (statusElement) statusElement.textContent = '上传失败';
+                if (progressElement) progressElement.innerHTML = '<span class="error-text">❌</span>';
+                
+                uploadItem.classList.add('upload-error');
+                
+                // 3秒后移除
+                setTimeout(() => {
+                    uploadItem.remove();
+                }, 3000);
+            }
+        });
+        this.uploadingFiles.clear();
+
+        // 显示错误消息
+        this.showUploadResultDialog({
+            success: false,
+            uploaded_count: 0,
+            failed_count: this.uploadingFiles.size,
+            message: errorMessage,
+            uploaded_files: [],
+            failed_files: []
+        });
+    }
+
+    // 显示上传结果对话框
+    showUploadResultDialog(result) {
+        const modalHtml = `
+            <div id="upload-result-modal" class="modal-overlay">
+                <div class="modal-content upload-result-modal">
+                    <div class="modal-header">
+                        <h3>上传结果</h3>
+                    </div>
+                    <div class="modal-body">
+                        <div class="upload-result ${result.success ? 'success' : 'error'}">
+                            <div class="result-icon">${result.success ? '✅' : '❌'}</div>
+                            <div class="result-message">
+                                <p><strong>${result.message}</strong></p>
+                                ${result.uploaded_count > 0 ? `<p>成功上传: ${result.uploaded_count} 个文件</p>` : ''}
+                                ${result.failed_count > 0 ? `<p>上传失败: ${result.failed_count} 个文件</p>` : ''}
+                            </div>
+                        </div>
+                        
+                        ${result.failed_files && result.failed_files.length > 0 ? `
+                            <div class="failed-files">
+                                <h4>失败文件详情:</h4>
+                                <ul>
+                                    ${result.failed_files.map(file => 
+                                        `<li><strong>${file.filename}:</strong> ${file.error}</li>`
+                                    ).join('')}
+                                </ul>
+                            </div>
+                        ` : ''}
+                    </div>
+                    <div class="modal-footer">
+                        <button onclick="fileBrowser.closeUploadResultDialog()" class="btn btn-primary">确认</button>
+                    </div>
+                </div>
+            </div>
+        `;
+
+        document.body.insertAdjacentHTML('beforeend', modalHtml);
+    }
+
+    // 关闭上传结果对话框
+    closeUploadResultDialog() {
+        const modal = document.getElementById('upload-result-modal');
+        if (modal) {
+            modal.remove();
+        }
+        
+        // 刷新文件列表
+        this.loadFiles();
+    }
+
+    // 更新删除目录按钮的显示状态
+    updateDeleteDirectoryButton(files) {
+        const deleteBtn = document.getElementById('delete-dir-btn');
+        
+        // 只有在非根目录且目录为空时才显示删除按钮
+        if (this.currentPath && files.length === 0) {
+            deleteBtn.style.display = 'inline-block';
+        } else {
+            deleteBtn.style.display = 'none';
+        }
+    }
+
+    // 显示创建目录对话框
+    showCreateDirectoryDialog() {
+        const modalHtml = `
+            <div id="create-dir-modal" class="modal-overlay">
+                <div class="modal-content">
+                    <div class="modal-header">
+                        <h3>创建目录</h3>
+                        <button onclick="fileBrowser.closeCreateDirectoryDialog()" class="modal-close">×</button>
+                    </div>
+                    <div class="modal-body">
+                        <div class="form-group">
+                            <label class="form-label">目录名称:</label>
+                            <input type="text" id="dir-name-input" class="form-input" placeholder="请输入目录名称" maxlength="255">
+                            <small style="color: #666; margin-top: 5px; display: block;">
+                                目录名称不能包含以下字符: / \\ : * ? " < > |
+                            </small>
+                        </div>
+                    </div>
+                    <div class="modal-footer">
+                        <button onclick="fileBrowser.closeCreateDirectoryDialog()" class="btn btn-cancel">取消</button>
+                        <button onclick="fileBrowser.createDirectory()" class="btn btn-primary" id="create-dir-confirm">创建</button>
+                    </div>
+                </div>
+            </div>
+        `;
+
+        document.body.insertAdjacentHTML('beforeend', modalHtml);
+        
+        // 聚焦到输入框
+        setTimeout(() => {
+            const input = document.getElementById('dir-name-input');
+            if (input) {
+                input.focus();
+                // 回车键确认
+                input.addEventListener('keypress', (e) => {
+                    if (e.key === 'Enter') {
+                        this.createDirectory();
+                    }
+                });
+            }
+        }, 100);
+    }
+
+    // 关闭创建目录对话框
+    closeCreateDirectoryDialog() {
+        const modal = document.getElementById('create-dir-modal');
+        if (modal) {
+            modal.remove();
+        }
+    }
+
+    // 创建目录
+    async createDirectory() {
+        const input = document.getElementById('dir-name-input');
+        const confirmBtn = document.getElementById('create-dir-confirm');
+        
+        if (!input) return;
+        
+        const dirName = input.value.trim();
+        
+        // 验证目录名称
+        if (!dirName) {
+            alert('请输入目录名称');
+            input.focus();
+            return;
+        }
+        
+        // 检查非法字符
+        const invalidChars = /[\/\\:*?"<>|]/;
+        if (invalidChars.test(dirName)) {
+            alert('目录名称不能包含以下字符: / \\ : * ? " < > |');
+            input.focus();
+            return;
+        }
+        
+        // 禁用按钮，防止重复提交
+        confirmBtn.disabled = true;
+        confirmBtn.textContent = '创建中...';
+        
+        try {
+            const response = await fetch('/api/create-directory', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                    path: this.currentPath,
+                    name: dirName
+                })
+            });
+            
+            const result = await response.json();
+            
+            if (response.ok && result.success) {
+                // 成功创建
+                this.closeCreateDirectoryDialog();
+                this.loadFiles(); // 刷新文件列表
+                
+                // 显示成功消息
+                this.showMessage('目录创建成功', 'success');
+            } else {
+                // 创建失败
+                alert(result.error || '创建目录失败');
+                confirmBtn.disabled = false;
+                confirmBtn.textContent = '创建';
+                input.focus();
+            }
+        } catch (error) {
+            console.error('创建目录失败:', error);
+            alert('创建目录失败: ' + error.message);
+            confirmBtn.disabled = false;
+            confirmBtn.textContent = '创建';
+        }
+    }
+
+    // 显示删除目录对话框
+    showDeleteDirectoryDialog() {
+        const currentDirName = this.currentPath.split('/').pop() || this.currentPath;
+        
+        const modalHtml = `
+            <div id="delete-dir-modal" class="modal-overlay">
+                <div class="modal-content">
+                    <div class="modal-header">
+                        <h3>删除目录</h3>
+                        <button onclick="fileBrowser.closeDeleteDirectoryDialog()" class="modal-close">×</button>
+                    </div>
+                    <div class="modal-body">
+                        <div class="delete-warning">
+                            <div class="warning-icon">⚠️</div>
+                            <div class="warning-text">
+                                <p>您确定要删除目录 "<span class="file-name-highlight">${currentDirName}</span>" 吗？</p>
+                                <p><strong>此操作不可撤销！</strong></p>
+                            </div>
+                        </div>
+                    </div>
+                    <div class="modal-footer">
+                        <button onclick="fileBrowser.closeDeleteDirectoryDialog()" class="btn btn-cancel">取消</button>
+                        <button onclick="fileBrowser.deleteDirectory()" class="btn btn-danger" id="delete-dir-confirm">删除</button>
+                    </div>
+                </div>
+            </div>
+        `;
+
+        document.body.insertAdjacentHTML('beforeend', modalHtml);
+    }
+
+    // 关闭删除目录对话框
+    closeDeleteDirectoryDialog() {
+        const modal = document.getElementById('delete-dir-modal');
+        if (modal) {
+            modal.remove();
+        }
+    }
+
+    // 删除目录
+    async deleteDirectory() {
+        const confirmBtn = document.getElementById('delete-dir-confirm');
+        
+        // 禁用按钮，防止重复提交
+        confirmBtn.disabled = true;
+        confirmBtn.textContent = '删除中...';
+        
+        try {
+            const response = await fetch('/api/delete-directory', {
+                method: 'DELETE',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                    path: this.currentPath
+                })
+            });
+            
+            const result = await response.json();
+            
+            if (response.ok && result.success) {
+                // 成功删除
+                this.closeDeleteDirectoryDialog();
+                
+                // 导航到父目录
+                const parentPath = this.currentPath.substring(0, this.currentPath.lastIndexOf('/'));
+                this.navigateToPath(parentPath);
+                
+                // 显示成功消息
+                this.showMessage('目录删除成功', 'success');
+            } else {
+                // 删除失败
+                alert(result.error || '删除目录失败');
+                confirmBtn.disabled = false;
+                confirmBtn.textContent = '删除';
+            }
+        } catch (error) {
+            console.error('删除目录失败:', error);
+            alert('删除目录失败: ' + error.message);
+            confirmBtn.disabled = false;
+            confirmBtn.textContent = '删除';
+        }
+    }
+
+    // 显示消息提示
+    showMessage(text, type = 'info') {
+        const messageHtml = `
+            <div class="delete-message ${type}">
+                <div class="message-icon">${type === 'success' ? '✅' : type === 'error' ? '❌' : 'ℹ️'}</div>
+                <div class="message-text">${text}</div>
+                <button onclick="this.parentElement.remove()" class="message-close">×</button>
+            </div>
+        `;
+        
+        document.body.insertAdjacentHTML('beforeend', messageHtml);
+        
+        // 3秒后自动消失
+        setTimeout(() => {
+            const message = document.querySelector('.delete-message');
+            if (message) {
+                message.remove();
+            }
+        }, 3000);
+    }
+}
+
+// 初始化文件浏览器
+let fileBrowser;
+document.addEventListener('DOMContentLoaded', function() {
+    fileBrowser = new FileBrowser();
+}); 

mildoc_admin/static/js/file_detail.js

@@ -0,0 +1,374 @@
+// 文件详情页面功能
+class FileDetail {
+    constructor() {
+        this.filePath = this.getFilePathFromURL();
+        this.init();
+    }
+
+    // 从 URL 获取文件路径
+    getFilePathFromURL() {
+        const pathParts = window.location.pathname.split('/');
+        const encodedPath = pathParts.slice(2).join('/'); // 去掉 /file/ 前缀
+        // 解码路径
+        return decodeURIComponent(encodedPath);
+    }
+
+    init() {
+        this.loadFileDetail();
+        this.bindEvents();
+    }
+
+    // 绑定事件
+    bindEvents() {
+        document.getElementById('expand-all').addEventListener('click', () => {
+            this.expandAllChunks();
+        });
+
+        document.getElementById('collapse-all').addEventListener('click', () => {
+            this.collapseAllChunks();
+        });
+
+        document.getElementById('back-btn').addEventListener('click', (e) => {
+            e.preventDefault();
+            this.goBack();
+        });
+
+        document.getElementById('download-btn').addEventListener('click', (e) => {
+            e.preventDefault();
+            this.downloadFile();
+        });
+
+        document.getElementById('delete-btn').addEventListener('click', (e) => {
+            e.preventDefault();
+            this.showDeleteConfirmDialog();
+        });
+    }
+
+    // 返回文件所在目录
+    goBack() {
+        // 计算父目录路径
+        const pathParts = this.filePath.split('/');
+        pathParts.pop(); // 移除文件名
+        const parentPath = pathParts.join('/');
+        
+        // 构建返回 URL
+        if (parentPath) {
+            window.location.href = `/files?path=${encodeURIComponent(parentPath)}`;
+        } else {
+            window.location.href = '/files';
+        }
+    }
+
+    // 下载文件
+    downloadFile() {
+        const downloadBtn = document.getElementById('download-btn');
+        const originalText = downloadBtn.textContent;
+        
+        // 显示下载状态
+        downloadBtn.textContent = '下载中...';
+        downloadBtn.classList.add('downloading');
+        
+        // 构建下载URL
+        const downloadUrl = `/api/file/${this.filePath.split('/').map(part => encodeURIComponent(part)).join('/')}/download`;
+        
+        // 创建隐藏的下载链接
+        const link = document.createElement('a');
+        link.href = downloadUrl;
+        link.style.display = 'none';
+        document.body.appendChild(link);
+        
+        // 触发下载
+        link.click();
+        
+        // 清理
+        document.body.removeChild(link);
+        
+        // 恢复按钮状态
+        setTimeout(() => {
+            downloadBtn.textContent = originalText;
+            downloadBtn.classList.remove('downloading');
+        }, 1000);
+    }
+
+    // 显示删除确认对话框
+    showDeleteConfirmDialog() {
+        const fileName = decodeURIComponent(this.filePath.split('/').pop());
+        
+        const modalHtml = `
+            <div id="delete-confirm-modal" class="modal-overlay" onclick="fileDetail.closeDeleteModal(event)">
+                <div class="modal-content delete-modal">
+                    <div class="modal-header">
+                        <h3>确认删除文件</h3>
+                        <button onclick="fileDetail.closeDeleteModal()" class="modal-close">&times;</button>
+                    </div>
+                    <div class="modal-body">
+                        <div class="delete-warning">
+                            <div class="warning-icon">⚠️</div>
+                            <div class="warning-text">
+                                <p><strong>您确定要删除以下文件吗？</strong></p>
+                                <p class="file-name-highlight">${this.escapeHtml(fileName)}</p>
+                                <p class="warning-message">此操作将永久删除文件，无法恢复。如果文件已被索引，相关的向量数据也会一并删除。</p>
+                            </div>
+                        </div>
+                    </div>
+                    <div class="modal-footer">
+                        <button onclick="fileDetail.closeDeleteModal()" class="btn btn-cancel">取消</button>
+                        <button onclick="fileDetail.confirmDeleteFile()" class="btn btn-danger" id="confirm-delete-btn">确认删除</button>
+                    </div>
+                </div>
+            </div>
+        `;
+        
+        document.body.insertAdjacentHTML('beforeend', modalHtml);
+    }
+
+    // 关闭删除确认对话框
+    closeDeleteModal(event) {
+        if (!event || event.target.classList.contains('modal-overlay') || event.target.classList.contains('modal-close')) {
+            const modal = document.getElementById('delete-confirm-modal');
+            if (modal) {
+                modal.remove();
+            }
+        }
+    }
+
+    // 确认删除文件
+    confirmDeleteFile() {
+        const confirmBtn = document.getElementById('confirm-delete-btn');
+        const originalText = confirmBtn.textContent;
+        
+        // 显示删除中状态
+        confirmBtn.textContent = '删除中...';
+        confirmBtn.disabled = true;
+        confirmBtn.classList.add('loading');
+        
+        // 构建删除URL
+        const encodedPath = this.filePath.split('/').map(part => encodeURIComponent(part)).join('/');
+        const deleteUrl = `/api/file/${encodedPath}/delete`;
+        
+        Utils.ajax(deleteUrl, 'DELETE')
+            .then(data => {
+                // 删除成功
+                this.closeDeleteModal();
+                this.showDeleteSuccessMessage(data.message);
+                
+                // 2秒后返回文件列表
+                setTimeout(() => {
+                    this.goBack();
+                }, 2000);
+            })
+            .catch(error => {
+                // 删除失败
+                this.showDeleteErrorMessage(error.message);
+                
+                // 恢复按钮状态
+                confirmBtn.textContent = originalText;
+                confirmBtn.disabled = false;
+                confirmBtn.classList.remove('loading');
+            });
+    }
+
+    // 显示删除成功消息
+    showDeleteSuccessMessage(message) {
+        const messageHtml = `
+            <div id="delete-success-message" class="delete-message success">
+                <div class="message-icon">✅</div>
+                <div class="message-text">
+                    <strong>删除成功</strong><br>
+                    ${message}<br>
+                    <small>2秒后自动返回文件列表...</small>
+                </div>
+            </div>
+        `;
+        
+        const fileInfoElement = document.getElementById('file-info');
+        fileInfoElement.insertAdjacentHTML('beforebegin', messageHtml);
+        
+        // 3秒后自动移除消息
+        setTimeout(() => {
+            const msg = document.getElementById('delete-success-message');
+            if (msg) msg.remove();
+        }, 3000);
+    }
+
+    // 显示删除错误消息
+    showDeleteErrorMessage(message) {
+        const messageHtml = `
+            <div id="delete-error-message" class="delete-message error">
+                <div class="message-icon">❌</div>
+                <div class="message-text">
+                    <strong>删除失败</strong><br>
+                    ${message}
+                </div>
+                <button onclick="document.getElementById('delete-error-message').remove()" class="message-close">&times;</button>
+            </div>
+        `;
+        
+        const fileInfoElement = document.getElementById('file-info');
+        fileInfoElement.insertAdjacentHTML('beforebegin', messageHtml);
+    }
+
+    // 加载文件详情
+    loadFileDetail() {
+        const fileInfoElement = document.getElementById('file-info');
+        Utils.showLoading(fileInfoElement);
+
+        // 更新页面标题
+        document.getElementById('file-title').textContent = `文件详情 - ${decodeURIComponent(this.filePath.split('/').pop())}`;
+
+        const url = `/api/file/${this.filePath.split('/').map(part => encodeURIComponent(part)).join('/')}`;
+        
+        Utils.ajax(url)
+            .then(data => {
+                this.renderFileInfo(data);
+                if (data.chunks && data.chunks.length > 0) {
+                    this.renderChunks(data.chunks);
+                }
+            })
+            .catch(error => {
+                Utils.showError(fileInfoElement, error.message);
+            });
+    }
+
+    // 渲染文件信息
+    renderFileInfo(fileInfo) {
+        const fileInfoElement = document.getElementById('file-info');
+        
+        const statusBadge = fileInfo.indexed 
+            ? '<span class="status-badge status-indexed">已索引</span>'
+            : '<span class="status-badge status-not-indexed">未索引</span>';
+
+        let html = `
+            <div class="info-grid">
+                <div class="info-item">
+                    <div class="info-label">文件名</div>
+                    <div class="info-value">${fileInfo.doc_name}</div>
+                </div>
+                <div class="info-item">
+                    <div class="info-label">文件路径</div>
+                    <div class="info-value">${fileInfo.doc_path_name}</div>
+                </div>
+                <div class="info-item">
+                    <div class="info-label">文件大小</div>
+                    <div class="info-value">${Utils.formatFileSize(fileInfo.file_size)}</div>
+                </div>
+                <div class="info-item">
+                    <div class="info-label">MD5值</div>
+                    <div class="info-value">${fileInfo.file_md5}</div>
+                </div>
+                <div class="info-item">
+                    <div class="info-label">最后修改时间</div>
+                    <div class="info-value">${fileInfo.last_modified}</div>
+                </div>
+                <div class="info-item">
+                    <div class="info-label">索引状态</div>
+                    <div class="info-value">
+                        ${statusBadge}
+                    </div>
+                </div>
+        `;
+
+        if (fileInfo.indexed) {
+            html += `
+                <div class="info-item">
+                    <div class="info-label">文档类型</div>
+                    <div class="info-value">${fileInfo.doc_type || '-'}</div>
+                </div>
+                <div class="info-item">
+                    <div class="info-label">索引MD5</div>
+                    <div class="info-value">${fileInfo.indexed_md5 || '-'}</div>
+                </div>
+                <div class="info-item">
+                    <div class="info-label">文档长度</div>
+                    <div class="info-value">${fileInfo.doc_length || 0} 字符</div>
+                </div>
+                <div class="info-item">
+                    <div class="info-label">嵌入模型</div>
+                    <div class="info-value">${fileInfo.embedding_model || '-'}</div>
+                </div>
+                <div class="info-item">
+                    <div class="info-label">分片数量</div>
+                    <div class="info-value">${fileInfo.chunks ? fileInfo.chunks.length : 0} 个</div>
+                </div>
+            `;
+        }
+
+        html += '</div>';
+        fileInfoElement.innerHTML = html;
+    }
+
+    // 渲染文件分片
+    renderChunks(chunks) {
+        const chunksElement = document.getElementById('file-chunks');
+        const chunksContent = document.getElementById('chunks-content');
+        
+        if (chunks.length === 0) {
+            chunksElement.style.display = 'none';
+            return;
+        }
+
+        chunksElement.style.display = 'block';
+        
+        let html = '';
+        chunks.forEach((chunk, index) => {
+            const chunkId = `chunk-${index}`;
+            html += `
+                <div class="chunk-item">
+                    <div class="chunk-header" onclick="fileDetail.toggleChunk('${chunkId}')">
+                        <div class="chunk-title">分片 ${index + 1} (${chunk.length} 字符)</div>
+                        <div class="chunk-toggle" id="${chunkId}-toggle">展开</div>
+                    </div>
+                    <div class="chunk-content collapsed" id="${chunkId}">${this.escapeHtml(chunk.content)}</div>
+                </div>
+            `;
+        });
+
+        chunksContent.innerHTML = html;
+    }
+
+    // 切换分片展开/折叠
+    toggleChunk(chunkId) {
+        const content = document.getElementById(chunkId);
+        const toggle = document.getElementById(chunkId + '-toggle');
+        
+        if (content.classList.contains('collapsed')) {
+            content.classList.remove('collapsed');
+            toggle.textContent = '折叠';
+        } else {
+            content.classList.add('collapsed');
+            toggle.textContent = '展开';
+        }
+    }
+
+    // 展开所有分片
+    expandAllChunks() {
+        const chunks = document.querySelectorAll('.chunk-content');
+        const toggles = document.querySelectorAll('.chunk-toggle');
+        
+        chunks.forEach(chunk => chunk.classList.remove('collapsed'));
+        toggles.forEach(toggle => toggle.textContent = '折叠');
+    }
+
+    // 折叠所有分片
+    collapseAllChunks() {
+        const chunks = document.querySelectorAll('.chunk-content');
+        const toggles = document.querySelectorAll('.chunk-toggle');
+        
+        chunks.forEach(chunk => chunk.classList.add('collapsed'));
+        toggles.forEach(toggle => toggle.textContent = '展开');
+    }
+
+
+    // HTML 转义
+    escapeHtml(text) {
+        const div = document.createElement('div');
+        div.textContent = text;
+        return div.innerHTML;
+    }
+}
+
+// 初始化文件详情页面
+let fileDetail;
+document.addEventListener('DOMContentLoaded', function() {
+    fileDetail = new FileDetail();
+}); 

mildoc_admin/static/js/main.js

@@ -0,0 +1,75 @@
+// 通用工具函数
+const Utils = {
+    // 格式化文件大小
+    formatFileSize: function(bytes) {
+        if (bytes === 0) return '0 Bytes';
+        const k = 1024;
+        const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
+        const i = Math.floor(Math.log(bytes) / Math.log(k));
+        return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+    },
+
+    // 显示加载状态
+    showLoading: function(element) {
+        element.innerHTML = '<div class="loading">加载中...</div>';
+    },
+
+    // 显示错误信息
+    showError: function(element, message) {
+        element.innerHTML = `<div class="error">错误: ${message}</div>`;
+    },
+
+    // 发送 AJAX 请求
+    ajax: function(url, methodOrOptions = {}) {
+        let options = {};
+        
+        // 如果第二个参数是字符串，视为 HTTP method
+        if (typeof methodOrOptions === 'string') {
+            options = { method: methodOrOptions };
+        } else {
+            options = methodOrOptions;
+        }
+        
+        return fetch(url, {
+            method: 'GET',
+            headers: {
+                'Content-Type': 'application/json',
+                ...options.headers
+            },
+            ...options
+        }).then(response => {
+            if (!response.ok) {
+                // 对于404错误，尝试解析JSON响应以获取更详细的错误信息
+                if (response.status === 404) {
+                    return response.json().then(data => {
+                        throw new Error(data.error || `文件不存在 (${response.status})`);
+                    }).catch(() => {
+                        throw new Error(`文件不存在 (${response.status})`);
+                    });
+                }
+                
+                // 尝试解析错误响应
+                return response.json().then(data => {
+                    throw new Error(data.error || `HTTP ${response.status}: ${response.statusText}`);
+                }).catch(() => {
+                    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                });
+            }
+            return response.json();
+        });
+    }
+};
+
+// 初始化页面
+document.addEventListener('DOMContentLoaded', function() {
+    // 自动隐藏闪现消息
+    const flashMessages = document.querySelectorAll('.flash-message');
+    flashMessages.forEach(function(message) {
+        setTimeout(function() {
+            message.style.opacity = '0';
+            setTimeout(function() {
+                message.remove();
+            }, 300);
+        }, 5000);
+    });
+}); 

mildoc_admin/templates/base.html

@@ -0,0 +1,45 @@
+<!DOCTYPE html>
+<html lang="zh">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{% block title %}MilDoc 管理后台{% endblock %}</title>
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/style.css') }}">
+    {% block head %}{% endblock %}
+</head>
+<body>
+    <div class="container">
+        {% if session.username %}
+        <header class="header">
+            <div class="header-content">
+                <h1 class="logo">MilDoc 管理后台</h1>
+                <nav class="nav">
+                    <a href="{{ url_for('file_browser') }}" class="nav-link">文件浏览</a>
+                    <div class="user-info">
+                        <span>欢迎，{{ session.username }}</span>
+                    </div>
+                    <a href="{{ url_for('logout') }}" class="nav-link logout">退出</a>
+                </nav>
+            </div>
+        </header>
+        {% endif %}
+
+        <main class="main">
+            {% with messages = get_flashed_messages(with_categories=true) %}
+                {% if messages %}
+                    <div class="flash-messages">
+                        {% for category, message in messages %}
+                            <div class="flash-message flash-{{ category }}">{{ message }}</div>
+                        {% endfor %}
+                    </div>
+                {% endif %}
+            {% endwith %}
+
+            {% block content %}{% endblock %}
+        </main>
+    </div>
+
+    <script src="{{ url_for('static', filename='js/main.js') }}"></script>
+    {% block scripts %}{% endblock %}
+</body>
+</html> 

mildoc_admin/templates/file_browser.html

@@ -0,0 +1,39 @@
+{% extends "base.html" %}
+
+{% block title %}文件浏览 - Mildoc 管理后台{% endblock %}
+
+{% block content %}
+<div class="file-browser">
+    <div class="file-browser-header">
+        <div class="breadcrumb">
+            <span class="breadcrumb-item">
+                <a href="{{ url_for('file_browser') }}" class="breadcrumb-link">根目录</a>
+            </span>
+            <span id="breadcrumb-path"></span>
+        </div>
+        <div class="browser-actions">
+            <button id="create-dir-btn" class="create-dir-btn">创建目录</button>
+            <button id="delete-dir-btn" class="delete-dir-btn" style="display: none;">删除目录</button>
+            <button id="upload-btn" class="upload-btn">上传文件</button>
+        </div>
+    </div>
+    
+    <!-- 文件上传对话框 -->
+    <input type="file" id="file-input" multiple style="display: none;" accept="*/*">
+
+    <div class="file-list">
+        <div class="file-list-header">
+            <div class="file-item-name">名称</div>
+            <div class="file-item-size">大小</div>
+            <div class="file-item-modified">修改时间</div>
+        </div>
+        <div id="file-list-content" class="file-list-content">
+            <div class="loading">加载中...</div>
+        </div>
+    </div>
+</div>
+{% endblock %}
+
+{% block scripts %}
+<script src="{{ url_for('static', filename='js/file_browser.js') }}"></script>
+{% endblock %} 

mildoc_admin/templates/file_detail.html

@@ -0,0 +1,33 @@
+{% extends "base.html" %}
+
+{% block title %}文件详情 - Mildoc 管理后台{% endblock %}
+
+{% block content %}
+<div class="file-detail">
+    <div class="file-detail-header">
+        <h2 class="file-title" id="file-title">文件详情</h2>
+        <div class="header-actions">
+            <a href="#" id="download-btn" class="download-btn">下载地址</a>
+            <a href="#" id="delete-btn" class="delete-btn">删除文件</a>
+            <a href="#" id="back-btn" class="back-btn">← 返回文件列表</a>
+        </div>
+    </div>
+
+    <div id="file-info" class="file-info">
+        <div class="loading">加载中...</div>
+    </div>
+
+    <div id="file-chunks" class="file-chunks" style="display: none;">
+        <h3 class="chunks-title">文件分片信息</h3>
+        <div class="chunks-toggle">
+            <button id="expand-all" class="btn btn-secondary">展开全部</button>
+            <button id="collapse-all" class="btn btn-secondary">折叠全部</button>
+        </div>
+        <div id="chunks-content" class="chunks-content"></div>
+    </div>
+</div>
+{% endblock %}
+
+{% block scripts %}
+<script src="{{ url_for('static', filename='js/file_detail.js') }}"></script>
+{% endblock %} 

mildoc_admin/templates/login.html

@@ -0,0 +1,27 @@
+{% extends "base.html" %}
+
+{% block title %}登录 - Mildoc 管理后台{% endblock %}
+
+{% block content %}
+<div class="login-container">
+    <div class="login-box">
+        <h2 class="login-title">Mildoc 管理后台</h2>
+        <p class="login-subtitle">请输入您的用户名和密码</p>
+        
+        <form method="POST" class="login-form">
+            <div class="form-group">
+                <label for="username" class="form-label">用户名</label>
+                <input type="text" id="username" name="username" class="form-input" required>
+            </div>
+            
+            <div class="form-group">
+                <label for="password" class="form-label">密码</label>
+                <input type="password" id="password" name="password" class="form-input" required>
+            </div>
+            
+            <button type="submit" class="login-btn">登录</button>
+        </form>
+        
+    </div>
+</div>
+{% endblock %} 

mildoc_index/.env

@@ -0,0 +1,45 @@
+# mildoc index
+
+# minio
+MINIO_ENDPOINT=127.0.0.1:9000
+MINIO_ACCESS_KEY=minioadmin
+MINIO_SECRET_KEY=minioadmin
+MINIO_BUCKET=mildoc
+MINIO_REGION=minio
+MINIO_USE_VIRTUAL_HOST=false
+MINIO_USE_SSL=false
+
+
+# aliyun oss & mns
+# MINIO_ENDPOINT=oss-cn-hangzhou.aliyuncs.com:443
+# MINIO_ACCESS_KEY=LTAI5tHYKMyo8fNScFHkaNUS
+# MINIO_SECRET_KEY=
+# MINIO_BUCKET=mildoc
+# MINIO_REGION=oss-cn-hangzhou
+# MINIO_USE_VIRTUAL_HOST=true
+# MINIO_USE_SSL=true
+#
+# MNS_ACCESS_KEY_ID=LTAI5tHYKMyo8fNScFHkaNUS
+# MNS_ACCESS_KEY_SECRET=
+# MNS_ENDPOINT=https://1539444852632014.mns.cn-hangzhou.aliyuncs.com
+# MNS_QUEUE_NAME=mildoc-oss-notify
+
+# milvus
+MILVUS_HOST=127.0.0.1
+MILVUS_PORT=19530
+MILVUS_USER=root
+MILVUS_PASSWORD=admin123
+MILVUS_DATABASE=mildoc
+MILVUS_COLLECTION=mildoc_collection
+MILVUS_INDEX_NAME=mildoc_index
+MILVUS_VECTOR_DIM=768
+
+# bailian embedding llm
+OPENAI_API_KEY=sk-
+OPENAI_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
+ENBEDDING_MODEL=text-embedding-v4
+
+
+# MinerU
+# 启用MinerU前，先安装： uv add mineru[core]==2.7.0
+# MINERU_SERVER_URL=http://127.0.0.1:8000

mildoc_index/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+mildoc_index.log

mildoc_index/embedding.py

@@ -0,0 +1,154 @@
+import os
+from openai import OpenAI
+from dotenv import load_dotenv
+from typing import List
+from logger.logging import setup_logging
+
+load_dotenv()
+
+logger = setup_logging()
+
+class EmbeddingTool:
+    def __init__(self):
+        """
+        初始化embedding工具
+        """
+        self.client = OpenAI(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            base_url=os.getenv("OPENAI_BASE_URL")
+        )
+        self.model = os.getenv("ENBEDDING_MODEL")
+        self.dimensions = int(os.getenv("MILVUS_VECTOR_DIM"))
+        self.encoding_format = "float"
+
+        if not self.client or not self.model or not self.dimensions or not self.encoding_format:
+            logger.error("Embedding工具初始化失败")
+            raise ValueError("Embedding工具初始化失败")
+        
+    def get_embedding(self, text: str) -> List[float]:
+        """
+        获取单个文本的embedding向量
+        
+        Args:
+            text (str): 输入文本
+            
+        Returns:
+            List[float]: 向量列表
+        """
+        try:
+            response = self.client.embeddings.create(
+                model=self.model,
+                input=[text],
+                dimensions=self.dimensions,
+                encoding_format=self.encoding_format
+            )
+            
+            if not response.data:
+                logger.error("获取embedding失败")
+                return []
+            
+            embedding = response.data[0].embedding
+            if not embedding:
+                logger.error("获取embedding失败")
+                return []
+            
+            if len(embedding) != self.dimensions:
+                logger.error("获取embedding失败")
+                return []
+            
+            return embedding
+            
+        except Exception as e:
+            print(f"获取embedding失败: {e}")
+            return []
+    
+    def get_embeddings_batch(self, texts: List[str]) -> List[List[float]]:
+        """
+        批量获取多个文本的embedding向量
+        
+        Args:
+            texts (List[str]): 文本列表
+            
+        Returns:
+            List[List[float]]: 向量列表的列表
+        """
+        try:
+            response = self.client.embeddings.create(
+                model=self.model,
+                input=texts,
+                dimensions=self.dimensions,
+                encoding_format=self.encoding_format
+            )
+            
+            if not response.data:
+                logger.error("批量获取embedding失败")
+                return []
+            
+            embeddings = [data.embedding for data in response.data] 
+            
+            if len(embeddings) != len(texts):
+                logger.error("批量获取embedding失败")
+                return []
+
+            for embedding in embeddings:
+                if len(embedding) != self.dimensions:
+                    logger.error("批量获取embedding失败")
+                    return []
+            
+            return embeddings
+            
+        except Exception as e:
+            print(f"批量获取embedding失败: {e}")
+            return []
+    
+    def get_model_info(self) -> dict:
+        """
+        获取模型信息
+        
+        Returns:
+            dict: 模型信息
+        """
+        return {
+            "model": self.model,
+            "dimensions": self.dimensions,
+            "encoding_format": self.encoding_format,
+            "base_url": self.client.base_url
+        }
+
+
+# 使用示例和测试
+if __name__ == "__main__":
+    print("=== Embedding工具测试 ===")
+    
+    # 创建embedding工具实例
+    embedding_tool = EmbeddingTool()
+    
+    # 显示模型信息
+    print("模型信息:", embedding_tool.get_model_info())
+    
+    # 单个文本embedding测试
+    print("\n=== 单个文本embedding测试 ===")
+    test_text = "这是一个测试文档的内容，用于生成向量表示。"
+    embedding = embedding_tool.get_embedding(test_text)
+    if embedding:
+        print(f"文本: {test_text}")
+        print(f"向量维度: {len(embedding)}")
+        print(f"向量前5个值: {embedding[:5]}")
+    
+    # 批量文本embedding测试
+    print("\n=== 批量文本embedding测试 ===")
+    test_texts = [
+        "风急天高猿啸哀",
+        "渚清沙白鸟飞回", 
+        "无边落木萧萧下",
+        "不尽长江滚滚来"
+    ]
+    
+    embeddings = embedding_tool.get_embeddings_batch(test_texts)
+    if embeddings:
+        print(f"处理了 {len(embeddings)} 个文本")
+        for i, (text, emb) in enumerate(zip(test_texts, embeddings)):
+            print(f"文本{i+1}: {text}")
+            print(f"向量维度: {len(emb)}, 前3个值: {emb[:5]}")
+    
+    print("\n=== 测试完成 ===")

mildoc_index/logger/logging.py

@@ -0,0 +1,21 @@
+import logging
+
+def setup_logging(name="default", level=logging.INFO):
+    """配置日志系统"""
+    # 创建日志格式
+    log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    
+    # 配置根日志器
+    logging.basicConfig(
+        level=level,
+        format=log_format,
+        handlers=[
+            logging.StreamHandler(),  # 控制台输出
+        ]
+    )
+    
+    # 创建应用专用日志器
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+    
+    return logger

mildoc_index/main.py

@@ -0,0 +1,106 @@
+import argparse
+
+from logger.logging import setup_logging
+
+logger = setup_logging()
+    
+
+
+def main():
+    """主函数"""
+    # 创建参数解析器
+    parser = argparse.ArgumentParser(
+        description="Minio文档处理系统 - 将Minio中的文档解析并存储到Milvus向量数据库",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+使用示例:
+  # 全量刷新模式
+  python main.py --provider minio --mode full-refresh
+  
+  # 排查补漏模式
+  python main.py --provider minio --mode backfill
+  
+  # 增量更新模式（实时监听）
+  python main.py --provider minio --mode listen
+
+  # 使用 OSS 作为对象存储提供商
+  python main.py --provider oss --mode listen
+        """
+    )
+    
+    parser.add_argument(
+        "--mode",
+        choices=["full-refresh", "backfill", "listen"],
+        required=True,
+        help="运行模式选择: full-refresh=全量刷新, backfill=排查补漏, listen=增量更新(实时监听)"
+    )
+    
+    parser.add_argument(
+        "--provider",
+        choices=["oss", "minio"],
+        default="minio",
+        type=str,
+        help="对象存储提供商: oss=阿里云OSS, minio=Minio"
+    )
+    
+    # 解析命令行参数
+    args = parser.parse_args()
+    
+    logger.info("=== Minio文档处理系统 ===")
+    logger.info(f"运行模式: {args.mode}")
+    
+    try:
+        # 创建监听器实例
+        if args.provider == "oss":
+            from oss_event_handler import OSSEventHandler
+            listener = OSSEventHandler()
+            logger.info(f"使用 OSS 最为对象存储提供商")
+        else:
+            from minio_event_handler import MinioEventHandler
+            listener = MinioEventHandler()
+            logger.info(f"默认使用 Minio 作为对象存储提供商")
+        
+        logger.info("=== 系统初始化完成 ===")
+        
+        # 根据模式执行相应操作
+        if args.mode == "full-refresh":
+            logger.info("\n执行全量刷新模式...")
+            listener.full_update()
+            
+        elif args.mode == "backfill":
+            logger.info("\n执行排查补漏模式...")
+            listener.backfill_update()
+            
+        elif args.mode == "listen":
+            logger.info("\n执行增量更新模式（实时监听）...")
+            logger.info("提示: 使用 Ctrl+C 停止监听，或使用 nohup 在后台运行")
+            listener.start_listening()
+        else:
+            ## 使用方式说明
+            logger.info("""
+使用示例:
+  # 全量刷新模式
+  python main.py --provider minio --mode full-refresh
+  
+  # 排查补漏模式
+  python main.py --provider minio --mode backfill
+  
+  # 增量更新模式（实时监听）
+  python main.py --provider minio --mode listen
+
+  # 使用 OSS 作为对象存储提供商
+  python main.py --provider oss --mode listen            
+            """)
+
+
+        logger.info("\n程序执行完成")
+        
+    except KeyboardInterrupt:
+        logger.info("\n用户中断程序")
+    except Exception as e:
+        logger.error(f"程序运行出错: {e}")
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

mildoc_index/milvus_api.py

@@ -0,0 +1,348 @@
+from enum import Enum
+from pymilvus import MilvusClient, DataType
+import os
+from dotenv import load_dotenv
+from logger.logging import setup_logging
+from pprint import pprint
+from dataclasses import dataclass, asdict
+
+load_dotenv()
+
+logger = setup_logging()
+
+
+@dataclass
+class MilvusDocument:
+    doc_name: str # 文档名称
+    doc_path_name: str # 文档路径（含名字）
+    doc_type: str # 文档类型
+    doc_md5: str # 文档MD5
+    doc_length: int # 文档字节数
+    content: str # 文档分段内容
+    content_vector: list # 分段内容向量
+    embedding_model: str # embedding模型名称
+
+class MilvusDocumentField(str, Enum):
+    ID = "id" # 主键ID
+    DOC_NAME = "doc_name" # 文档名称
+    DOC_PATH_NAME = "doc_path_name" # 文档路径（含名字）
+    DOC_TYPE = "doc_type" # 文档类型
+    DOC_MD5 = "doc_md5" # 文档MD5
+    DOC_LENGTH = "doc_length" # 文档字节数
+    CONTENT = "content" # 文档分段内容
+    CONTENT_VECTOR = "content_vector" # 分段内容向量
+    EMBEDDING_MODEL = "embedding_model" # embedding模型名称
+
+class MilvusAPI:
+    def __init__(self):
+        """初始化Milvus客户端连接"""
+        self.database_name = os.getenv("MILVUS_DATABASE")
+        self.collection_name = os.getenv("MILVUS_COLLECTION") 
+        self.index_name = os.getenv("MILVUS_INDEX_NAME")
+        self.vector_dim = int(os.getenv("MILVUS_VECTOR_DIM")) 
+        
+        if not self.database_name or not self.collection_name or not self.index_name or not self.vector_dim:
+            logger.error("Milvus配置错误")
+            raise ValueError("Milvus配置错误")
+
+        # 创建客户端连接，指定数据库
+        self.client = MilvusClient(
+            uri=f"http://{os.getenv('MILVUS_HOST')}:{os.getenv('MILVUS_PORT')}",
+            user=os.getenv('MILVUS_USER'),
+            password=os.getenv('MILVUS_PASSWORD'),
+            db_name=self.database_name
+        )
+        
+        init_result =   self._initialize()
+        if not init_result:
+            logger.error("Milvus初始化失败")
+            raise ValueError("Milvus初始化失败")
+                
+    def _create_collection_if_not_exists(self) -> bool:
+        """创建集合（如果不存在则创建）"""
+        try:
+            # 检查集合是否存在
+            if self.client.has_collection(collection_name=self.collection_name):
+                logger.info(f"集合 '{self.collection_name}' 已存在")
+                return True
+            
+            # 定义schema
+            schema = self.client.create_schema(
+                auto_id=True,  # 自动生成ID
+                enable_dynamic_field=False
+            )
+            
+            # 添加字段
+            # 主键ID字段（自动生成）
+            schema.add_field(
+                field_name=MilvusDocumentField.ID.value,
+                datatype=DataType.INT64,
+                is_primary=True,
+                auto_id=True
+            )
+            
+            # 文档名称
+            schema.add_field(
+                field_name=MilvusDocumentField.DOC_NAME.value,
+                datatype=DataType.VARCHAR,
+                max_length=500
+            )
+            
+            # 文档路径（含名字）
+            schema.add_field(
+                field_name=MilvusDocumentField.DOC_PATH_NAME.value,
+                datatype=DataType.VARCHAR,
+                max_length=1000
+            )
+            
+            # 文档类型
+            schema.add_field(
+                field_name=MilvusDocumentField.DOC_TYPE.value,
+                datatype=DataType.VARCHAR,
+                max_length=50
+            )
+            
+            # 文档MD5
+            schema.add_field(
+                field_name=MilvusDocumentField.DOC_MD5.value,
+                datatype=DataType.VARCHAR,
+                max_length=32
+            )
+            
+            # 文档字节数
+            schema.add_field(
+                field_name=MilvusDocumentField.DOC_LENGTH.value,
+                datatype=DataType.INT64
+            )
+            
+            # 文档内容
+            schema.add_field(
+                field_name=MilvusDocumentField.CONTENT.value,
+                datatype=DataType.VARCHAR,
+                max_length=65535  # 最大长度
+            )
+            
+            # 内容向量（text-embedding-v4的维度是1536）
+            schema.add_field(
+                field_name=MilvusDocumentField.CONTENT_VECTOR.value,
+                datatype=DataType.FLOAT_VECTOR,
+                dim=self.vector_dim
+            )
+            
+            # embedding模型名称
+            schema.add_field(
+                field_name=MilvusDocumentField.EMBEDDING_MODEL.value,
+                datatype=DataType.VARCHAR,
+                max_length=100
+            )
+            
+            # 创建集合
+            self.client.create_collection(
+                collection_name=self.collection_name,
+                schema=schema
+            )
+            
+            logger.info(f"集合 '{self.collection_name}' 创建成功")
+            return True
+            
+        except Exception as e:
+            logger.error(f"创建集合失败: {e}")
+            return False
+    
+    def _create_index_if_not_exists(self) -> bool:
+        """创建索引"""
+        try:
+            # 检查索引是否已存在
+            indexes = self.client.list_indexes(collection_name=self.collection_name)
+            if self.index_name in indexes:
+                logger.info(f"索引 '{self.index_name}' 已存在")
+                return True
+            
+            # 创建向量索引
+            index_params = self.client.prepare_index_params()
+            index_params.add_index(
+                field_name=MilvusDocumentField.CONTENT_VECTOR.value,
+                index_type="IVF_FLAT",
+                metric_type="COSINE",
+                params={"nlist": 1024}
+            )
+            
+            self.client.create_index(
+                collection_name=self.collection_name,
+                index_params=index_params
+            )
+            
+            logger.info(f"索引 '{self.index_name}' 创建成功")
+            return True
+            
+        except Exception as e:
+            logger.error(f"创建索引失败: {e}")
+            return False
+    
+    def _load_collection(self) -> bool:
+        """加载集合到内存"""
+        try:
+            self.client.load_collection(collection_name=self.collection_name)
+            logger.info(f"集合 '{self.collection_name}' 加载成功")
+            return True
+        except Exception as e:
+            logger.error(f"加载集合失败: {e}")
+            return False
+    
+    def _initialize(self) -> bool:
+        """初始化数据库、集合和索引"""
+        logger.info("开始初始化Milvus...")
+                
+        # 创建集合
+        if not self._create_collection_if_not_exists():
+            return False
+        
+        # 创建索引
+        if not self._create_index_if_not_exists():
+            return False
+        
+        # 加载集合到内存
+        if not self._load_collection():
+            return False
+        
+        logger.info("Milvus初始化完成!")
+        return True
+    
+    def check_document_exists(self, doc_path_name: str) -> bool:
+        """
+        检查文档是否已存在
+        
+        Args:
+            doc_path_name (str): 文档路径
+            
+        Returns:
+            bool: 文档是否已存在
+        """
+        try:
+            # 先确保集合已加载
+            self._load_collection()
+            
+            # 根据路径查询
+            filter_expr = f'doc_path_name == "{doc_path_name}"'
+            
+            results = self.client.query(
+                collection_name=self.collection_name,
+                filter=filter_expr,
+                output_fields=[MilvusDocumentField.ID.value],
+                limit=1
+            )
+            
+            return len(results) > 0
+            
+        except Exception as e:
+            logger.error(f"检查文档是否存在失败: {e}")
+            raise e
+    
+    def delete_existing_document(self, doc_path_name: str) -> bool:
+        """
+        删除已存在的文档记录
+        
+        Args:
+            doc_path_name (str): 文档路径
+            
+        Returns:
+            bool: 删除是否成功
+        """
+        try:
+            # 安全检查：确保doc_path_name不为空，避免删除所有文档
+            if not doc_path_name or not doc_path_name.strip():
+                logger.error("错误: 文档路径名不能为空，拒绝执行删除操作")
+                return False
+            
+            # 构建删除表达式
+            delete_expr = f'doc_path_name == "{doc_path_name}"'
+            
+            # 执行删除操作
+            result = self.client.delete(
+                collection_name=self.collection_name,
+                filter=delete_expr
+            )
+            
+            logger.info(f"删除已存在的文档记录: {doc_path_name}")
+            return True
+            
+        except Exception as e:
+            logger.error(f"删除已存在文档失败: {e}")
+            raise e
+
+    def insert_document(self, doc_data: MilvusDocument) -> bool:
+        """插入文档数据"""
+        try:
+            self.client.insert(
+                collection_name=self.collection_name,
+                data= asdict(doc_data)
+            )
+            logger.info(f"文档 '{doc_data.doc_name}' 插入成功")
+            return True
+        except Exception as e:
+            logger.error(f"插入文档失败: {e}")
+            return False
+            
+    def flush_collection(self) -> bool:
+        """刷新集合"""
+        try:
+            self.client.flush(collection_name=self.collection_name)
+            logger.info(f"集合 '{self.collection_name}' 刷新成功")
+            return True
+        except Exception as e:
+            logger.error(f"刷新集合失败: {e}")
+            return False
+
+    
+    def search_similar_documents(self, query_vector, limit=10):
+        """搜索相似文档
+        
+        Args:
+            query_vector (list): 查询向量
+            limit (int): 返回结果数量限制
+            
+        Returns:
+            list: 搜索结果
+        """
+        try:
+            search_params = {
+                "metric_type": "COSINE",
+                "params": {"nprobe": 64}
+            }
+            
+            results = self.client.search(
+                collection_name=self.collection_name,
+                data=[query_vector],
+                anns_field=MilvusDocumentField.CONTENT_VECTOR.value,
+                search_params=search_params,
+                limit=limit,
+                output_fields=[MilvusDocumentField.DOC_NAME.value, MilvusDocumentField.DOC_PATH_NAME.value, MilvusDocumentField.DOC_TYPE.value, MilvusDocumentField.CONTENT.value, MilvusDocumentField.EMBEDDING_MODEL.value]
+            )
+            
+            return results[0] if results else []
+            
+        except Exception as e:
+            logger.error(f"搜索失败: {e}")
+            return []
+    
+    def get_collection_info(self):
+        """获取集合信息"""
+        try:
+            info = self.client.describe_collection(collection_name=self.collection_name)
+            return info
+        except Exception as e:
+            logger.error(f"获取集合信息失败: {e}")
+            return None
+
+
+# 使用示例
+if __name__ == "__main__":
+    # 创建MilvusAPI实例
+    milvus_api = MilvusAPI()
+    
+    
+    # 获取集合信息
+    info = milvus_api.get_collection_info()
+    if info:
+        pprint("集合信息:")
+        pprint(info)

mildoc_index/minio_event_handler.py

@@ -0,0 +1,384 @@
+import json
+from datetime import datetime
+from typing import Dict, Any
+import os
+from dotenv import load_dotenv
+
+from minio import Minio
+from parser.simple_object_parser import SimpleObjectParser
+from embedding import EmbeddingTool
+from milvus_api import MilvusAPI, MilvusDocument
+from logger.logging import setup_logging
+
+load_dotenv()
+
+logger = setup_logging()
+
+
+# Minio 配置信息。
+MINIO_BUCKET = os.getenv('MINIO_BUCKET')
+MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT')
+MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY')
+MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY')
+MINIO_REGION = os.getenv('MINIO_REGION')
+MINIO_USE_VIRTUAL_HOST = os.getenv('MINIO_USE_VIRTUAL_HOST', 'false').lower() == 'true'
+MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true'
+
+## 获取MinIO客户端
+def _get_minio_client() -> Minio:
+    client = Minio(
+        endpoint=MINIO_ENDPOINT,
+        access_key=MINIO_ACCESS_KEY,
+        secret_key=MINIO_SECRET_KEY,
+        secure=MINIO_USE_SSL,
+        region=MINIO_REGION,
+    )
+
+    if MINIO_USE_VIRTUAL_HOST:
+        client.enable_virtual_style_endpoint()
+        
+    return client
+
+class MinioEventHandler:
+    """Minio事件监听器"""
+    
+    def __init__(self, bucket_name: str = None):
+        """
+        初始化监听器
+        
+        Args:
+            bucket_name (str): 要监听的桶名称，默认从环境变量获取
+        """
+        self.bucket_name = bucket_name or os.getenv("MINIO_BUCKET", "mildoc")
+        
+        # 初始化各个组件
+        self.minio_client = _get_minio_client()
+
+        # 初始化解析器
+        logger.info("初始化解析器...")
+        self.parser: SimpleObjectParser = SimpleObjectParser(minio_client=self.minio_client)
+        
+        # 初始化Milvus
+        logger.info("初始化Milvus...")
+        self.milvus_api: MilvusAPI = MilvusAPI()
+        
+        # 测试embedding工具
+        logger.info("测试embedding工具...")
+        self.embedding_tool: EmbeddingTool = EmbeddingTool()
+
+        logger.info("所有组件初始化完成！")
+    
+    def _extract_event_info(self, event_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        从事件数据中提取关键信息
+        
+        Args:
+            event_data (Dict[str, Any]): 事件数据
+            
+        Returns:
+            Dict[str, Any]: 提取的信息
+        """
+        try:
+
+            #logger.info(f"event_data: {event_data}")
+            logger.info(f"数据: {json.dumps(event_data, ensure_ascii=False, indent=2)}")
+
+            record = event_data.get('Records', [{}])[0]
+            s3_info = record.get('s3', {})
+            
+            return {
+                'event_name': record.get('eventName', ''),
+                'event_time': record.get('eventTime', ''),
+                'bucket_name': s3_info.get('bucket', {}).get('name', ''),
+                'object_name': s3_info.get('object', {}).get('key', ''),
+                'object_size': s3_info.get('object', {}).get('size', 0),
+                'content_type': s3_info.get('object', {}).get('contentType', ''),
+                'etag': s3_info.get('object', {}).get('eTag', ''),
+            }
+        except Exception as e:
+            logger.error(f"提取事件信息失败: {e}")
+            return {}
+    
+    def _handle_object_created(self, event_info: Dict[str, Any]):
+        """
+        处理对象创建事件
+        
+        Args:
+            event_info (Dict[str, Any]): 事件信息
+        """
+        try:
+            bucket_name = event_info['bucket_name']
+            object_name = event_info['object_name']
+            
+            logger.info(f"\n=== 处理新增对象: {bucket_name}/{object_name} ===")
+            logger.info(f"对象大小: {event_info['object_size']} 字节")
+            logger.info(f"内容类型: {event_info['content_type']}")
+            
+            # 直接调用_process_single_object方法处理
+            self._process_single_object(bucket_name, object_name, force_update=True)
+            
+        except Exception as e:
+            logger.error(f"处理对象创建事件失败: {e}")
+    
+    def _handle_object_deleted(self, event_info: Dict[str, Any]):
+        """
+        处理对象删除事件
+        
+        Args:
+            event_info (Dict[str, Any]): 事件信息
+        """
+        try:
+            bucket_name = event_info['bucket_name']
+            object_name = event_info['object_name']
+            doc_path_name = object_name  # 不再包含bucket_name前缀
+            
+            logger.info(f"\n=== 处理删除对象: {bucket_name}/{object_name} ===")
+            
+            # 从Milvus中删除相关记录
+            logger.info("从Milvus中查找并删除相关记录...")
+            
+            # 使用MilvusAPI的删除方法
+            if self.milvus_api.delete_existing_document(doc_path_name):
+                logger.info(f"成功删除文档记录: {doc_path_name}")
+            else:
+                logger.error(f"删除文档记录失败: {doc_path_name}")
+            
+        except Exception as e:
+            logger.error(f"处理对象删除事件失败: {e}")
+    
+    def _process_event(self, event_data: Dict[str, Any]):
+        """
+        处理单个事件
+        
+        Args:
+            event_data (Dict[str, Any]): 事件数据
+        """
+        try:
+            # 提取事件信息
+            event_info = self._extract_event_info(event_data)
+            if not event_info:
+                logger.error("无法提取事件信息，跳过处理")
+                return
+            
+            event_name = event_info['event_name']
+            timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            
+            logger.info(f"\n[{timestamp}] 收到事件: {event_name}")
+            logger.info(f"对象: {event_info['bucket_name']}/{event_info['object_name']}")
+            
+            # 根据事件类型进行处理
+            if 'ObjectCreated' in event_name:
+                self._handle_object_created(event_info)
+            elif 'ObjectRemoved' in event_name:
+                self._handle_object_deleted(event_info)
+            else:
+                logger.error(f"未处理的事件类型: {event_name}")
+                
+        except Exception as e:
+            logger.error(f"处理事件时出错: {e}")
+    
+    def _process_single_object(self, bucket_name: str, object_name: str, force_update: bool = False):
+        """
+        处理单个对象（用于全量刷新和排查补漏）
+        
+        Args:
+            bucket_name (str): 桶名称
+            object_name (str): 对象名称
+            force_update (bool): 是否强制更新（True=全量刷新，False=排查补漏）
+        
+        Returns:
+            bool: 处理是否成功
+        """
+        try:
+            doc_path_name = object_name  # 不再包含bucket_name前缀
+            
+            # 如果是排查补漏模式，先检查是否已存在
+            if not force_update:
+                if self.milvus_api.check_document_exists(doc_path_name):
+                    logger.info(f"  文档已存在，跳过: {object_name}")
+                    return True
+            
+            logger.info(f"  处理文档: {object_name}")
+            
+            # 解析对象内容
+            parse_result = self.parser.parse_object(bucket_name, object_name)
+            
+            if 'error' in parse_result:
+                logger.error(f"    解析失败: {parse_result['error']}")
+                return False
+            
+            if not parse_result['contents']:
+                logger.error(f"    未提取到文本内容，跳过")
+                return True
+            
+            logger.info(f"    解析成功，获得 {len(parse_result['contents'])} 个文本片段")
+            
+            # 如果是强制更新，先删除已存在的记录
+            if force_update:
+                self.milvus_api.delete_existing_document(doc_path_name)
+            
+            # 为每个文本片段生成embedding并存储到Milvus
+            success_count = 0
+            for i, content in enumerate(parse_result['contents']):
+                try:
+                    # 生成embedding向量
+                    embedding_vector = self.embedding_tool.get_embedding(content)
+                    if not embedding_vector:
+                        logger.error(f"    片段 {i+1} embedding生成失败，跳过")
+                        continue
+                    
+                    # 准备文档数据
+                    doc_data = MilvusDocument(
+                        doc_name=parse_result['doc_name'],
+                        doc_path_name=parse_result['doc_path_name'],
+                        doc_type=parse_result['doc_type'],
+                        doc_md5=parse_result['doc_md5'],
+                        doc_length=parse_result['doc_length'],
+                        content=content,
+                        content_vector=embedding_vector,
+                        embedding_model=self.embedding_tool.model
+                    )
+                    
+                    # 存储到Milvus（允许重复，因为我们已经处理了去重逻辑）
+                    if self.milvus_api.insert_document(doc_data):
+                        success_count += 1
+                    else:
+                        logger.error(f"    片段 {i+1} 存储失败")
+                
+                except Exception as e:
+                    logger.error(f"    处理片段 {i+1} 时出错: {e}")
+                    continue
+            
+            logger.info(f"    完成！成功存储 {success_count}/{len(parse_result['contents'])} 个片段")
+            return success_count > 0
+            
+        except Exception as e:
+            logger.error(f"  处理对象失败: {e}")
+            return False
+    
+    def full_update(self):
+        """
+        模式1：全量刷新 - 遍历Minio桶中的所有数据并更新到Milvus
+        """
+        logger.info(f"\n=== 模式1：全量刷新 ===")
+        logger.info(f"正在遍历桶 '{self.bucket_name}' 中的所有对象...")
+        
+        try:
+            # 获取桶中的所有对象
+            objects = self.minio_client.list_objects(self.bucket_name, recursive=True)
+            
+            total_objects = 0
+            processed_objects = 0
+            
+            for obj in objects:
+                object_name = obj.object_name
+                
+                # 跳过文件夹
+                if object_name.endswith('/'):
+                    continue
+
+                total_objects += 1
+                
+                logger.info(f"\n[{total_objects}] 处理对象: {object_name}")
+                
+                if self._process_single_object(self.bucket_name, object_name, force_update=True):
+                    processed_objects += 1
+                    
+            self.milvus_api.flush_collection()
+            
+            logger.info(f"\n=== 全量刷新完成 ===")
+            logger.info(f"总对象数: {total_objects}")
+            logger.info(f"成功处理: {processed_objects}")
+            logger.info(f"失败数量: {total_objects - processed_objects}")
+            
+        except Exception as e:
+            logger.error(f"全量刷新失败: {e}")
+    
+    def backfill_update(self):
+        """
+        模式2：排查补漏 - 检查Milvus中不存在的文档并新增
+        """
+        logger.info(f"\n=== 模式2：排查补漏 ===")
+        logger.info(f"正在检查桶 '{self.bucket_name}' 中缺失的文档...")
+        
+        try:
+            # 获取桶中的所有对象
+            objects = self.minio_client.list_objects(self.bucket_name, recursive=True)
+            
+            total_objects = 0
+            new_objects = 0
+            existing_objects = 0
+            
+            for obj in objects:
+                object_name = obj.object_name
+                
+                # 跳过文件夹
+                if object_name.endswith('/'):
+                    continue
+                
+                total_objects += 1
+
+                logger.info(f"\n[{total_objects}] 检查对象: {object_name}")
+                
+                # 检查是否已存在
+                if self.milvus_api.check_document_exists(object_name):
+                    logger.info(f"  已存在，跳过")
+                    existing_objects += 1
+                else:
+                    logger.info(f"  不存在，开始处理...")
+                    if self._process_single_object(self.bucket_name, object_name, force_update=False):
+                        new_objects += 1
+            
+            self.milvus_api.flush_collection()
+            
+            logger.info(f"\n=== 排查补漏完成 ===")
+            logger.info(f"总对象数: {total_objects}")
+            logger.info(f"已存在: {existing_objects}")
+            logger.info(f"新增: {new_objects}")
+            logger.info(f"失败数量: {total_objects - existing_objects - new_objects}")
+            
+        except Exception as e:
+            logger.error(f"排查补漏失败: {e}")
+    
+    def start_listening(self):
+        """
+        模式3：增量更新 - 根据消息通知进行增量更新
+        """
+        logger.info(f"\n=== 模式3：增量更新 ===")
+        logger.info(f"开始监听桶 '{self.bucket_name}' 的事件...")
+        logger.info("按 Ctrl+C 停止监听")
+        
+        try:
+            # 监听桶事件
+            events = self.minio_client.listen_bucket_notification(
+                bucket_name=self.bucket_name,
+                events=['s3:ObjectCreated:*', 's3:ObjectRemoved:*']
+            )
+            
+            for event in events:
+                try:
+                    if event:
+                        # 解析事件数据
+                        if isinstance(event, bytes):
+                            event_data = json.loads(event.decode('utf-8'))
+                        elif isinstance(event, str):
+                            event_data = json.loads(event)
+                        elif isinstance(event, dict):
+                            event_data = event
+                        else:
+                            logger.error(f"未知的事件数据类型: {type(event)}")
+                            continue
+                        
+                        # 处理事件
+                        self._process_event(event_data)
+                        
+                except json.JSONDecodeError as e:
+                    logger.error(f"解析事件数据失败: {e}")
+                except Exception as e:
+                    logger.error(f"处理事件失败: {e}")
+                    
+        except KeyboardInterrupt:
+            logger.info("\n监听已停止")
+        except Exception as e:
+            logger.error(f"监听过程中出错: {e}")
+ 

mildoc_index/oss_event_handler.py

@@ -0,0 +1,490 @@
+import json
+from datetime import datetime
+import time
+from enum import Enum
+from typing import Dict, Any, Optional, Callable
+import os
+from dotenv import load_dotenv
+from dataclasses import dataclass
+
+from minio import Minio
+from mns.account import Account
+from parser.simple_object_parser import SimpleObjectParser
+from embedding import EmbeddingTool
+from milvus_api import MilvusAPI, MilvusDocument
+from logger.logging import setup_logging
+
+load_dotenv()
+
+logger = setup_logging()
+
+# Minio 配置，因为兼容阿里云 OSS，所以使用同一套MinIO客户端。
+MINIO_BUCKET = os.getenv('MINIO_BUCKET')
+MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT')
+MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY')
+MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY')
+MINIO_REGION = os.getenv('MINIO_REGION')
+MINIO_USE_VIRTUAL_HOST = os.getenv('MINIO_USE_VIRTUAL_HOST', 'false').lower() == 'true'
+MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true'
+
+## 获取MinIO客户端
+def _get_minio_client() -> Minio:
+    client = Minio(
+        endpoint=MINIO_ENDPOINT,
+        access_key=MINIO_ACCESS_KEY,
+        secret_key=MINIO_SECRET_KEY,
+        secure=MINIO_USE_SSL,
+        region=MINIO_REGION,
+    )
+
+    if MINIO_USE_VIRTUAL_HOST:
+        client.enable_virtual_style_endpoint()
+        
+    return client
+
+
+## 配置阿里云消息队列
+MNS_ACCESS_KEY_ID = os.getenv("MNS_ACCESS_KEY_ID")
+MNS_ACCESS_KEY_SECRET = os.getenv("MNS_ACCESS_KEY_SECRET")
+MNS_ENDPOINT = os.getenv("MNS_ENDPOINT") 
+MNS_QUEUE_NAME = os.getenv("MNS_QUEUE_NAME", "mildoc-oss-notify")
+
+class OSSEventType(Enum):
+    """OSS 事件类型"""
+    OBJECT_CREATED = "ObjectCreated"  # 对象创建（上传）
+    OBJECT_REMOVED = "ObjectRemoved"  # 对象删除
+
+@dataclass
+class OSSEvent:
+    """OSS 事件数据结构"""
+    event_name: str  # 事件名称，如 "ObjectCreated:PutObject"
+    event_time: str  # 事件时间
+    bucket_name: str  # Bucket 名称
+    object_key: str  # 对象键（路径）
+    object_size: Optional[int] = None  # 对象大小（字节）
+    etag: Optional[str] = None  # 对象 ETag
+
+
+class OSSEventNotifier:
+    """OSS 事件通知管理器"""
+    
+    def __init__(self):
+        """初始化 MNS 队列"""
+        if not MNS_ENDPOINT or not MNS_ACCESS_KEY_ID or not MNS_ACCESS_KEY_SECRET:
+            logger.error(f"✗ MNS 队列初始化失败: MNS_ENDPOINT or MNS_ACCESS_KEY_ID or MNS_ACCESS_KEY_SECRET is not set")
+            raise Exception("MNS_ENDPOINT or MNS_ACCESS_KEY_ID or MNS_ACCESS_KEY_SECRET is not set")
+        
+            
+        mns_account = Account(MNS_ENDPOINT, MNS_ACCESS_KEY_ID, MNS_ACCESS_KEY_SECRET)
+        self.mildoc_queue = mns_account.get_queue(MNS_QUEUE_NAME)
+        logger.info(f"✓ MNS 队列初始化成功 (endpoint: {MNS_ENDPOINT})")
+        
+    def listen_mns_queue(
+        self,
+        handler: Callable[[Dict], None],
+        poll_interval: int = 5
+    ):
+        """
+        监听 MNS 队列，接收 OSS 事件通知。
+        
+        :param handler: 事件处理函数，接收 Dict 对象
+        :param poll_interval: 轮询间隔（秒）
+        """
+        if not self.mildoc_queue:
+            logger.error("✗ MNS 客户端未初始化，无法监听队列")
+            return
+        
+        logger.info(f"开始监听 MNS 队列: {MNS_QUEUE_NAME}")
+        logger.info("按 Ctrl+C 停止监听")
+        
+        try:
+            
+            while True:
+                try:
+                    # 接收消息（长轮询，等待最多 10 秒）
+                    try:
+                        msg = self.mildoc_queue.receive_message_with_str_body(wait_seconds=10)
+                    except Exception as e:
+                        # 如果没有消息，会抛出异常，继续轮询
+                        if "MessageNotExist" in str(e) or "not found" in str(e).lower():
+                            time.sleep(poll_interval)
+                            continue
+                        else:
+                            logger.error(f"✗ 接收消息失败: {e}")
+                            time.sleep(poll_interval)
+                            continue
+                    
+                    if msg:
+                        try:
+                            # 解析 OSS 事件
+                            # MNS 消息体可能是字符串，需要先解析
+                            message_body = msg.message_body
+                            if isinstance(message_body, str):
+                                event_data = json.loads(message_body)
+                            else:
+                                event_data = message_body
+                            
+                            #logger.info(f"event_data: {event_data}")
+                            logger.info(f"数据: {json.dumps(event_data, ensure_ascii=False, indent=2)}")
+                            
+                            if event_data:
+                                
+                                # 调用处理函数
+                                if handler:
+                                    handler(event_data)
+                                
+                                # 删除消息（确认已处理）
+                                self.mildoc_queue.delete_message(msg.receipt_handle)
+                                logger.info("  ✓ 消息已处理并删除")
+                            else:
+                                logger.warning(f"⚠ 无法解析事件数据: {message_body}")
+                                # 即使无法解析，也删除消息，避免重复处理
+                                self.mildoc_queue.delete_message(msg.receipt_handle)
+                                
+                        except json.JSONDecodeError as e:
+                            logger.error(f"✗ 解析消息失败: {e}, 消息内容: {msg.message_body}")
+                            # 删除无法解析的消息，避免阻塞队列
+                            try:
+                                self.mildoc_queue.delete_message(msg.receipt_handle)
+                            except:
+                                logger.error(f"✗ 删除消息失败: {e}")
+                                pass
+                        except Exception as e:
+                            logger.error(f"✗ 处理消息失败: {e}")
+                    
+                    time.sleep(poll_interval)
+                    
+                except KeyboardInterrupt:
+                    logger.info("\n停止监听...")
+                    break
+                except Exception as e:
+                    logger.error(f"✗ 接收消息失败: {e}")
+                    time.sleep(poll_interval)
+                    
+        except Exception as e:
+            logger.error(f"✗ 监听队列失败: {e}")
+    
+
+
+
+
+class OSSEventHandler:
+    """OSS 事件处理器"""
+    
+    def __init__(self, bucket_name: str = None):
+        """
+        初始化监听器
+        
+        Args:
+            bucket_name (str): 要监听的桶名称，默认从环境变量获取
+        """
+        self.bucket_name = bucket_name or os.getenv("OSS_BUCKET", "mildoc")
+        
+        # 初始化各个组件
+        self.minio_client = _get_minio_client()
+        
+        # 初始化解析器
+        logger.info("初始化解析器...")
+        self.parser: SimpleObjectParser = SimpleObjectParser(minio_client=self.minio_client)
+        
+        # 初始化Milvus
+        logger.info("初始化Milvus...")
+        self.milvus_api: MilvusAPI = MilvusAPI()
+        
+        # 测试embedding工具
+        logger.info("测试embedding工具...")
+        self.embedding_tool: EmbeddingTool = EmbeddingTool()
+
+        # OSS 事件监听
+        logger.info("初始化OSS事件监听...")
+        self.oss_event_notifier: OSSEventNotifier = OSSEventNotifier()
+
+        logger.info("所有组件初始化完成！")
+
+    def _process_event(self, event_data: Dict[str, Any]):
+        """
+        处理单个事件
+        
+        Args:
+            event_data (Dict[str, Any]): 事件数据
+        """
+        try:
+            # 提取事件信息
+            event_info = self._extract_event_info(event_data)
+            if not event_info:
+                logger.error("无法提取事件信息，跳过处理")
+                return
+            
+            event_name = event_info.event_name
+            timestamp = event_info.event_time
+            
+            logger.info(f"\n[{timestamp}] 收到事件: {event_name}")
+            logger.info(f"对象: {event_info.bucket_name}/{event_info.object_key}")
+            
+            # 根据事件类型进行处理
+            if OSSEventType.OBJECT_CREATED.value in event_name:
+                self._handle_object_created(event_info)
+            elif OSSEventType.OBJECT_REMOVED.value in event_name:
+                self._handle_object_deleted(event_info)
+            else:
+                logger.error(f"未处理的事件类型: {event_name}")
+                
+        except Exception as e:
+            logger.error(f"处理事件时出错: {e}")
+  
+
+    def _extract_event_info(self, event_data: Dict) -> Optional[OSSEvent]:
+        """解析 OSS 事件数据"""
+        try:
+            # OSS 事件通知格式
+            events = event_data.get("events", [])
+            if not events:
+                return None
+            
+            event = events[0]
+            event_name = event.get("eventName", "")
+            event_time = event.get("eventTime", "")
+            
+            # 获取对象信息
+            oss_obj = event.get("oss", {}).get("object", {})
+            bucket = event.get("oss", {}).get("bucket", {})
+            
+            return OSSEvent(
+                event_name=event_name,
+                event_time=event_time,
+                bucket_name=bucket.get("name", ""),
+                object_key=oss_obj.get("key", ""),
+                object_size=oss_obj.get("size"),
+                etag=oss_obj.get("etag"),
+            )
+        except Exception as e:
+            logger.error(f"✗ 解析事件数据失败: {e}")
+            return None
+    
+    def _handle_object_created(self, event_info: OSSEvent):
+        """
+        处理对象创建事件
+        
+        Args:
+            event_info (Dict[str, Any]): 事件信息
+        """
+        try:
+            bucket_name = event_info.bucket_name
+            object_name = event_info.object_key
+            
+            logger.info(f"\n=== 处理新增对象: {bucket_name}/{object_name} ===")
+            logger.info(f"对象大小: {event_info.object_size} 字节")
+            
+            # 直接调用_process_single_object方法处理
+            self._process_single_object(bucket_name, object_name, force_update=True)
+            
+        except Exception as e:
+            logger.error(f"处理对象创建事件失败: {e}")
+    
+    def _handle_object_deleted(self, event_info: OSSEvent):
+        """
+        处理对象删除事件
+        
+        Args:
+            event_info (Dict[str, Any]): 事件信息
+        """
+        try:
+            bucket_name = event_info.bucket_name
+            object_name = event_info.object_key
+            doc_path_name = object_name  # 不再包含bucket_name前缀
+            
+            logger.info(f"\n=== 处理删除对象: {bucket_name}/{object_name} ===")
+            
+            # 从Milvus中删除相关记录
+            logger.info("从Milvus中查找并删除相关记录...")
+            
+            # 使用MilvusAPI的删除方法
+            if self.milvus_api.delete_existing_document(doc_path_name):
+                logger.info(f"成功删除文档记录: {doc_path_name}")
+            else:
+                logger.error(f"删除文档记录失败: {doc_path_name}")
+            
+        except Exception as e:
+            logger.error(f"处理对象删除事件失败: {e}")
+    
+    
+    def _process_single_object(self, bucket_name: str, object_name: str, force_update: bool = False):
+        """
+        处理单个对象（用于全量刷新和排查补漏）
+        
+        Args:
+            bucket_name (str): 桶名称
+            object_name (str): 对象名称
+            force_update (bool): 是否强制更新（True=全量刷新，False=排查补漏）
+        
+        Returns:
+            bool: 处理是否成功
+        """
+        try:
+            doc_path_name = object_name  # 不再包含bucket_name前缀
+            
+            # 如果是排查补漏模式，先检查是否已存在
+            if not force_update:
+                if self.milvus_api.check_document_exists(doc_path_name):
+                    logger.info(f"  文档已存在，跳过: {object_name}")
+                    return True
+            
+            logger.info(f"  处理文档: {object_name}")
+            
+            # 解析对象内容
+            parse_result = self.parser.parse_object(bucket_name, object_name)
+            
+            if 'error' in parse_result:
+                logger.error(f"    解析失败: {parse_result['error']}")
+                return False
+            
+            if not parse_result['contents']:
+                logger.error(f"    未提取到文本内容，跳过")
+                return True
+            
+            logger.info(f"    解析成功，获得 {len(parse_result['contents'])} 个文本片段")
+            
+            # 如果是强制更新，先删除已存在的记录
+            if force_update:
+                self.milvus_api.delete_existing_document(doc_path_name)
+            
+            # 为每个文本片段生成embedding并存储到Milvus
+            success_count = 0
+            for i, content in enumerate(parse_result['contents']):
+                try:
+                    # 生成embedding向量
+                    embedding_vector = self.embedding_tool.get_embedding(content)
+                    if not embedding_vector:
+                        logger.error(f"    片段 {i+1} embedding生成失败，跳过")
+                        continue
+                    
+                    # 准备文档数据
+                    doc_data = MilvusDocument(
+                        doc_name=parse_result['doc_name'],
+                        doc_path_name=parse_result['doc_path_name'],
+                        doc_type=parse_result['doc_type'],
+                        doc_md5=parse_result['doc_md5'],
+                        doc_length=parse_result['doc_length'],
+                        content=content,
+                        content_vector=embedding_vector,
+                        embedding_model=self.embedding_tool.model
+                    )
+                    
+                    # 存储到Milvus（允许重复，因为我们已经处理了去重逻辑）
+                    if self.milvus_api.insert_document(doc_data):
+                        success_count += 1
+                    else:
+                        logger.error(f"    片段 {i+1} 存储失败")
+                
+                except Exception as e:
+                    logger.error(f"    处理片段 {i+1} 时出错: {e}")
+                    continue
+            
+            logger.info(f"    完成！成功存储 {success_count}/{len(parse_result['contents'])} 个片段")
+            return success_count > 0
+            
+        except Exception as e:
+            logger.error(f"  处理对象失败: {e}")
+            return False
+    
+    def full_update(self):
+        """
+        模式1：全量刷新 - 遍历Minio桶中的所有数据并更新到Milvus
+        """
+        logger.info(f"\n=== 模式1：全量刷新 ===")
+        logger.info(f"正在遍历桶 '{self.bucket_name}' 中的所有对象...")
+        
+        try:
+            # 获取桶中的所有对象
+            objects = self.minio_client.list_objects(self.bucket_name, recursive=True)
+            
+            total_objects = 0
+            processed_objects = 0
+            
+            for obj in objects:
+                object_name = obj.object_name
+                
+                # 跳过文件夹
+                if object_name.endswith('/'):
+                    continue
+                
+                total_objects += 1
+
+                logger.info(f"\n[{total_objects}] 处理对象: {object_name}")
+                
+                if self._process_single_object(self.bucket_name, object_name, force_update=True):
+                    processed_objects += 1
+                    
+            self.milvus_api.flush_collection()
+            
+            logger.info(f"\n=== 全量刷新完成 ===")
+            logger.info(f"总对象数: {total_objects}")
+            logger.info(f"成功处理: {processed_objects}")
+            logger.info(f"失败数量: {total_objects - processed_objects}")
+            
+        except Exception as e:
+            logger.error(f"全量刷新失败: {e}")
+    
+    def backfill_update(self):
+        """
+        模式2：排查补漏 - 检查Milvus中不存在的文档并新增
+        """
+        logger.info(f"\n=== 模式2：排查补漏 ===")
+        logger.info(f"正在检查桶 '{self.bucket_name}' 中缺失的文档...")
+        
+        try:
+            # 获取桶中的所有对象
+            objects = self.minio_client.list_objects(self.bucket_name, recursive=True)
+            
+            total_objects = 0
+            new_objects = 0
+            existing_objects = 0
+            
+            for obj in objects:
+                object_name = obj.object_name
+                
+                # 跳过文件夹
+                if object_name.endswith('/'):
+                    continue
+                
+                total_objects += 1
+
+                logger.info(f"\n[{total_objects}] 检查对象: {object_name}")
+                
+                # 检查是否已存在
+                if self.milvus_api.check_document_exists(object_name):
+                    logger.info(f"  已存在，跳过")
+                    existing_objects += 1
+                else:
+                    logger.info(f"  不存在，开始处理...")
+                    if self._process_single_object(self.bucket_name, object_name, force_update=False):
+                        new_objects += 1
+            
+            self.milvus_api.flush_collection()
+            
+            logger.info(f"\n=== 排查补漏完成 ===")
+            logger.info(f"总对象数: {total_objects}")
+            logger.info(f"已存在: {existing_objects}")
+            logger.info(f"新增: {new_objects}")
+            logger.info(f"失败数量: {total_objects - existing_objects - new_objects}")
+            
+        except Exception as e:
+            logger.error(f"排查补漏失败: {e}")
+    
+    def start_listening(self):
+        """
+        模式3：增量更新 - 根据消息通知进行增量更新
+        """
+        logger.info(f"\n=== 模式3：增量更新 ===")
+        logger.info(f"开始监听桶 '{self.bucket_name}' 的事件...")
+        logger.info("按 Ctrl+C 停止监听")
+        
+        try:
+            # 监听桶事件
+            self.oss_event_notifier.listen_mns_queue(self._process_event)
+                
+        except KeyboardInterrupt:
+            logger.info("\n监听已停止")
+        except Exception as e:
+            logger.error(f"监听过程中出错: {e}")
+ 

mildoc_index/parser/document_parser.py

@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+
+class DocumentParser(ABC):
+    """文档解析器抽象基类"""
+    
+    @abstractmethod
+    def parse(self, data: bytes) -> str:
+        """
+        解析文档内容
+        
+        Args:
+            data (bytes): 文档二进制数据
+            
+        Returns:
+            str: 解析出的文本内容
+        """
+        pass
+    
+    @abstractmethod
+    def supports(self, content_type: str) -> bool:
+        """
+        检查是否支持指定的内容类型
+        
+        Args:
+            content_type (str): 内容类型
+            
+        Returns:
+            bool: 是否支持
+        """
+        pass

mildoc_index/parser/markdown_parser.py

@@ -0,0 +1,46 @@
+import sys, os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from parser.document_parser import DocumentParser
+from logger.logging import setup_logging
+
+logger = setup_logging()
+
+class MarkdownParser(DocumentParser):
+    """Markdown文档解析器"""
+    
+    def parse(self, data: bytes) -> str:
+        """解析Markdown文档，直接返回原始内容"""
+        try:
+            # 尝试不同的编码
+            encodings = ['utf-8', 'gbk', 'gb2312', 'latin-1']
+            for encoding in encodings:
+                try:
+                    return data.decode(encoding)
+                except UnicodeDecodeError:
+                    continue
+            
+            # 如果所有编码都失败，使用错误处理
+            return data.decode('utf-8', errors='ignore')
+            
+        except Exception as e:
+            logger.error(f"Markdown解析失败: {e}")
+            return ""
+    
+    def supports(self, content_type: str) -> bool:
+        """检查是否支持Markdown格式"""
+        supported_types = [
+            'text/markdown',
+            'text/x-markdown', 
+            'application/markdown',
+            'md'
+        ]
+        return content_type.lower() in [t.lower() for t in supported_types]
+
+
+if __name__ == "__main__":
+    parser = MarkdownParser()
+    with open("../data/md1.md", "rb") as f:
+        data = f.read()
+    result = parser.parse(data)
+    print(result[:2000])

mildoc_index/parser/mineru_parser.py

@@ -0,0 +1,80 @@
+import os, sys
+from document_parser import DocumentParser
+from dotenv import load_dotenv
+# 导入MinerU相关模块
+from mineru.cli.common import convert_pdf_bytes_to_bytes_by_pypdfium2
+from mineru.backend.vlm.vlm_analyze import doc_analyze as vlm_doc_analyze
+from mineru.backend.vlm.vlm_middle_json_mkcontent import union_make as vlm_union_make
+from mineru.utils.enum_class import MakeMode
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from logger.logging import setup_logging
+
+load_dotenv()
+
+logger = setup_logging()
+
+class MinerUParser(DocumentParser):
+    """MinerU文档解析器，专门用于解析PDF文档"""
+    
+    def __init__(self):
+        """
+        初始化MinerU解析器
+        
+        Args:
+            server_url (str): MinerU服务器地址，默认从环境变量或使用默认值
+        """
+        self.server_url = os.getenv("MINERU_SERVER_URL")
+        if not self.server_url:
+            raise ValueError("MINERU_SERVER_URL 环境变量未设置")
+
+        logger.info(f"MinerU服务器地址: {self.server_url}")
+    
+    def parse(self, data: bytes) -> str:
+        """解析PDF文档"""
+        try:
+            logger.info("正在使用MinerU解析PDF文档...")
+            
+            # 转换PDF字节数据
+            logger.info("正在转换PDF字节数据...")
+            pdf_bytes = convert_pdf_bytes_to_bytes_by_pypdfium2(data, 0, None)
+            
+            # 使用VLM分析PDF
+            logger.info("正在分析PDF文档...")
+            middle_json, infer_result = vlm_doc_analyze(
+                pdf_bytes, 
+                backend="http-client", 
+                server_url=self.server_url, 
+                image_writer=None
+            )
+            
+            # 获取PDF信息
+            pdf_info = middle_json["pdf_info"]
+            
+            # 生成Markdown内容
+            logger.info("正在生成Markdown内容...")
+            md_content_str = vlm_union_make(pdf_info, MakeMode.MM_MD, None)
+            
+            logger.info(f"MinerU解析完成，生成了{len(md_content_str)}字符的Markdown内容")
+            return md_content_str.strip()
+            
+        except ImportError as e:
+            logger.error(f"MinerU模块导入失败: {e}")
+            logger.error("请确保已正确安装MinerU相关依赖")
+            return ""
+        except Exception as e:
+            logger.error(f"MinerU解析失败: {e}")
+            return ""
+    
+    def supports(self, content_type: str) -> bool:
+        """检查是否支持PDF格式"""
+        return content_type.lower() in ['application/pdf', 'pdf']
+
+
+if __name__ == "__main__":
+    parser = MinerUParser()
+    with open("../data/pdf3.pdf", "rb") as f:
+        data = f.read()
+    result = parser.parse(data)
+    print("-"*100)
+    print(result)

mildoc_index/parser/office_parser.py

@@ -0,0 +1,68 @@
+import os, sys
+from io import BytesIO
+from markitdown import MarkItDown
+
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from parser.document_parser import DocumentParser
+from logger.logging import setup_logging
+
+logger = setup_logging()
+
+
+
+
+class OfficeParser(DocumentParser):
+    """Office文档解析器，使用markitdown"""
+    
+    def __init__(self):
+        """初始化markitdown实例"""
+        self.markitdown = MarkItDown(enable_plugins=False)
+    
+    def parse(self, data: bytes) -> str:
+        """解析Office文档"""
+        try:
+            # 使用BytesIO创建文件类对象
+            file_stream = BytesIO(data)
+            
+            # 使用markitdown的convert_stream方法解析
+            result = self.markitdown.convert_stream(file_stream)
+            
+            if result and hasattr(result, 'text_content'):
+                return result.text_content.strip()
+            else:
+                logger.error("markitdown解析结果为空或格式异常")
+                return ""
+                
+        except Exception as e:
+            logger.error(f"Office文档解析失败: {e}")
+            return ""
+    
+    def supports(self, content_type: str) -> bool:
+        """检查是否支持Office文档格式"""
+        supported_types = [
+            # Word文档
+            'application/msword',  # .doc
+            'application/vnd.openxmlformats-officedocument.wordprocessingml.document',  # .docx
+            
+            # Excel文档
+            'application/vnd.ms-excel',  # .xls
+            'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',  # .xlsx
+            
+            # PowerPoint文档
+            'application/vnd.ms-powerpoint',  # .ppt
+            'application/vnd.openxmlformats-officedocument.presentationml.presentation',  # .pptx
+            
+            # PDF (markitdown也支持PDF)
+            'application/pdf',
+        ]
+        
+        return content_type.lower() in [t.lower() for t in supported_types]
+
+
+if __name__ == "__main__":
+    parser = OfficeParser()
+    with open("../data/msdoc3.docx", "rb") as f:
+        data = f.read()
+    result = parser.parse(data)
+    print(result[:2000])

mildoc_index/parser/pdf_parser.py

@@ -0,0 +1,49 @@
+from io import BytesIO
+from PyPDF2 import PdfReader
+
+import sys, os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from parser.document_parser import DocumentParser
+from logger.logging import setup_logging
+
+logger = setup_logging()
+
+class PDFParser(DocumentParser):
+    """PDF文档解析器"""
+    
+    def parse(self, data: bytes) -> str:
+        """解析PDF文档"""
+        try:
+            reader = PdfReader(BytesIO(data))
+            text_content = ""
+            
+            for page_num in range(len(reader.pages)):
+                page = reader.pages[page_num]
+                page_text = page.extract_text()
+                if page_text:
+                    text_content += page_text + "\n"
+            
+            return text_content.strip()
+            
+        except Exception as e:
+            logger.error(f"PDF解析失败: {e}")
+            return ""
+    
+    def supports(self, content_type: str) -> bool:
+        """检查是否支持PDF"""
+        return content_type.lower() in ['application/pdf', 'pdf']
+
+
+if __name__ == "__main__":
+    parser = PDFParser()
+
+    file_path = ["../data/pdf1.pdf", "../data/pdf2.pdf", "../data/pdf3.pdf", "../data/pdf4.pdf"]
+    for path in file_path:
+        with open(path, "rb") as f:
+            data = f.read()
+            result = parser.parse(data)
+            print("\n\n")
+            print(f"#################   文件路径: {path}    #################")
+            print(result)
+

mildoc_index/parser/simple_object_parser.py

@@ -0,0 +1,340 @@
+from minio import Minio
+from dotenv import load_dotenv
+import os, sys
+import hashlib
+import argparse
+from typing import List, Dict, Any, Optional
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from parser.document_parser import DocumentParser
+from parser.office_parser import OfficeParser
+from parser.pdf_parser import PDFParser
+from parser.markdown_parser import MarkdownParser
+from parser.text_parser import TextParser
+
+from logger.logging import setup_logging
+
+load_dotenv()
+logger = setup_logging()
+
+class SimpleObjectParser:
+    """简单对象解析器"""
+    
+    def __init__(self,minio_client: Minio, chunk_size: int = 2048, overlap_size: int = 128):
+        """
+        初始化解析器
+        
+        Args:
+            chunk_size (int): 文本片段最大长度，默认2048
+            overlap_size (int): 重叠区域大小，默认128
+        """
+        self.chunk_size = chunk_size
+        self.overlap_size = overlap_size
+        
+        # 初始化Minio客户端
+        self.minio_client = minio_client
+        
+        # 注册解析器（按优先级排序）
+        self.parsers = [
+            PDFParser(),             # PyPDF2作为PDF解析器
+            OfficeParser(),          # markitdown解析Office文档（Word、Excel、PowerPoint）
+            #MinerUParser(),          # MinerU解析器，专门处理PDF，暂不开启
+            MarkdownParser(),        # Markdown解析器，
+            TextParser(),            # 纯文本解析器作为最后的备选
+        ]
+    
+    def add_parser(self, parser: DocumentParser):
+        """
+        添加新的解析器
+        
+        Args:
+            parser (DocumentParser): 文档解析器实例
+        """
+        self.parsers.append(parser)
+    
+    def _get_parser(self, content_type: str) -> Optional[DocumentParser]:
+        """
+        根据内容类型获取合适的解析器
+        
+        Args:
+            content_type (str): 内容类型
+            
+        Returns:
+            Optional[DocumentParser]: 解析器实例，如果没有找到则返回None
+        """
+        for parser in self.parsers:
+            if parser.supports(content_type):
+                return parser
+        return None
+    
+    def _extract_doc_name(self, object_path: str) -> str:
+        """
+        从对象路径中提取文档名称
+        
+        Args:
+            object_path (str): 对象路径
+            
+        Returns:
+            str: 文档名称
+        """
+        return os.path.basename(object_path)
+    
+    def _extract_doc_type(self, content_type: str) -> str:
+        """
+        从content-type中提取文档类型
+        
+        Args:
+            content_type (str): 内容类型
+            
+        Returns:
+            str: 文档类型
+        """
+        if not content_type:
+            return "unknown"
+        
+        # 提取主要类型
+        main_type = content_type.split('/')[0].lower()
+        sub_type = content_type.split('/')[-1].lower()
+        
+        # 映射常见类型
+        type_mapping = {
+            'application/pdf': 'pdf',
+            'text/plain': 'txt',
+            'text/html': 'html',
+            'text/markdown': 'md',
+            'text/x-markdown': 'md',
+            'application/markdown': 'md',
+            
+            # Word文档
+            'application/msword': 'doc',
+            'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx',
+            
+            # Excel文档
+            'application/vnd.ms-excel': 'xls',
+            'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx',
+            
+            # PowerPoint文档
+            'application/vnd.ms-powerpoint': 'ppt',
+            'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'pptx',
+        }
+        
+        return type_mapping.get(content_type.lower(), sub_type)
+    
+    def _calculate_md5(self, data: bytes) -> str:
+        """
+        计算数据的MD5值
+        
+        Args:
+            data (bytes): 二进制数据
+            
+        Returns:
+            str: MD5值
+        """
+        return hashlib.md5(data).hexdigest()
+    
+    def _split_text_by_langchain(self, text: str) -> List[str]:
+        """
+        使用LangChain分割文本
+        """
+
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=self.chunk_size, chunk_overlap=self.overlap_size)
+        return text_splitter.split_text(text)
+    
+    def parse_object(self, bucket_name: str, object_name: str) -> Dict[str, Any]:
+        """
+        解析简单对象
+        
+        Args:
+            bucket_name (str): 存储桶名称
+            object_path (str): 对象路径
+            
+        Returns:
+            Dict[str, Any]: 解析结果
+        """
+        try:
+            # 先获取对象信息，检查文件大小
+            logger.info(f"正在检查对象信息: {bucket_name}/{object_name}")
+            try:
+                stat = self.minio_client.stat_object(bucket_name, object_name)
+                file_size = stat.size
+                max_size = 512 * 1024 * 1024  # 512MB
+                
+                logger.info(f"对象大小: {file_size} 字节 ({file_size / 1024 / 1024:.2f} MB)")
+                
+                if file_size > max_size:
+                    logger.info(f"文件过大 ({file_size / 1024 / 1024:.2f} MB > 512 MB)，跳过解析")
+                    return None
+            except Exception as e:
+                logger.info(f"获取对象信息失败: {e}")
+                # 如果无法获取文件信息，继续尝试解析
+            
+            # 从Minio获取对象
+            logger.info(f"正在获取对象内容: {bucket_name}/{object_name}")
+            response = self.minio_client.get_object(bucket_name, object_name)
+            
+            # 获取对象数据和元数据
+            data = response.data
+            headers = response.headers
+            
+            logger.info(f"对象大小: {len(data)} 字节")
+            logger.info(f"Content-Type: {headers.get('Content-Type', 'unknown')}")
+            
+            # 提取基本信息
+            doc_name = self._extract_doc_name(object_name)
+            doc_path_name = object_name  # 不再包含bucket_name前缀
+            content_type = headers.get('Content-Type', '')
+            doc_type = self._extract_doc_type(content_type)
+            doc_md5 = headers.get('ETag', '').strip('"')
+            if len(doc_md5) != 32:  # 如果ETag不是32位，则重新计算MD5, 多部分上传：ETag = {复合MD5}-{部分数量}（超过32字符）
+                doc_md5 = self._calculate_md5(data)
+            #doc_md5 = self._calculate_md5(data)
+            doc_length = int(headers.get('Content-Length', len(data)))
+            
+            # 选择合适的解析器
+            parser = self._get_parser(content_type)
+            if not parser:
+                logger.info(f"警告: 未找到适合 {content_type} 的解析器，尝试使用文本解析器")
+                parser = TextParser()
+            
+            # 解析文档内容
+            logger.info(f"使用解析器: {parser.__class__.__name__}")
+            text_content = parser.parse(data)
+            
+            if not text_content:
+                logger.info("警告: 未提取到文本内容")
+                contents = []
+            else:
+                logger.info(f"提取到文本: {len(text_content)} 字符")
+                # 分割文本为片段
+                contents = self._split_text_by_langchain(text_content)
+
+                logger.info(f"分割为 {len(contents)} 个片段")
+            
+            return {
+                "doc_name": doc_name,
+                "doc_path_name": doc_path_name,
+                "doc_type": doc_type,
+                "doc_md5": doc_md5,
+                "doc_length": doc_length,
+                "contents": contents
+            }
+            
+        except Exception as e:
+            logger.info(f"解析对象失败: {e}")
+            return None
+        finally:
+            # 确保响应被正确关闭
+            if 'response' in locals():
+                response.close()
+                response.release_conn()
+    
+    def get_parser_info(self) -> List[str]:
+        """
+        获取已注册的解析器信息
+        
+        Returns:
+            List[str]: 解析器类名列表
+        """
+        return [parser.__class__.__name__ for parser in self.parsers]
+
+
+# 使用示例和测试
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='简单对象解析器 - 支持多种文档格式解析')
+    
+    # 基本参数
+    parser.add_argument('--bucket', type=str, default='test', help='Minio桶名称 (默认: test)')
+    parser.add_argument('--file', type=str, action='append', help='要解析的文件路径 (可指定多个文件)')
+    parser.add_argument('--test', action='store_true', help='运行默认测试案例')
+    
+    # 解析器配置
+    parser.add_argument('--chunk-size', type=int, default=2048, help='文本片段最大长度 (默认: 2048)')
+    parser.add_argument('--overlap-size', type=int, default=128, help='重叠区域大小 (默认: 128)')
+    
+    # 输出选项
+    parser.add_argument('--verbose', action='store_true', help='显示详细信息')
+    
+    args = parser.parse_args()
+    
+    print("=== 简单对象解析器 ===")
+    
+    # 创建解析器实例
+    object_parser = SimpleObjectParser(
+        chunk_size=args.chunk_size, 
+        overlap_size=args.overlap_size,
+    )
+    
+    # 显示已注册的解析器
+    print(f"已注册的解析器: {object_parser.get_parser_info()}")
+    
+    results = []
+    
+    if args.test:
+        # 运行默认测试案例
+        print("\n运行默认测试案例...")
+        test_files = [
+            ("test", "pdf1.pdf"),
+            ("test", "demo/不屈的精神.docx"),
+            ("test", "demo/demo2/中华人民共和国刑法.txt")
+        ]
+        
+        for bucket_name, object_path in test_files:
+            print(f"\n=== 解析文档: {bucket_name}/{object_path} ===")
+            result = object_parser.parse_object(bucket_name, object_path)
+            results.append(result)
+            
+            # 显示解析结果
+            print(f"文档名称: {result['doc_name']}")
+            print(f"文档路径: {result['doc_path_name']}")
+            print(f"文档类型: {result['doc_type']}")
+            print(f"文档MD5: {result['doc_md5']}")
+            print(f"文档大小: {result['doc_length']} 字节")
+            print(f"文本片段数量: {len(result['contents'])}")
+            
+            if 'error' in result:
+                print(f"错误信息: {result['error']}")
+            elif result['contents']:
+                # 显示所有片段的信息
+                for content in result['contents']:
+                    print(f"*************片段 (长度: {len(content)} 字符)")
+                    print(f" {content}")
+            
+            print("-" * 100)
+    
+    elif args.file:
+        # 解析指定的文件
+        for file_path in args.file:
+            print(f"\n=== 解析文档: {args.bucket}/{file_path} ===")
+            result = object_parser.parse_object(args.bucket, file_path)
+            results.append(result)
+            
+            # 显示解析结果
+            print(f"文档名称: {result['doc_name']}")
+            print(f"文档路径: {result['doc_path_name']}")
+            print(f"文档类型: {result['doc_type']}")
+            print(f"文档MD5: {result['doc_md5']}")
+            print(f"文档大小: {result['doc_length']} 字节")
+            print(f"文本片段数量: {len(result['contents'])}")
+            
+            if 'error' in result:
+                print(f"错误信息: {result['error']}")
+            elif result['contents']:
+                # 显示第一个片段的信息
+                for content in result['contents']:
+                    print(f"*************片段 (长度: {len(content)} 字符)")
+                    print(f" {content}")
+            
+            print("-" * 100)
+    
+    else:
+        # 显示使用说明
+        parser.print_help()
+        print("\n使用示例:")
+        print("  python simple_object_parser.py --test")
+        print("  python simple_object_parser.py --bucket test --file test/demo/GoogleIO.pdf")
+        print("  python simple_object_parser.py --bucket test --file test/demo/不屈的精神.docx")
+    
+    
+    print(f"\n=== 处理完成 ===")

mildoc_index/parser/text_parser.py

@@ -0,0 +1,40 @@
+import sys, os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from parser.document_parser import DocumentParser
+from logger.logging import setup_logging
+
+logger = setup_logging()
+
+class TextParser(DocumentParser):
+    """纯文本文档解析器"""
+    
+    def parse(self, data: bytes) -> str:
+        """解析文本文档"""
+        try:
+            # 尝试不同的编码
+            encodings = ['utf-8', 'gbk', 'gb2312', 'latin-1']
+            for encoding in encodings:
+                try:
+                    return data.decode(encoding)
+                except UnicodeDecodeError:
+                    continue
+            
+            # 如果所有编码都失败，使用错误处理
+            return data.decode('utf-8', errors='ignore')
+            
+        except Exception as e:
+            logger.error(f"文本解析失败: {e}")
+            return ""
+    
+    def supports(self, content_type: str) -> bool:
+        """检查是否支持文本"""
+        return content_type.lower() in ['text/plain', 'text/html', 'text/markdown', 'txt']
+
+
+if __name__ == "__main__":
+    parser = TextParser()
+    with open("../data/text.txt", "rb") as f:
+        data = f.read()
+    result = parser.parse(data)
+    print(result[:200])

mildoc_index/requirements.txt

@@ -0,0 +1,26 @@
+# Minio对象存储客户端
+minio==7.2.20
+
+# Milvus向量数据库客户端
+pymilvus==2.6.5
+
+# OpenAI API客户端（用于embedding）
+openai==2.11.0
+
+# 环境变量管理
+python-dotenv==1.2.1
+
+# PDF文档解析
+PyPDF2==3.0.1
+
+# Office文档解析
+markitdown[all]==0.1.4
+
+# text splitters
+langchain-text-splitters==1.1.0
+# langchain-core==1.2.0
+
+
+# 阿里云
+alibabacloud-oss-v2==1.2.1
+aliyun-mns-sdk==1.3.0

mildoc_index/run_mildoc_index

@@ -0,0 +1,15 @@
+#!/bin/bash
+echo "Mildoc index Starting... "
+
+uv sync
+
+
+nohup uv run main.py --provider minio --mode listen >> mildoc_index.log 2>&1 &
+
+# 使用阿里云 OSS 作为存储
+# nohup uv run main.py --provider oss --mode listen >> mildoc_index.log 2>&1 &
+
+# 重建索引
+# uv run main.py --provider minio --mode full-refresh
+# 补建索引
+# uv run main.py --provider minio --mode backfill

mildoc_milvus/milvus_local/.env

@@ -0,0 +1 @@
+DOCKER_VOLUME_DIRECTORY=/docker_data_volume/milvus_local

mildoc_milvus/milvus_local/docker-compose.yml

@@ -0,0 +1,70 @@
+services:
+  etcd:
+    container_name: milvus-etcd
+    image: quay.io/coreos/etcd:v3.5.25
+    environment:
+      - TZ=Asia/Shanghai
+      - ETCD_AUTO_COMPACTION_MODE=revision
+      - ETCD_AUTO_COMPACTION_RETENTION=1000
+      - ETCD_QUOTA_BACKEND_BYTES=4294967296
+      - ETCD_SNAPSHOT_COUNT=50000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
+    command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+    healthcheck:
+      test: ["CMD", "etcdctl", "endpoint", "health"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  minio:
+    container_name: milvus-minio
+    image: minio/minio:RELEASE.2024-12-18T13-15-44Z
+    environment:
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    ports:
+      - "9001:9001"
+      - "9000:9000"
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
+    command: minio server /minio_data --console-address ":9001"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  standalone:
+    container_name: milvus-standalone
+    image: milvusdb/milvus:v2.6.7
+    command: ["milvus", "run", "standalone"]
+    security_opt:
+    - seccomp:unconfined
+    environment:
+      ETCD_ENDPOINTS: etcd:2379
+      MINIO_ADDRESS: minio:9000
+      MQ_TYPE: woodpecker
+      # milvus 账号权限
+      COMMON_SECURITY_AUTHORIZATION_ENABLED: true
+      COMMON_SECURITY_SUPER_USERS: root
+      COMMON_SECURITY_DEFAULT_ROOT_PASSWORD: admin123
+      
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
+      interval: 30s
+      start_period: 90s
+      timeout: 20s
+      retries: 3
+    ports:
+      - "19530:19530"
+      - "9091:9091"
+    depends_on:
+      - "etcd"
+      - "minio"
+
+networks:
+  default:
+    name: milvus

mildoc_milvus/milvus_minio/.env

@@ -0,0 +1 @@
+DOCKER_VOLUME_DIRECTORY=/docker_data_volume/milvus_minio

mildoc_milvus/milvus_minio/docker-compose.yml

@@ -0,0 +1,59 @@
+services:
+  etcd:
+    container_name: milvus-minio-etcd
+    image: quay.io/coreos/etcd:v3.5.25
+    environment:
+      - TZ=Asia/Shanghai
+      - ETCD_AUTO_COMPACTION_MODE=revision
+      - ETCD_AUTO_COMPACTION_RETENTION=1000
+      - ETCD_QUOTA_BACKEND_BYTES=4294967296
+      - ETCD_SNAPSHOT_COUNT=50000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
+    command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+    healthcheck:
+      test: ["CMD", "etcdctl", "endpoint", "health"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  standalone:
+    container_name: milvus-minio-standalone
+    image: milvusdb/milvus:v2.6.7
+    command: ["milvus", "run", "standalone"]
+    security_opt:
+    - seccomp:unconfined
+    environment:
+      # etcd 配置
+      ETCD_ENDPOINTS: etcd:2379
+
+      # 外部 MinIO 配置
+      MINIO_ADDRESS: 172.31.154.203
+      MINIO_PORT: 9000
+      MINIO_ACCESS_KEY_ID: qT63tdsO66jqUjEHeXv3
+      MINIO_SECRET_ACCESS_KEY: YRYrBJzOJxD8e4WKlmSl5qqMbzd64ECTDarEQVAg
+      MINIO_USE_SSL: false
+      MINIO_BUCKET_NAME: milvus
+
+      # milvus 账号权限
+      COMMON_SECURITY_AUTHORIZATION_ENABLED: true
+      COMMON_SECURITY_SUPER_USERS: root  # 不要更改root用户名。
+      COMMON_SECURITY_DEFAULT_ROOT_PASSWORD: admin123
+      
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
+      interval: 30s
+      start_period: 90s
+      timeout: 20s
+      retries: 3
+    ports:
+      - "19530:19530"
+      - "9091:9091"
+    depends_on:
+      - "etcd"
+
+networks:
+  default:
+    name: milvus

mildoc_milvus/milvus_oss/.env

@@ -0,0 +1 @@
+DOCKER_VOLUME_DIRECTORY=/docker_data_volume/milvus_oss

mildoc_milvus/milvus_oss/docker-compose.yml

@@ -0,0 +1,62 @@
+services:
+  etcd:
+    container_name: milvus-oss-etcd
+    image: quay.io/coreos/etcd:v3.5.25
+    environment:
+      - TZ=Asia/Shanghai
+      - ETCD_AUTO_COMPACTION_MODE=revision
+      - ETCD_AUTO_COMPACTION_RETENTION=1000
+      - ETCD_QUOTA_BACKEND_BYTES=4294967296
+      - ETCD_SNAPSHOT_COUNT=50000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
+    command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+    healthcheck:
+      test: ["CMD", "etcdctl", "endpoint", "health"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  standalone:
+    container_name: milvus-oss-standalone
+    image: milvusdb/milvus:v2.6.7
+    command: ["milvus", "run", "standalone"]
+    security_opt:
+    - seccomp:unconfined
+    environment:
+      # etcd 配置
+      ETCD_ENDPOINTS: etcd:2379
+
+      # OSS 配置
+      MINIO_ADDRESS: oss-cn-hangzhou.aliyuncs.com
+      MINIO_PORT: 443 # 使用oss时一定要设置port，否则默认是9000
+      MINIO_ACCESS_KEY_ID: YOUR_ACCESS_KEY_ID # 替换为自己的access key id
+      MINIO_SECRET_ACCESS_KEY: YOUR_SECRET_ACCESS_KEY # 替换为自己的secret access key
+      MINIO_USE_VIRTUAL_HOST: true # 使用虚拟主机模式路径
+      MINIO_USE_SSL: true
+      MINIO_BUCKET_NAME: milvus-alioss
+      MINIO_ROOT_PATH: milvus_db # 指定存储路径
+      MINIO_REGION: cn-hangzhou
+      MINIO_CLOUD_PROVIDER: aliyun # 指定云服务商
+      # milvus 账号权限
+      COMMON_SECURITY_AUTHORIZATION_ENABLED: true
+      COMMON_SECURITY_SUPER_USERS: root
+      COMMON_SECURITY_DEFAULT_ROOT_PASSWORD: admin123
+      
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
+      interval: 30s
+      start_period: 90s
+      timeout: 20s
+      retries: 3
+    ports:
+      - "19530:19530"
+      - "9091:9091"
+    depends_on:
+      - "etcd"
+
+networks:
+  default:
+    name: milvus

mildoc_milvus/run_attu

@@ -0,0 +1,7 @@
+#! /bin/bash
+
+## start attu
+docker run -d --rm  -p 8000:3000 --name attu26 zilliz/attu:v2.6
+
+## stop attu
+# docker stop attu26

mildoc_wxkf/.env

@@ -0,0 +1,49 @@
+# 微信应用配置
+CORP_ID=ww29348715f7b93d0d
+AGENT_ID=1000009
+APP_SECRET=
+TOKEN=vvtJGvnKTzh4BPscesb
+ENCODING_AES_KEY=c4KTmM9URnBGmdnV8xbkexD2QtAr2TIXsfLwgvmNwp3
+
+# Flask服务器配置
+HOST=0.0.0.0
+PORT=8890
+DEBUG=false
+
+# SQLite数据库配置
+DATABASE_PATH=messages.db
+
+
+# milvus
+MILVUS_HOST=127.0.0.1
+MILVUS_PORT=19530
+MILVUS_USER=root
+MILVUS_PASSWORD=admin123
+MILVUS_DATABASE=mildoc
+MILVUS_COLLECTION=mildoc_collection
+MILVUS_INDEX_NAME=mildoc_index
+MILVUS_VECTOR_DIM=768
+
+# LLM配置 百炼
+LLM_MODEL_NAME=qwen-plus
+LLM_API_KEY=
+LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
+
+# LLM Embedding配置 百炼
+LLM_EMBEDDING_MODEL_NAME=text-embedding-v4
+LLM_EMBEDDING_API_KEY=
+LLM_EMBEDDING_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
+
+
+# rerank
+# 阿里百炼平台配置
+RERANK_PROVIDER=dashscope
+RERANK_API_KEY=
+RERANK_MODEL_NAME=gte-rerank-v2
+RERANK_ENDPOINT=https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank
+
+# langfuse
+# LANGFUSE_ENABLE=false
+# LANGFUSE_SECRET_KEY=sk-lf-3badb70b-3482-4f87-9581-7b2118ea43fc
+# LANGFUSE_PUBLIC_KEY=pk-lf-9c98917b-7875-4f34-8f72-1afd87491f84
+# LANGFUSE_BASE_URL=https://cdbece33249643e581c6525392ba1dda--3000.ap-shanghai2.cloudstudio.club

mildoc_wxkf/.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+messages.db
+mildoc_wxkf.log
+

mildoc_wxkf/WXBizMsgCrypt.py

@@ -0,0 +1,289 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#########################################################################
+# Author: 
+# Created Time: 
+# File Name: WXBizMsgCrypt.py
+# Description: 企业微信回调消息加解密示例代码
+#########################################################################
+
+import base64
+import string
+import random
+import hashlib
+import time
+import struct
+from Crypto.Cipher import AES
+import xml.etree.cElementTree as ET
+
+"""
+关于Crypto.Cipher模块，ImportError: No module named 'Crypto'解决方案
+请到官方网站 https://www.dlitz.net/software/pycrypto/ 下载pycrypto。
+下载后，按照README中的"Installation"小节的提示进行pycrypto安装。
+"""
+class FormatException(Exception):
+    pass
+
+def throw_exception(message, exception_class=FormatException):
+    """抛出异常信息"""
+    raise exception_class(message)
+
+class SHA1:
+    """计算企业微信签名的SHA1算法"""
+
+    def getSHA1(self, token, timestamp, nonce, encrypt):
+        """用SHA1算法生成安全签名
+        @param token:  票据
+        @param timestamp: 时间戳
+        @param nonce: 随机字符串
+        @param encrypt: 密文
+        @return: 安全签名
+        """
+        try:
+            sortlist = [token, timestamp, nonce, encrypt]
+            sortlist.sort()
+            sha = hashlib.sha1()
+            sha.update("".join(sortlist).encode('utf-8'))
+            return sha.hexdigest()
+        except Exception as e:
+            print(e)
+            return ""
+
+class XMLParse:
+    """提供提取消息格式中的密文及生成回复消息格式的接口"""
+
+    # xml消息模板
+    AES_TEXT_RESPONSE_TEMPLATE = """<xml>
+<Encrypt><![CDATA[%(msg_encrypt)s]]></Encrypt>
+<MsgSignature><![CDATA[%(msg_signaturet)s]]></MsgSignature>
+<TimeStamp>%(timestamp)s</TimeStamp>
+<Nonce><![CDATA[%(nonce)s]]></Nonce>
+</xml>"""
+
+    def extract(self, xmltext):
+        """提取出xml数据包中的加密消息
+        @param xmltext: 待提取的xml字符串
+        @return: 提取出的加密消息字符串
+        """
+        try:
+            xml_tree = ET.fromstring(xmltext)
+            encrypt = xml_tree.find("Encrypt")
+            touser_name = xml_tree.find("ToUserName")
+            return 0, encrypt.text, touser_name.text
+        except Exception as e:
+            print(e)
+            return -40003, None, None
+
+    def generate(self, encrypt, signature, timestamp, nonce):
+        """生成xml消息
+        @param encrypt: 加密后的消息
+        @param signature: 安全签名
+        @param timestamp: 时间戳
+        @param nonce: 随机字符串
+        @return: 生成的xml字符串
+        """
+        resp_dict = {
+            'msg_encrypt': encrypt,
+            'msg_signaturet': signature,
+            'timestamp': timestamp,
+            'nonce': nonce,
+        }
+        resp_xml = self.AES_TEXT_RESPONSE_TEMPLATE % resp_dict
+        return resp_xml
+
+class PKCS7Encoder():
+    """提供基于PKCS7算法的加解密接口"""
+
+    block_size = 32
+
+    def encode(self, text):
+        """ 对需要加密的明文进行填充补位
+        @param text: 需要进行填充补位操作的明文
+        @return: 补齐明文字符串
+        """
+        text_length = len(text)
+        # 计算需要填充的位数
+        amount_to_pad = self.block_size - (text_length % self.block_size)
+        if amount_to_pad == 0:
+            amount_to_pad = self.block_size
+        
+        # 处理bytes和str类型的兼容性
+        if isinstance(text, bytes):
+            pad = bytes([amount_to_pad] * amount_to_pad)
+            return text + pad
+        else:
+            # 获得补位所用的字符
+            pad = chr(amount_to_pad)
+            return text + pad * amount_to_pad
+
+    def decode(self, decrypted):
+        """删除解密后明文的补位字符
+        @param decrypted: 解密后的明文
+        @return: 删除补位字符后的明文
+        """
+        # 兼容Python 3.6：处理bytes和str类型
+        if isinstance(decrypted[-1], int):
+            pad = decrypted[-1]  # 在Python 3中，bytes的索引返回int
+        else:
+            pad = ord(decrypted[-1])  # 在Python 2中或str类型时需要ord()
+            
+        if pad < 1 or pad > 32:
+            pad = 0
+        return decrypted[:-pad]
+
+class Prpcrypt(object):
+    """提供接收和推送给企业微信消息的加解密接口"""
+
+    def __init__(self, key):
+        # self.key = base64.b64decode(key+"=")
+        self.key = key
+        # 设置加解密模式为AES的CBC模式
+        self.mode = AES.MODE_CBC
+
+    def encrypt(self, text, receiveid):
+        """对明文进行加密
+        @param text: 需要加密的明文
+        @return: 加密得到的字符串
+        """
+        # 16位随机字符串添加到明文开头
+        text = text.encode('utf-8')
+        # 使用大端序（网络字节序）打包长度字段，与企业微信标准保持一致
+        text = self.get_random_str() + struct.pack(">I", len(text)) + text + receiveid.encode('utf-8')
+        # 使用自定义的填充方式对明文进行补位填充
+        pkcs7 = PKCS7Encoder()
+        text = pkcs7.encode(text)
+        # 加密
+        cryptor = AES.new(self.key, self.mode, self.key[:16])
+        try:
+            ciphertext = cryptor.encrypt(text)
+            # 使用BASE64对加密后的字符串进行编码
+            return base64.b64encode(ciphertext)
+        except Exception as e:
+            print(e)
+            return None
+
+    def decrypt(self, text, receiveid, verify_receiveid=True):
+        """对解密后的明文进行补位删除
+        @param text: 密文
+        @param receiveid: 接收者ID
+        @param verify_receiveid: 是否验证receiveid，URL验证时设为False
+        @return: 删除填充补位后的明文
+        """
+        try:
+            cryptor = AES.new(self.key, self.mode, self.key[:16])
+            # 使用BASE64对密文进行解码，然后AES-CBC解密
+            plain_text = cryptor.decrypt(base64.b64decode(text))
+        except Exception as e:
+            print(e)
+            return None
+        try:
+            # 去除补位字符
+            pkcs7 = PKCS7Encoder()
+            plain_text = pkcs7.decode(plain_text)
+            # 去除16位随机字符串
+            content = plain_text[16:]
+            # 使用大端序（网络字节序）解析长度字段，这是企业微信使用的标准
+            xml_len = struct.unpack(">I", content[:4])[0]
+            xml_content = content[4:xml_len + 4]
+            from_receiveid = content[xml_len + 4:]
+        except Exception as e:
+            print(e)
+            return None
+        
+        # 如果需要验证receiveid，则进行验证
+        if verify_receiveid and from_receiveid.decode('utf-8') != receiveid:
+            return None
+        
+        return xml_content
+
+    def get_random_str(self):
+        """ 随机生成16位字符串
+        @return: 16位随机字符串
+        """
+        return "".join(random.choice(string.ascii_letters + string.digits) for _ in range(16)).encode('utf-8')
+
+class WXBizMsgCrypt(object):
+    # 构造函数
+    # @param sToken: 企业微信后台，开发者设置的Token
+    # @param sEncodingAESKey: 企业微信后台，开发者设置的EncodingAESKey
+    # @param sReceiveId: 企业微信的CorpId 或者 应用的AgentId
+    def __init__(self, sToken, sEncodingAESKey, sReceiveId):
+        try:
+            self.key = base64.b64decode(sEncodingAESKey + "=")
+            assert len(self.key) == 32
+        except:
+            throw_exception("[error]: EncodingAESKey unvalid !", FormatException)
+            # return WXBizMsgCrypt_ERROR_InvalidAesKey)
+        self.token = sToken
+        self.receiveid = sReceiveId
+
+        # 验证URL
+        # @param sMsgSignature: 签名串，对应URL参数的msg_signature
+        # @param sTimeStamp: 时间戳，对应URL参数的timestamp
+        # @param sNonce: 随机串，对应URL参数的nonce
+        # @param sEchoStr: 随机串，对应URL参数的echostr
+        # @param sReplyEchoStr: 解密之后的echostr，当return返回0时有效
+        # @return：成功0，失败返回对应的错误码
+
+    def VerifyURL(self, sMsgSignature, sTimeStamp, sNonce, sEchoStr):
+        sha1 = SHA1()
+        ret = sha1.getSHA1(self.token, sTimeStamp, sNonce, sEchoStr)
+        if ret != sMsgSignature:
+            return -40001
+        pc = Prpcrypt(self.key)
+        # URL验证时不进行receiveid验证，只解密获取实际内容
+        ret = pc.decrypt(sEchoStr, self.receiveid, verify_receiveid=False)
+        if ret is None:
+            return -40002
+        sReplyEchoStr = ret.decode('utf-8')
+        return 0, sReplyEchoStr
+
+    def DecryptMsg(self, sPostData, sMsgSignature, sTimeStamp, sNonce):
+        # 检验消息的真实性，并且获取解密后的明文
+        # @param sMsgSignature: 签名串，对应URL参数的msg_signature
+        # @param sTimeStamp: 时间戳，对应URL参数的timestamp
+        # @param sNonce: 随机串，对应URL参数的nonce
+        # @param sPostData: 密文，对应POST请求的数据
+        # @param sMsg: 解密后的原文，当return返回0时有效
+        # @return: 成功0，失败返回对应的错误码
+        xmlParse = XMLParse()
+        ret, sEncryptMsg, sToUserName = xmlParse.extract(sPostData)
+        if ret != 0:
+            return ret, None
+        sha1 = SHA1()
+        ret = sha1.getSHA1(self.token, sTimeStamp, sNonce, sEncryptMsg)
+        if ret != sMsgSignature:
+            return -40001, None
+        pc = Prpcrypt(self.key)
+        ret = pc.decrypt(sEncryptMsg, self.receiveid)
+        if ret is None:
+            return -40002, None
+        sMsg = ret.decode('utf-8')
+        return 0, sMsg
+
+    def EncryptMsg(self, sReplyMsg, sNonce, timestamp=None):
+        # 将企业回复用户的消息加密打包
+        # @param sReplyMsg: 企业号待回复用户的消息，xml格式的字符串
+        # @param sTimeStamp: 时间戳，可以自己生成，也可以用URL参数的timestamp,如为None则自动用当前时间
+        # @param sNonce: 随机串，可以自己生成，也可以用URL参数的nonce
+        # @param sEncryptMsg: 加密后的可以直接回复用户的密文，包括msg_signature, timestamp, nonce, encrypt的xml格式的字符串,当return返回0时有效
+        # return：成功0，失败返回对应的错误码
+        pc = Prpcrypt(self.key)
+        ret = pc.encrypt(sReplyMsg, self.receiveid)
+        if ret is None:
+            return -40006, None
+        if timestamp is None:
+            timestamp = str(int(time.time()))
+        
+        # 确保ret是bytes类型，然后解码为字符串用于签名和XML生成
+        if isinstance(ret, bytes):
+            encrypted_str = ret.decode('utf-8')
+        else:
+            encrypted_str = str(ret)
+        
+        # 生成安全签名
+        sha1 = SHA1()
+        signature = sha1.getSHA1(self.token, timestamp, sNonce, encrypted_str)
+        xmlParse = XMLParse()
+        result_xml = xmlParse.generate(encrypted_str, signature, timestamp, sNonce)
+        return 0, result_xml 

mildoc_wxkf/config.py

@@ -0,0 +1,115 @@
+"""
+企业微信回调服务配置文件
+
+作者：开发工程师
+日期：2025年01月
+"""
+
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+class Config:
+    """配置类"""
+    
+    # Langfuse配置
+    LANGFUSE_ENABLE = os.getenv("LANGFUSE_ENABLE", "false").lower() == "true"
+    LANGFUSE_SECRET_KEY = os.getenv('LANGFUSE_SECRET_KEY')
+    LANGFUSE_PUBLIC_KEY = os.getenv('LANGFUSE_PUBLIC_KEY')
+    LANGFUSE_BASE_URL = os.getenv('LANGFUSE_BASE_URL')
+    
+    # 企业微信基础配置
+    CORP_ID = os.getenv('CORP_ID')
+    TOKEN = os.getenv('TOKEN')
+    ENCODING_AES_KEY = os.getenv('ENCODING_AES_KEY')
+    
+    # 应用配置
+    APP_SECRET = os.getenv('APP_SECRET')  # 应用密钥，用于主动调用API
+    AGENT_ID = os.getenv('AGENT_ID')     # 应用ID
+    
+    
+    # 服务器配置
+    HOST = os.getenv('HOST')
+    PORT = int(os.getenv('PORT'))
+    DEBUG = os.getenv('DEBUG', 'False').lower() == 'true'
+    
+    # 数据库配置
+    DATABASE_PATH = os.getenv('DATABASE_PATH')
+    
+    # 客服配置
+    KF_MAX_REPLY_LENGTH = 2048  # 客服回复最大长度
+
+    # Milvus配置
+    MILVUS_HOST = os.getenv('MILVUS_HOST')
+    MILVUS_PORT = int(os.getenv('MILVUS_PORT'))
+    MILVUS_USER = os.getenv('MILVUS_USER')
+    MILVUS_PASSWORD = os.getenv('MILVUS_PASSWORD')
+    MILVUS_DATABASE = os.getenv('MILVUS_DATABASE')
+    MILVUS_COLLECTION_NAME = os.getenv('MILVUS_COLLECTION')  # 默认集合名称
+    MILVUS_INDEX_TYPE = os.getenv('MILVUS_INDEX_TYPE')
+    MILVUS_VECTOR_DIM = int(os.getenv('MILVUS_VECTOR_DIM'))
+
+    # LLM配置
+    LLM_MODEL_NAME = os.getenv('LLM_MODEL_NAME')
+    LLM_API_KEY = os.getenv('LLM_API_KEY')
+    LLM_BASE_URL = os.getenv('LLM_BASE_URL')
+
+    # LLM Embedding配置
+    LLM_EMBEDDING_MODEL_NAME = os.getenv('LLM_EMBEDDING_MODEL_NAME')
+    LLM_EMBEDDING_API_KEY = os.getenv('LLM_EMBEDDING_API_KEY')
+    LLM_EMBEDDING_BASE_URL = os.getenv('LLM_EMBEDDING_BASE_URL')
+
+    # Rerank服务配置
+    RERANK_PROVIDER = os.getenv('RERANK_PROVIDER')  # dashscope 或 siliconflow
+    RERANK_API_KEY = os.getenv('RERANK_API_KEY')
+    RERANK_MODEL_NAME = os.getenv('RERANK_MODEL_NAME')  # 百炼: gte-rerank-v2, 硅基: BAAI/bge-reranker-v2-m3
+    RERANK_ENDPOINT = os.getenv('RERANK_ENDPOINT')  # 自定义API端点
+
+    @classmethod
+    def validate_config(cls):
+        """验证配置的有效性"""
+        errors = []
+        
+        if not cls.TOKEN:
+            errors.append("缺少TOKEN配置")
+        
+        if not cls.ENCODING_AES_KEY:
+            errors.append("缺少ENCODING_AES_KEY配置")
+        
+        if not cls.CORP_ID:
+            errors.append("缺少CORP_ID配置")
+        
+        return errors
+    
+    @classmethod
+    def get_config_info(cls):
+        """获取配置信息摘要"""
+        return {
+            'corp_id': cls.CORP_ID,
+            'token_configured': bool(cls.TOKEN),
+            'encoding_key_configured': bool(cls.ENCODING_AES_KEY),
+            'app_secret_configured': bool(cls.APP_SECRET),
+            'agent_id_configured': bool(cls.AGENT_ID),
+            'database_path': cls.DATABASE_PATH,
+            'host': cls.HOST,
+            'port': cls.PORT,
+            'debug': cls.DEBUG,
+            'host_whitelist_enabled': cls.ENABLE_HOST_WHITELIST,
+            'allowed_hosts': cls.ALLOWED_HOSTS
+        }
+
+# 功能配置
+class FeatureConfig:
+    """功能配置"""
+    
+    # 支持的消息类型
+    SUPPORTED_MESSAGE_TYPES = [
+        'text', 'image', 'voice', 'video', 'file', 'location', 'link'
+    ]
+    
+    # 支持的事件类型
+    SUPPORTED_EVENT_TYPES = [
+        'subscribe', 'unsubscribe', 'LOCATION', 'CLICK'
+    ]
+    

mildoc_wxkf/cursor_manager.py

@@ -0,0 +1,322 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+微信客服消息cursor持久化管理模块
+用于管理每个客服账号的消息拉取cursor，避免重复处理消息
+
+作者：开发工程师
+日期：2025年01月
+"""
+
+import os
+import json
+import logging
+import sqlite3
+import threading
+from typing import Dict, Optional
+from config import Config
+
+logger = logging.getLogger(__name__)
+
+class CursorManager:
+    """Cursor持久化管理器"""
+    
+    def __init__(self, db_path: str = None):
+        self.db_path = db_path or Config.DATABASE_PATH
+        self.lock = threading.Lock()
+        self._init_database()
+    
+    def _init_database(self):
+        """初始化数据库表"""
+        try:
+            with sqlite3.connect(self.db_path) as conn:
+                cursor = conn.cursor()
+                
+                # 创建cursor存储表
+                cursor.execute('''
+                    CREATE TABLE IF NOT EXISTS kf_cursors (
+                        open_kfid TEXT PRIMARY KEY,
+                        cursor TEXT NOT NULL,
+                        last_updated INTEGER NOT NULL,
+                        message_count INTEGER DEFAULT 0,
+                        created_time INTEGER DEFAULT (strftime('%s', 'now'))
+                    )
+                ''')
+                
+                # 创建消息去重表
+                cursor.execute('''
+                    CREATE TABLE IF NOT EXISTS processed_messages (
+                        msgid TEXT PRIMARY KEY,
+                        open_kfid TEXT NOT NULL,
+                        external_userid TEXT,
+                        msgtype TEXT,
+                        origin INTEGER,
+                        processed_time INTEGER DEFAULT (strftime('%s', 'now')),
+                        reply_sent INTEGER DEFAULT 0
+                    )
+                ''')
+                
+                # 创建索引
+                cursor.execute('''
+                    CREATE INDEX IF NOT EXISTS idx_processed_messages_time 
+                    ON processed_messages(processed_time)
+                ''')
+                
+                cursor.execute('''
+                    CREATE INDEX IF NOT EXISTS idx_processed_messages_kfid 
+                    ON processed_messages(open_kfid)
+                ''')
+                
+                conn.commit()
+                logger.info("Cursor管理数据库初始化完成")
+                
+        except Exception as e:
+            logger.error(f"初始化cursor数据库失败: {e}")
+    
+    def get_cursor(self, open_kfid: str) -> str:
+        """
+        获取指定客服账号的cursor
+        
+        Args:
+            open_kfid: 客服账号ID
+            
+        Returns:
+            cursor字符串，如果不存在则返回空字符串
+        """
+        try:
+            with self.lock:
+                with sqlite3.connect(self.db_path) as conn:
+                    cursor = conn.cursor()
+                    cursor.execute(
+                        'SELECT cursor FROM kf_cursors WHERE open_kfid = ?',
+                        (open_kfid,)
+                    )
+                    result = cursor.fetchone()
+                    
+                    if result:
+                        logger.debug(f"获取cursor成功 - {open_kfid}: {result[0][:20]}...")
+                        return result[0]
+                    else:
+                        logger.info(f"客服账号 {open_kfid} 无历史cursor，将进行首次拉取")
+                        return ""
+                        
+        except Exception as e:
+            logger.error(f"获取cursor失败: {e}")
+            return ""
+    
+    def save_cursor(self, open_kfid: str, cursor: str, message_count: int = 0) -> bool:
+        """
+        保存指定客服账号的cursor
+        
+        Args:
+            open_kfid: 客服账号ID
+            cursor: 新的cursor值
+            message_count: 本次处理的消息数量
+            
+        Returns:
+            保存是否成功
+        """
+        try:
+            with self.lock:
+                with sqlite3.connect(self.db_path) as conn:
+                    db_cursor = conn.cursor()
+                    
+                    # 使用UPSERT语法
+                    db_cursor.execute('''
+                        INSERT OR REPLACE INTO kf_cursors 
+                        (open_kfid, cursor, last_updated, message_count)
+                        VALUES (?, ?, strftime('%s', 'now'), 
+                               COALESCE((SELECT message_count FROM kf_cursors WHERE open_kfid = ?), 0) + ?)
+                    ''', (open_kfid, cursor, open_kfid, message_count))
+                    
+                    conn.commit()
+                    logger.info(f"保存cursor成功 - {open_kfid}: {cursor[:20]}..., 消息数: {message_count}")
+                    return True
+                    
+        except Exception as e:
+            logger.error(f"保存cursor失败: {e}")
+            return False
+    
+    def is_message_processed(self, msgid: str) -> bool:
+        """
+        检查消息是否已经处理过
+        
+        Args:
+            msgid: 消息ID
+            
+        Returns:
+            是否已处理
+        """
+        try:
+            with self.lock:
+                with sqlite3.connect(self.db_path) as conn:
+                    cursor = conn.cursor()
+                    cursor.execute(
+                        'SELECT 1 FROM processed_messages WHERE msgid = ?',
+                        (msgid,)
+                    )
+                    result = cursor.fetchone()
+                    return result is not None
+                    
+        except Exception as e:
+            logger.error(f"检查消息处理状态失败: {e}")
+            return False
+    
+    def mark_message_processed(self, msgid: str, open_kfid: str, external_userid: str = "", 
+                             msgtype: str = "", origin: int = 0, reply_sent: bool = False) -> bool:
+        """
+        标记消息为已处理
+        
+        Args:
+            msgid: 消息ID
+            open_kfid: 客服账号ID
+            external_userid: 外部用户ID
+            msgtype: 消息类型
+            origin: 消息来源
+            reply_sent: 是否已发送回复
+            
+        Returns:
+            标记是否成功
+        """
+        try:
+            with self.lock:
+                with sqlite3.connect(self.db_path) as conn:
+                    cursor = conn.cursor()
+                    cursor.execute('''
+                        INSERT OR REPLACE INTO processed_messages 
+                        (msgid, open_kfid, external_userid, msgtype, origin, reply_sent)
+                        VALUES (?, ?, ?, ?, ?, ?)
+                    ''', (msgid, open_kfid, external_userid, msgtype, origin, int(reply_sent)))
+                    
+                    conn.commit()
+                    logger.debug(f"标记消息已处理: {msgid}")
+                    return True
+                    
+        except Exception as e:
+            logger.error(f"标记消息处理状态失败: {e}")
+            return False
+    
+    def cleanup_old_records(self, days: int = 30) -> bool:
+        """
+        清理旧的处理记录
+        
+        Args:
+            days: 保留天数
+            
+        Returns:
+            清理是否成功
+        """
+        try:
+            with self.lock:
+                with sqlite3.connect(self.db_path) as conn:
+                    cursor = conn.cursor()
+                    
+                    # 清理旧的消息处理记录
+                    cursor.execute('''
+                        DELETE FROM processed_messages 
+                        WHERE processed_time < strftime('%s', 'now', '-{} days')
+                    '''.format(days))
+                    
+                    deleted_count = cursor.rowcount
+                    conn.commit()
+                    
+                    if deleted_count > 0:
+                        logger.info(f"清理了 {deleted_count} 条旧的消息处理记录")
+                    
+                    return True
+                    
+        except Exception as e:
+            logger.error(f"清理旧记录失败: {e}")
+            return False
+    
+    def get_statistics(self) -> Dict:
+        """
+        获取cursor管理统计信息
+        
+        Returns:
+            统计信息字典
+        """
+        try:
+            with self.lock:
+                with sqlite3.connect(self.db_path) as conn:
+                    cursor = conn.cursor()
+                    
+                    # 获取客服账号数量
+                    cursor.execute('SELECT COUNT(*) FROM kf_cursors')
+                    kf_count = cursor.fetchone()[0]
+                    
+                    # 获取总消息处理数量
+                    cursor.execute('SELECT SUM(message_count) FROM kf_cursors')
+                    total_messages = cursor.fetchone()[0] or 0
+                    
+                    # 获取今日处理消息数量
+                    cursor.execute('''
+                        SELECT COUNT(*) FROM processed_messages 
+                        WHERE processed_time >= strftime('%s', 'now', 'start of day')
+                    ''')
+                    today_messages = cursor.fetchone()[0]
+                    
+                    # 获取已回复消息数量
+                    cursor.execute('SELECT COUNT(*) FROM processed_messages WHERE reply_sent = 1')
+                    replied_messages = cursor.fetchone()[0]
+                    
+                    return {
+                        'kf_accounts': kf_count,
+                        'total_messages': total_messages,
+                        'today_messages': today_messages,
+                        'replied_messages': replied_messages,
+                        'reply_rate': round(replied_messages / max(total_messages, 1) * 100, 2)
+                    }
+                    
+        except Exception as e:
+            logger.error(f"获取统计信息失败: {e}")
+            return {}
+    
+    def get_kf_account_info(self, open_kfid: str) -> Optional[Dict]:
+        """
+        获取指定客服账号的详细信息
+        
+        Args:
+            open_kfid: 客服账号ID
+            
+        Returns:
+            账号信息字典
+        """
+        try:
+            with self.lock:
+                with sqlite3.connect(self.db_path) as conn:
+                    cursor = conn.cursor()
+                    
+                    # 获取cursor信息
+                    cursor.execute('''
+                        SELECT cursor, last_updated, message_count, created_time 
+                        FROM kf_cursors WHERE open_kfid = ?
+                    ''', (open_kfid,))
+                    cursor_info = cursor.fetchone()
+                    
+                    if not cursor_info:
+                        return None
+                    
+                    # 获取今日消息数量
+                    cursor.execute('''
+                        SELECT COUNT(*) FROM processed_messages 
+                        WHERE open_kfid = ? AND processed_time >= strftime('%s', 'now', 'start of day')
+                    ''', (open_kfid,))
+                    today_count = cursor.fetchone()[0]
+                    
+                    return {
+                        'open_kfid': open_kfid,
+                        'cursor': cursor_info[0][:20] + '...' if cursor_info[0] else '',
+                        'last_updated': cursor_info[1],
+                        'total_messages': cursor_info[2],
+                        'today_messages': today_count,
+                        'created_time': cursor_info[3]
+                    }
+                    
+        except Exception as e:
+            logger.error(f"获取客服账号信息失败: {e}")
+            return None
+
+# 全局cursor管理器实例
+cursor_manager = CursorManager() 

mildoc_wxkf/kf_message_handler.py

@@ -0,0 +1,490 @@
+import logging
+import time
+from typing import Dict
+from wecom_api import wecom_api
+from config import Config
+from cursor_manager import cursor_manager
+from rag_service import get_rag_service
+logger = logging.getLogger(__name__)
+
+
+WELCOME_MESSAGE = '''🎉 欢迎使用微信客服！
+
+我是您的专属客服助手，很高兴为您服务！
+
+🔹 有任何问题随时咨询
+🔹 输入"帮助"查看功能菜单
+🔹 我们提供7×24小时服务
+
+请问有什么可以帮助您的吗？'''
+
+
+
+
+class KfMessageHandler:
+    """微信客服消息处理器"""
+
+    def __init__(self):
+        self.processed_messages = set()  # 内存中的消息去重缓存
+    
+    def process_kf_event(self, token: str, open_kfid: str) -> bool:
+        """
+        处理客服事件，拉取并处理消息（优化版本）
+        
+        Args:
+            token: 回调事件中的token
+            open_kfid: 客服账号ID
+            
+        Returns:
+            处理是否成功
+        """
+        try:
+            # 从持久化存储获取该客服账号的最后cursor
+            cursor = cursor_manager.get_cursor(open_kfid)
+            
+            # 确定拉取限制：如果没有cursor（首次或丢失），只拉取最近1条消息作为保底
+            limit = 1 if not cursor else 100
+            
+            logger.info(f"开始拉取客服消息 - OpenKfId: {open_kfid}, Cursor: {'有' if cursor else '无'}, Limit: {limit}")
+            
+            # 拉取消息
+            result = wecom_api.sync_kf_messages(token, open_kfid, cursor, limit)
+            if not result:
+                logger.error("拉取客服消息失败")
+                return False
+            
+            msg_list = result.get('msg_list', [])
+            next_cursor = result.get('next_cursor', '')
+            has_more = result.get('has_more', 0)
+            
+            logger.info(f"拉取到 {len(msg_list)} 条客服消息，next_cursor: {'有' if next_cursor else '无'}, has_more: {has_more}")
+            
+            # 处理每条消息（带去重）
+            processed_count = 0
+            new_messages = 0
+            
+            for msg in msg_list:
+                msgid = msg.get('msgid', '')
+                
+                # 检查消息是否已处理（双重检查：内存缓存 + 数据库）
+                if msgid in self.processed_messages or cursor_manager.is_message_processed(msgid):
+                    logger.debug(f"消息已处理，跳过: {msgid}")
+                    continue
+                
+                # 处理新消息
+                if self.process_single_kf_message(msg):
+                    processed_count += 1
+                    new_messages += 1
+                    # 添加到内存缓存
+                    self.processed_messages.add(msgid)
+                    
+                    # 限制内存缓存大小
+                    if len(self.processed_messages) > 1000:
+                        # 清理一半的缓存
+                        self.processed_messages = set(list(self.processed_messages)[500:])
+            
+            # 保存cursor到持久化存储
+            if next_cursor:
+                cursor_manager.save_cursor(open_kfid, next_cursor, new_messages)
+                logger.info(f"已保存新cursor - OpenKfId: {open_kfid}, 新消息数: {new_messages}")
+            
+            # 如果还有更多消息，继续拉取（但要避免无限循环）
+            if has_more == 1 and new_messages > 0:
+                logger.info("还有更多消息，继续拉取...")
+                return self.process_kf_event(token, open_kfid)
+            
+            logger.info(f"客服消息处理完成 - 总拉取: {len(msg_list)}, 新处理: {processed_count}")
+            return True
+            
+        except Exception as e:
+            logger.error(f"处理客服事件异常: {e}")
+            return False
+    
+    def process_single_kf_message(self, msg: Dict) -> bool:
+        """
+        处理单条客服消息（优化版本）
+        
+        Args:
+            msg: 消息数据
+            
+        Returns:
+            处理是否成功
+        """
+        try:
+            msgid = msg.get('msgid', '')
+            open_kfid = msg.get('open_kfid', '')
+            external_userid = msg.get('external_userid', '')
+            send_time = msg.get('send_time', 0)
+            origin = msg.get('origin', 0)  # 3-微信客户发送 4-系统推送事件 5-接待人员发送
+            servicer_userid = msg.get('servicer_userid', '')
+            msgtype = msg.get('msgtype', '')
+            
+            logger.info(f"处理客服消息 - msgid: {msgid}, 类型: {msgtype}, 来源: {origin}, 客户: {external_userid}")
+            
+            # 检查消息时效性（仅对客户发送的消息进行时效检查）
+            current_time = int(time.time())
+            message_age_seconds = current_time - send_time
+            message_age_minutes = message_age_seconds / 60
+            
+            # 标记消息为已处理
+            reply_sent = False
+            
+            # 只处理微信客户发送的消息
+            if origin == 3 and external_userid:
+                # 检查消息是否在10分钟内发送
+                if message_age_minutes <= 10:
+                    reply_sent = self.handle_customer_message(msg)
+                    logger.info(f"消息已处理，消息年龄：{message_age_minutes:.1f}分钟")
+                else:
+                    logger.info(f"消息超过10分钟时效限制，不予回复 - msgid: {msgid}, "
+                              f"消息时间: {send_time}, 当前时间: {current_time}, "
+                              f"消息年龄: {message_age_minutes:.1f}分钟")
+                    # 虽然不回复，但仍然标记为已处理，避免重复处理
+                    reply_sent = True
+            elif origin == 4:
+                # 处理系统事件
+                self.handle_system_event(msg)
+            elif origin == 5:
+                # 接待人员发送的消息，记录日志
+                logger.info(f"接待人员 {servicer_userid} 发送消息: {msgtype}")
+            
+            # 标记消息为已处理
+            cursor_manager.mark_message_processed(
+                msgid, open_kfid, external_userid, msgtype, origin, reply_sent
+            )
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"处理单条客服消息异常: {e}")
+            return False
+    
+    
+
+    
+    def handle_customer_message(self, msg: Dict) -> bool:
+        """
+        处理客户发送的消息
+        
+        Returns:
+            是否发送了回复
+        """
+        try:
+            msgtype = msg.get('msgtype', '')
+            external_userid = msg.get('external_userid', '')
+            open_kfid = msg.get('open_kfid', '')
+                        
+            reply_sent = False
+
+            service_state = self.get_service_session_state(external_userid, open_kfid)
+
+            if (service_state != 1 and service_state != 0):
+                # 如果不是智能助手接待状态，则不处理
+                logger.info(f"非智能助手接待状态，不处理消息 - open_kfid: {open_kfid}, 客户: {external_userid}")
+                return False
+            
+            if msgtype == 'text':
+                # 处理文本消息，使用智能回复生成回复内容
+                text_data = msg.get('text', {})
+                content = text_data.get('content', '')
+                
+                logger.info(f"收到客户文本消息: {content}")
+                
+                if '转人工' in content:
+                    # 转人工
+                    reply_content = "🤖 收到您的转人工请求，我会尽快转接人工服务。"
+                    reply_sent = self.send_kf_reply(external_userid, open_kfid, reply_content)
+                    
+                    self.update_service_session_state_to_service_pool(external_userid, open_kfid)
+                else:
+                    # 生成智能回复
+                    reply_content = self.get_smart_reply(content)
+                    # 发送回复
+                    if reply_content:
+                        reply_sent = self.send_kf_reply(external_userid, open_kfid, reply_content)
+            
+            elif msgtype in ['image', 'voice', 'video', 'file']:
+                # 处理多媒体消息，暂不处理，仅作简单回复收到消息
+                replies = {
+                    'image': '''📷 收到您发送的图片，请简单描述一下图片内容，我能更好地为您服务！''',
+                    
+                    'voice': '''🎤 收到您的语音消息，感谢您的留言！''',
+                    
+                    'video': '''🎬 收到您发送的视频，感谢分享！''',
+                    
+                    'file': '''📎 收到您发送的文件，我会尽快查看处理。'''
+                }
+                reply_content = replies.get(msgtype, '收到您的消息，感谢分享！我会尽快为您处理。')
+                if reply_content:
+                    reply_sent = self.send_kf_reply(external_userid, open_kfid, reply_content)
+            
+            elif msgtype == 'location':
+                # 处理位置消息，暂不处理，仅作简单回复收到消息
+                location = msg.get('location', {})
+                name = location.get('name', '')
+                address = location.get('address', '')
+                reply_content = f"📍 收到您分享的位置：{name}\n地址：{address}\n\n感谢分享！如需导航或周边服务，请告诉我具体需求。"
+                reply_sent = self.send_kf_reply(external_userid, open_kfid, reply_content)
+            
+            elif msgtype == 'link':
+                # 处理链接消息，暂不处理，仅作简单回复收到消息
+                link = msg.get('link', {})
+                title = link.get('title', '')
+                reply_content = f"🔗 收到您分享的链接：{title}\n\n感谢分享！我会查看相关内容。"
+                reply_sent = self.send_kf_reply(external_userid, open_kfid, reply_content)
+            
+            elif msgtype == 'business_card':
+                # 处理名片消息，暂不处理，仅作简单回复收到消息
+                reply_content = "👤 收到您的名片，感谢分享联系方式！\n\n如有业务合作需求，我们会及时与您联系。"
+                reply_sent = self.send_kf_reply(external_userid, open_kfid, reply_content)
+            
+            elif msgtype == 'miniprogram':
+                # 处理小程序消息，暂不处理，仅作简单回复收到消息
+                mini = msg.get('miniprogram', {})
+                title = mini.get('title', '')
+                reply_content = f"📱 收到您分享的小程序：{title}\n\n感谢分享！我会查看相关功能。"
+                reply_sent = self.send_kf_reply(external_userid, open_kfid, reply_content)
+            
+            elif msgtype == 'channels_shop_product':
+                # 处理视频号商品消息，暂不处理，仅作简单回复收到消息
+                product = msg.get('channels_shop_product', {})
+                title = product.get('title', '')
+                price = product.get('sales_price', '')
+                reply_content = f"🛍️ 收到您关注的商品：{title}\n价格：{price}分\n\n如需了解更多商品信息或购买咨询，请告诉我！"
+                reply_sent = self.send_kf_reply(external_userid, open_kfid, reply_content)
+            
+            elif msgtype == 'channels_shop_order':
+                # 处理视频号订单消息，暂不处理，仅作简单回复收到消息
+                order = msg.get('channels_shop_order', {})
+                order_id = order.get('order_id', '')
+                state = order.get('state', '')
+                reply_content = f"📦 收到您的订单信息：{order_id}\n状态：{state}\n\n如需查询订单详情或有其他问题，请随时联系我！"
+                reply_sent = self.send_kf_reply(external_userid, open_kfid, reply_content)
+            
+            return reply_sent
+            
+        except Exception as e:
+            logger.error(f"处理客户消息异常: {e}")
+            return False
+    
+    def handle_system_event(self, msg: Dict) -> None:
+        """处理系统事件"""
+        try:
+            event_data = msg.get('event', {})
+            event_type = event_data.get('event_type', '')
+            
+            logger.info(f"处理系统事件: {event_type}")
+            
+            if event_type == 'enter_session':
+                # 用户进入会话事件
+                self.handle_enter_session_event(event_data)
+            elif event_type == 'msg_send_fail':
+                # 消息发送失败事件
+                self.handle_send_fail_event(event_data)
+            elif event_type == 'servicer_status_change':
+                # 接待人员状态变更事件
+                self.handle_servicer_status_change_event(event_data)
+            elif event_type == 'session_status_change':
+                # 会话状态变更事件
+                self.handle_session_change_event(event_data)
+            elif event_type == 'user_recall_msg':
+                # 用户撤回消息事件
+                self.handle_user_recall_event(event_data)
+            elif event_type == 'servicer_recall_msg':
+                # 接待人员撤回消息事件
+                self.handle_servicer_recall_event(event_data)
+            
+        except Exception as e:
+            logger.error(f"处理系统事件异常: {e}")
+    
+    def handle_enter_session_event(self, event_data: Dict) -> None:
+        """处理用户进入会话事件"""
+        try:
+            external_userid = event_data.get('external_userid', '')
+            open_kfid = event_data.get('open_kfid', '')
+            scene = event_data.get('scene', '')
+            welcome_code = event_data.get('welcome_code', '')
+            
+            logger.info(f"用户 {external_userid} 进入会话 - 场景: {scene} - 欢迎码: {welcome_code} - open_kfid: {open_kfid}")
+            
+            # 发送欢迎消息
+            welcome_msg = WELCOME_MESSAGE
+            # 使用事件响应消息接口发送欢迎语
+            wecom_api.send_event_response_message(welcome_code, welcome_msg)
+            
+        except Exception as e:
+            logger.error(f"处理进入会话事件异常: {e}")
+    
+    def handle_send_fail_event(self, event_data: Dict) -> None:
+        """处理消息发送失败事件，暂不处理，仅做日志记录"""
+        try:
+            external_userid = event_data.get('external_userid', '')
+            fail_msgid = event_data.get('fail_msgid', '')
+            fail_type = event_data.get('fail_type', 0)
+            
+            fail_reasons = {
+                0: "未知原因",
+                1: "客服账号已删除",
+                2: "应用已关闭",
+                4: "会话已过期，超过48小时",
+                5: "会话已关闭",
+                6: "超过5条限制",
+                7: "未绑定视频号",
+                8: "主体未验证",
+                9: "未绑定视频号且主体未验证",
+                10: "用户拒收"
+            }
+            
+            reason = fail_reasons.get(fail_type, "未知原因")
+            logger.warning(f"消息发送失败 - 用户: {external_userid}, 消息ID: {fail_msgid}, 原因: {reason}")
+            
+        except Exception as e:
+            logger.error(f"处理发送失败事件异常: {e}")
+    
+    def handle_servicer_status_change_event(self, event_data: Dict) -> None:
+        """处理接待人员状态变更事件，暂不处理，仅做日志记录"""
+        try:
+            servicer_userid = event_data.get('servicer_userid', '')
+            status = event_data.get('status', 0)
+            open_kfid = event_data.get('open_kfid', '')
+            
+            logger.info(f"接待人员 {servicer_userid} 状态变更: {status}, open_kfid: {open_kfid}")
+            
+        except Exception as e:
+            logger.error(f"处理接待人员状态变更事件异常: {e}")
+    
+    def handle_session_change_event(self, event_data: Dict) -> None:
+        """处理会话状态变更，回复欢迎语或者结束语"""
+        try:
+            external_userid = event_data.get('external_userid', '')
+            change_type = event_data.get('change_type', 0)
+            msg_code = event_data.get('msg_code', '')
+            
+            change_types = {
+                1: "从接待池接入会话",
+                2: "转接会话", 
+                3: "结束会话",
+                4: "重新接入已结束/已转接会话"
+            }
+            
+            change_text = change_types.get(change_type, "未知变更")
+            logger.info(f"会话状态变更 - 用户: {external_userid}, 变更: {change_text}")
+            
+            # 如果有消息码，可以发送相应的回复语或结束语
+            if msg_code:
+                if change_type == 1:  # 接入会话
+                    response_msg = "您好！我是您的专属客服，很高兴为您服务！有什么可以帮助您的吗？"
+                elif change_type == 3:  # 结束会话
+                    response_msg = "感谢您的咨询！如有其他问题，欢迎随时联系我们。祝您生活愉快！"
+                else:
+                    response_msg = None
+                
+                if response_msg:
+                    wecom_api.send_event_response_message(msg_code, response_msg)
+            
+        except Exception as e:
+            logger.error(f"处理会话状态变更事件异常: {e}")
+    
+    def handle_user_recall_event(self, event_data: Dict) -> None:
+        """处理用户撤回消息事件，暂不处理，仅做日志记录"""
+        try:
+            external_userid = event_data.get('external_userid', '')
+            recall_msgid = event_data.get('recall_msgid', '')
+            
+            logger.info(f"用户 {external_userid} 撤回消息: {recall_msgid}")
+            
+        except Exception as e:
+            logger.error(f"处理用户撤回消息事件异常: {e}")
+    
+    def handle_servicer_recall_event(self, event_data: Dict) -> None:
+        """处理接待人员撤回消息事件，暂不处理，仅做日志记录"""
+        try:
+            external_userid = event_data.get('external_userid', '')
+            servicer_userid = event_data.get('servicer_userid', '')
+            recall_msgid = event_data.get('recall_msgid', '')
+            
+            logger.info(f"接待人员 {servicer_userid} 撤回消息: {recall_msgid}")
+            
+        except Exception as e:
+            logger.error(f"处理接待人员撤回消息事件异常: {e}")
+        
+    def get_smart_reply(self, content: str) -> str:
+        """智能回复，使用RAG服务获取回复内容，并记录token消耗情况和参考文档"""
+        logger.info(f"智能客服回复开始")
+
+        try:
+            # 调用智能客服接口，获取智能回复内容和token消耗情况
+            response = get_rag_service().query_service(content)
+
+            if response.success:
+                # 记录token使用情况
+                if response.token_usage:
+                    logger.info(f"💰 本次查询Token使用: 输入{response.token_usage.prompt_tokens}, "
+                            f"输出{response.token_usage.completion_tokens}, "
+                            f"总计{response.token_usage.total_tokens}")
+                
+                # 记录参考文档
+                if response.source_documents:
+                    logger.info(f"📚 参考文档: {[doc.doc_name for doc in response.source_documents]}")
+                
+                return response.content
+            else:
+                logger.error(f"RAG查询失败: {response.error_message}")
+                return "抱歉，我暂时无法理解您的问题，请稍后再试或联系人工客服。"
+
+        except Exception as e:
+            logger.error(f"智能回复处理异常: {e}")
+            return "抱歉，我暂时无法理解您的问题，请稍后再试或联系人工客服。"
+    
+    
+    def send_kf_reply(self, external_userid: str, open_kfid: str, content: str) -> bool:
+        """发送客服回复"""
+        try:
+            # 检查回复内容长度
+            if len(content) > Config.KF_MAX_REPLY_LENGTH:
+                content = content[:Config.KF_MAX_REPLY_LENGTH-10] + "...(内容过长已截断)"
+            
+            result = wecom_api.send_kf_text_message(external_userid, open_kfid, content)
+            if result:
+                logger.info(f"发送客服回复成功 - 用户: {external_userid}")
+                return True
+            else:
+                logger.error(f"发送客服回复失败 - 用户: {external_userid}")
+                return False
+        except Exception as e:
+            logger.error(f"发送客服回复异常: {e}")
+            return False
+    
+
+    def get_service_session_state(self, external_userid: str, open_kfid: str) -> int:
+        """获取服务会话状态"""
+        try:
+            result = wecom_api.get_service_session_state(external_userid, open_kfid)
+            if result:
+                logger.info(f"获取服务会话状态成功 - 用户: {external_userid}")
+                logger.info(f"服务会话状态: {result}")
+                return result.get('service_state', -1)
+            else:
+                logger.error(f"获取服务会话状态失败 - 用户: {external_userid}")
+                return -1
+        except Exception as e:
+            logger.error(f"获取服务会话状态异常: {e}")
+            return -1
+
+    def update_service_session_state_to_service_pool(self, external_userid: str, open_kfid: str) -> bool:
+        """更新服务会话状态"""
+        try:
+            result = wecom_api.update_service_session_state(external_userid, open_kfid, 2) # 2: 转接会话进入接待池，后续由人工接待
+            if result:
+                logger.info(f"更新服务会话状态成功 - 用户: {external_userid}")
+                return True
+            else:
+                logger.error(f"更新服务会话状态失败 - 用户: {external_userid}")
+                return False
+        except Exception as e:
+            logger.error(f"更新服务会话状态异常: {e}")
+            return False
+    
+
+# 全局客服消息处理器实例
+kf_message_handler = KfMessageHandler() 

mildoc_wxkf/rag_service.py

@@ -0,0 +1,674 @@
+"""
+RAG服务工具类 (基于LangChain实现)
+
+使用LangChain + Milvus实现RAG服务
+支持从Milvus向量数据库检索相关文档并通过大模型生成回答
+
+作者：开发工程师
+日期：2025年01月
+"""
+
+import logging
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel
+from langchain_milvus import Milvus
+from langchain_openai import ChatOpenAI
+from langchain_community.callbacks.manager import get_openai_callback
+from langfuse.langchain import CallbackHandler as LangfuseCallbackHandler
+from config import Config
+from rerank_service import get_rerank_service
+
+# 配置日志
+logger = logging.getLogger(__name__)
+
+langfuse_handler = LangfuseCallbackHandler()
+
+class SourceDocument(BaseModel):
+    """源文档信息模型"""
+    doc_name: str  # 文档名称
+    doc_path_name: str  # 文档完整路径
+    doc_type: str  # 文档类型
+    content_preview: str  # 内容预览（前200字符）
+    similarity_score: Optional[float] = None  # 相似度分数
+
+
+class TokenUsage(BaseModel):
+    """Token使用情况模型"""
+    prompt_tokens: int  # 输入token数
+    completion_tokens: int  # 输出token数
+    total_tokens: int  # 总token数
+
+
+class RAGResponse(BaseModel):
+    """RAG服务响应模型"""
+    content: str  # 大模型回复给用户的文本内容
+    source_documents: List[SourceDocument]  # 检索使用的源文档列表
+    token_usage: Optional[TokenUsage] = None  # token使用情况
+    success: bool = True  # 查询是否成功
+    error_message: Optional[str] = None  # 错误信息
+    scene_info: Optional[Dict[str, Any]] = None  # 场景检测信息
+
+
+class RAGService:
+    """RAG服务类 (基于LangChain实现)
+    
+    使用LangChain + Milvus向量数据库实现检索增强生成服务
+    支持OpenAI兼容的大模型和嵌入模型
+    """
+    
+    # 场景检测提示词模板
+    SCENE_DETECTION_TEMPLATE = """请分析用户问题属于以下哪种客服场景类型，只返回对应的数字：
+
+1. 产品咨询类 - 询问产品功能、规格、价格等基本信息
+2. 售后服务类 - 退换货、维修、质量问题等售后相关
+3. 账户相关类 - 登录、注册、密码、个人信息等账户问题  
+4. 投诉建议类 - 对服务或产品的投诉、意见、建议
+5. 技术支持类 - 使用方法、故障排除、技术配置等
+6. 其他咨询类 - 不属于以上分类的一般性咨询
+
+用户问题: {question}
+
+请只返回场景类型对应的数字（1-6）："""
+    
+    # 统一的提示词模板 - 专业客服版本
+    PROMPT_TEMPLATE = """你是一位专业的客服人员，请根据提供的知识库内容来回答用户的问题。
+
+知识库内容:
+{context}
+
+用户问题: {question}
+
+回答要求：
+1. 【角色定位】你是一位专业、耐心、友善的客服代表
+2. 【回答原则】严格基于知识库内容回答，不得编造或推测信息
+3. 【准确性要求】
+   - 如果知识库中有明确答案，请准确完整地回答
+   - 如果知识库中信息不完整，说明现有信息并提示用户可联系人工客服获取更详细信息
+   - 如果知识库中完全没有相关信息，请礼貌地说明无法找到相关资料，建议用户转接人工客服
+4. 【回答格式】
+   - 使用纯文本格式，不使用markdown格式
+   - 语言简洁明了，适合微信对话环境
+   - 使用礼貌、专业的语调
+   - 如需列举，使用数字序号或简单的分行
+5. 【转人工提示】当遇到以下情况时，主动建议用户转接人工客服：
+   - 复杂的售后问题
+   - 需要个人账户信息查询的问题
+   - 投诉或纠纷相关问题
+   - 知识库无法覆盖的专业技术问题
+
+请基于以上要求，为用户提供专业的客服回答："""
+    
+    def __init__(self):
+        """初始化RAG服务"""
+        self.vector_store = None
+        self.embeddings = None
+        self.llm = None
+        self.rerank_service = None  # 重排序服务
+        self._initialize_components()
+    
+    def _initialize_components(self):
+        """初始化所有组件"""
+        try:
+            # 初始化嵌入模型
+            self._initialize_embeddings()
+            
+            # 初始化大语言模型
+            self._initialize_llm()
+            
+            # 初始化向量存储
+            self._initialize_vector_store()
+            
+            # 初始化重排序服务
+            self._initialize_rerank_service()
+            
+            logger.info("RAG服务初始化完成")
+            
+        except Exception as e:
+            logger.error(f"RAG服务初始化失败: {e}")
+            raise
+    
+    def _initialize_embeddings(self):
+        """初始化嵌入模型"""
+        try:
+            # 使用自定义嵌入类，兼容OpenAI API
+            from openai import OpenAI
+            
+            class CustomEmbeddings:
+                def __init__(self, model_name: str, api_key: str, api_base: str, dimensions: int):
+                    self.model_name = model_name
+                    self.client = OpenAI(api_key=api_key, base_url=api_base)
+                    self.dimensions = dimensions
+                
+                def embed_query(self, text: str) -> List[float]:
+                    """嵌入单个查询"""
+                    return self.embed_documents([text])[0]
+                
+                def embed_documents(self, texts: List[str]) -> List[List[float]]:
+                    """嵌入多个文档"""
+                    try:
+                        response = self.client.embeddings.create(
+                            model=self.model_name,
+                            input=texts,
+                            dimensions=self.dimensions,
+                            encoding_format="float"
+                        )
+                        return [data.embedding for data in response.data]
+                    except Exception as e:
+                        logger.error(f"嵌入生成失败: {e}")
+                        raise
+            
+            self.embeddings = CustomEmbeddings(
+                model_name=Config.LLM_EMBEDDING_MODEL_NAME,
+                api_key=Config.LLM_EMBEDDING_API_KEY,
+                api_base=Config.LLM_EMBEDDING_BASE_URL,
+                dimensions=Config.MILVUS_VECTOR_DIM
+            )
+            
+            # 测试嵌入模型
+            test_embedding = self.embeddings.embed_query("测试")
+            actual_dim = len(test_embedding)
+            
+            logger.info(f"嵌入模型初始化成功: {Config.LLM_EMBEDDING_MODEL_NAME}")
+            logger.info(f"向量维度: {actual_dim}")
+            
+            if actual_dim != Config.MILVUS_VECTOR_DIM:
+                logger.warning(f"向量维度不匹配! 实际({actual_dim}) != 期望({Config.MILVUS_VECTOR_DIM})")
+            
+        except Exception as e:
+            logger.error(f"嵌入模型初始化失败: {e}")
+            raise
+    
+    def _initialize_llm(self):
+        """初始化大语言模型"""
+        try:
+            self.llm = ChatOpenAI(
+                model=Config.LLM_MODEL_NAME,
+                openai_api_key=Config.LLM_API_KEY,
+                openai_api_base=Config.LLM_BASE_URL,
+                temperature=0.1,
+                max_tokens=800  # 调整为800，平衡详细度和简洁性
+            )
+            
+            logger.info(f"大语言模型初始化成功: {Config.LLM_MODEL_NAME}")
+            
+        except Exception as e:
+            logger.error(f"大语言模型初始化失败: {e}")
+            raise
+    
+    def _initialize_vector_store(self):
+        """初始化向量存储"""
+        try:
+            # 构建Milvus连接参数
+            connection_args = {
+                "host": Config.MILVUS_HOST,
+                "port": Config.MILVUS_PORT,
+                "db_name": Config.MILVUS_DATABASE
+            }
+            
+            # 如果有用户名和密码，添加到连接参数中
+            if Config.MILVUS_USER:
+                connection_args["user"] = Config.MILVUS_USER
+            if Config.MILVUS_PASSWORD:
+                connection_args["password"] = Config.MILVUS_PASSWORD
+            
+            # 配置搜索参数，针对IVF_FLAT索引优化
+            search_params = {
+                "metric_type": "COSINE",  # 与索引保持一致
+                "params": {
+                    "nprobe": 64  # 建议设置为nlist的6.25% (64/1024)，平衡性能和召回率
+                }
+            }
+            
+            # 初始化Milvus向量存储
+            self.vector_store = Milvus(
+                embedding_function=self.embeddings,
+                collection_name=Config.MILVUS_COLLECTION_NAME,
+                connection_args=connection_args,
+                text_field="content",  # 文本内容字段
+                vector_field="content_vector",  # 向量字段
+                auto_id=True,
+                search_params=search_params  # 添加搜索参数
+            )
+            
+            logger.info(f"Milvus向量存储初始化成功: {Config.MILVUS_COLLECTION_NAME}")
+            logger.info(f"搜索参数: nprobe={search_params['params']['nprobe']}")
+            
+        except Exception as e:
+            logger.error(f"Milvus向量存储初始化失败: {e}")
+            raise
+    
+
+
+    def _initialize_rerank_service(self):
+        """初始化重排序服务"""
+        try:
+            self.rerank_service = get_rerank_service()
+            if self.rerank_service:
+                logger.info("重排序服务初始化成功")
+            else:
+                logger.info("重排序服务未配置，将跳过rerank步骤")
+        except Exception as e:
+            logger.warning(f"重排序服务初始化失败: {e}")
+            self.rerank_service = None
+
+    def query_service(self, query: str, use_rerank: bool = True) -> RAGResponse:
+        """核心查询服务方法
+        
+        Args:
+            query: 用户输入的查询内容
+            use_rerank: 是否使用重排序功能
+            
+        Returns:
+            RAGResponse: 包含回答内容、源文档和token使用情况的响应对象
+        """
+        try:
+            logger.info(f"🔍 开始处理查询（rerank={use_rerank}): {query}")
+            
+            if not query or not query.strip():
+                return RAGResponse(
+                    content="请输入有效的查询内容",
+                    source_documents=[],
+                    success=False,
+                    error_message="查询内容为空"
+                )
+            
+            # 第0步：场景检测（可选）
+            # scene_info = self.detect_user_scene(query)
+            scene_info = None # 暂时不使用场景检测
+            
+            # 第一步：向量检索获取候选文档
+            initial_k = 10 if use_rerank and self.rerank_service else 3
+            retriever = self.vector_store.as_retriever(
+                search_type="similarity",
+                search_kwargs={"k": initial_k}
+            )
+            
+            # 获取候选文档
+            candidate_docs = retriever.invoke(query)
+            logger.info(f"📄 初始检索到 {len(candidate_docs)} 个候选文档")
+            
+            # 第二步：重排序（如果启用）
+            final_docs = candidate_docs
+            if use_rerank and self.rerank_service and len(candidate_docs) > 1:
+                # 提取文档内容用于重排序
+                doc_contents = [doc.page_content for doc in candidate_docs]
+                
+                # 增加重排序的top_n数量，确保不会过滤掉高相关度文档
+                rerank_top_n = min(5, len(candidate_docs))  # 从3增加到5，但不超过候选文档数量
+                
+                # 执行重排序
+                rerank_response = self.rerank_service.rerank_documents(
+                    query=query,
+                    documents=doc_contents,
+                    top_n=rerank_top_n
+                )
+                
+                if rerank_response.success:
+                    # 根据重排序结果重新排列文档
+                    reranked_docs = []
+                    for rerank_doc in rerank_response.documents:
+                        if 0 <= rerank_doc.index < len(candidate_docs):
+                            original_doc = candidate_docs[rerank_doc.index]
+                            # 将相关性分数添加到元数据中
+                            if hasattr(original_doc, 'metadata'):
+                                original_doc.metadata['rerank_score'] = rerank_doc.relevance_score
+                            reranked_docs.append(original_doc)
+                    
+                    # 安全检查：确保原始最高相似度文档不会被完全过滤掉
+                    # 如果原始第1个文档不在重排序结果中，将其添加到结果中
+                    if candidate_docs and len(reranked_docs) > 0:
+                        first_doc = candidate_docs[0]
+                        first_doc_in_rerank = any(
+                            hasattr(doc, 'metadata') and 
+                            doc.metadata.get('doc_name') == first_doc.metadata.get('doc_name') and
+                            doc.page_content == first_doc.page_content
+                            for doc in reranked_docs
+                        )
+                        
+                        if not first_doc_in_rerank:
+                            # 将原始最高相似度文档添加到重排序结果的开头
+                            if hasattr(first_doc, 'metadata'):
+                                first_doc.metadata['rerank_score'] = 1.0  # 给予最高分数
+                            reranked_docs.insert(0, first_doc)
+                            logger.info(f"🔒 安全检查：将原始最高相似度文档添加到重排序结果中")
+                    
+                    final_docs = reranked_docs[:3]  # 最终仍然只取前3个
+                    logger.info(f"🔄 重排序完成，选择了 {len(final_docs)} 个文档")
+                else:
+                    logger.warning(f"重排序失败，使用原始检索结果: {rerank_response.error_message}")
+            
+            # 第三步：使用选定的文档生成回答
+            with get_openai_callback() as cb:  ## 在上下文中获取 OpenAI 回调处理器，方便地公开令牌和成本信息
+                # 构建上下文
+                context = "\n\n".join([doc.page_content for doc in final_docs])
+                
+                # 使用统一的提示模板生成回答
+                prompt = self.PROMPT_TEMPLATE.format(context=context, question=query)
+
+                if (Config.LANGFUSE_ENABLE):
+                    answer = self.llm.invoke(prompt, config={"callbacks":[langfuse_handler]}).content
+                else:
+                    answer = self.llm.invoke(prompt).content
+            
+            # 更新文档引用为最终选择的文档
+            source_documents = final_docs
+            
+            logger.info(f"✅ 查询完成，检索到 {len(source_documents)} 个相关文档")
+            logger.info(f"📄 答案长度: {len(answer)} 字符")
+            logger.info(f"💰 Token使用: 输入{cb.prompt_tokens}, 输出{cb.completion_tokens}, 总计{cb.total_tokens}")
+            
+            # 处理源文档信息
+            processed_source_docs = []
+            for i, doc in enumerate(source_documents):
+                try:
+                    # 提取文档元数据
+                    metadata = doc.metadata if hasattr(doc, 'metadata') else {}
+                    doc_name = metadata.get("doc_name", f"文档{i+1}")
+                    doc_path_name = metadata.get("doc_path_name", "")
+                    doc_type = metadata.get("doc_type", "unknown")
+                    rerank_score = metadata.get("rerank_score")
+                    
+                    # 获取内容预览
+                    content_preview = doc.page_content[:200] + "..." if len(doc.page_content) > 200 else doc.page_content
+                    
+                    source_doc = SourceDocument(
+                        doc_name=doc_name,
+                        doc_path_name=doc_path_name,
+                        doc_type=doc_type,
+                        content_preview=content_preview,
+                        similarity_score=rerank_score  # 使用rerank分数
+                    )
+                    processed_source_docs.append(source_doc)
+                    
+                    score_info = f" (rerank: {rerank_score:.3f})" if rerank_score else ""
+                    logger.info(f"📖 文档{i+1}: {doc_name}{score_info} - {content_preview[:50]}...")
+                    
+                except Exception as e:
+                    logger.warning(f"处理源文档{i+1}时出错: {e}")
+                    # 添加默认文档信息
+                    processed_source_docs.append(SourceDocument(
+                        doc_name=f"文档{i+1}",
+                        doc_path_name="",
+                        doc_type="unknown",
+                        content_preview="无法获取文档信息"
+                    ))
+            
+            # 构建token使用情况
+            token_usage = TokenUsage(
+                prompt_tokens=cb.prompt_tokens,
+                completion_tokens=cb.completion_tokens,
+                total_tokens=cb.total_tokens
+            )
+            
+            return RAGResponse(
+                content=answer if answer else "抱歉，我无法根据现有信息回答您的问题。",
+                source_documents=processed_source_docs,
+                token_usage=token_usage,
+                success=True,
+                scene_info=scene_info
+            )
+            
+        except Exception as e:
+            logger.error(f"❌ 查询服务失败: {e}")
+            return RAGResponse(
+                content="",
+                source_documents=[],
+                success=False,
+                error_message=f"查询过程中发生错误：{str(e)}",
+                scene_info=None
+            )
+    
+    def get_similar_documents(self, query: str, top_k: int = 3) -> List[Dict[str, Any]]:
+        """获取相似文档（用于调试和分析）
+        
+        Args:
+            query: 查询内容
+            top_k: 返回文档数量
+            
+        Returns:
+            List[Dict]: 相似文档列表
+        """
+        try:
+            logger.info(f"🔍 搜索相似文档: {query} (top_k={top_k})")
+            
+            # 使用向量存储进行相似性搜索
+            docs = self.vector_store.similarity_search_with_score(query, k=top_k)
+            
+            results = []
+            for doc, score in docs:
+                result = {
+                    "content": doc.page_content,
+                    "metadata": doc.metadata,
+                    "similarity_score": float(score),
+                    "doc_name": doc.metadata.get("doc_name", "未知文档"),
+                    "doc_path_name": doc.metadata.get("doc_path_name", ""),
+                    "doc_type": doc.metadata.get("doc_type", "unknown")
+                }
+                results.append(result)
+            
+            logger.info(f"✅ 找到 {len(results)} 个相似文档")
+            return results
+            
+        except Exception as e:
+            logger.error(f"❌ 获取相似文档失败: {e}")
+            return []
+    
+    def health_check(self) -> Dict[str, Any]:
+        """健康检查
+        
+        Returns:
+            Dict: 健康状态信息
+        """
+        status = {
+            "service": "RAGService",
+            "status": "healthy",
+            "components": {},
+            "timestamp": None
+        }
+        
+        try:
+            from datetime import datetime
+            status["timestamp"] = datetime.now().isoformat()
+            
+            # 检查嵌入模型
+            try:
+                test_embedding = self.embeddings.embed_query("健康检查")
+                status["components"]["embeddings"] = {
+                    "status": "healthy",
+                    "model": Config.LLM_EMBEDDING_MODEL_NAME,
+                    "dimension": len(test_embedding)
+                }
+            except Exception as e:
+                status["components"]["embeddings"] = {
+                    "status": "error",
+                    "error": str(e)
+                }
+                status["status"] = "degraded"
+            
+            # 检查大语言模型
+            try:
+                with get_openai_callback() as cb:
+                    test_response = self.llm.invoke("你好").content
+                status["components"]["llm"] = {
+                    "status": "healthy",
+                    "model": Config.LLM_MODEL_NAME,
+                    "response_length": len(test_response) if test_response else 0,
+                    "test_tokens": cb.total_tokens
+                }
+            except Exception as e:
+                status["components"]["llm"] = {
+                    "status": "error",
+                    "error": str(e)
+                }
+                status["status"] = "degraded"
+            
+            # 检查向量存储
+            try:
+                test_docs = self.vector_store.similarity_search("测试", k=1)
+                status["components"]["vector_store"] = {
+                    "status": "healthy",
+                    "collection": Config.MILVUS_COLLECTION_NAME,
+                    "test_results": len(test_docs)
+                }
+            except Exception as e:
+                status["components"]["vector_store"] = {
+                    "status": "error",
+                    "error": str(e)
+                }
+                status["status"] = "degraded"
+            
+            # 检查重排序服务
+            try:
+                if self.rerank_service:
+                    rerank_health = self.rerank_service.health_check()
+                    status["components"]["rerank_service"] = rerank_health
+                else:
+                    status["components"]["rerank_service"] = {
+                        "status": "not_configured",
+                        "message": "重排序服务未配置"
+                    }
+            except Exception as e:
+                status["components"]["rerank_service"] = {
+                    "status": "error",
+                    "error": str(e)
+                }
+                status["status"] = "degraded"
+            
+        except Exception as e:
+            status["status"] = "error"
+            status["error"] = str(e)
+        
+        return status
+
+    def detect_user_scene(self, query: str) -> Dict[str, Any]:
+        """检测用户问题的场景类型
+        
+        Args:
+            query: 用户问题
+            
+        Returns:
+            Dict: 包含场景类型和建议的字典
+        """
+        try:
+            # 使用LLM进行场景检测
+            prompt = self.SCENE_DETECTION_TEMPLATE.format(question=query)
+            response = self.llm.invoke(prompt).content.strip()
+            
+            # 解析场景类型
+            scene_mapping = {
+                "1": {"type": "产品咨询类", "priority": "normal", "suggest_human": False},
+                "2": {"type": "售后服务类", "priority": "high", "suggest_human": True},
+                "3": {"type": "账户相关类", "priority": "high", "suggest_human": True},
+                "4": {"type": "投诉建议类", "priority": "urgent", "suggest_human": True},
+                "5": {"type": "技术支持类", "priority": "normal", "suggest_human": False},
+                "6": {"type": "其他咨询类", "priority": "normal", "suggest_human": False}
+            }
+            
+            scene_info = scene_mapping.get(response, scene_mapping["6"])
+            scene_info["detected_number"] = response
+            
+            logger.info(f"🎯 场景检测结果: {query} -> {scene_info['type']} (优先级: {scene_info['priority']})")
+            
+            return scene_info
+            
+        except Exception as e:
+            logger.warning(f"场景检测失败: {e}")
+            return {"type": "其他咨询类", "priority": "normal", "suggest_human": False, "detected_number": "6"}
+
+
+# 全局RAG服务实例
+_rag_service_instance = None
+
+def get_rag_service() -> Optional[RAGService]:
+    """获取RAG服务实例（单例模式）"""
+    global _rag_service_instance
+    
+    if _rag_service_instance is None:
+        try:
+            _rag_service_instance = RAGService()
+            logger.info("RAG服务实例创建成功")
+        except Exception as e:
+            logger.error(f"RAG服务实例创建失败: {e}")
+            return None
+    
+    return _rag_service_instance
+
+
+# 便捷函数
+def query_question(question: str) -> RAGResponse:
+    """查询问题的便捷函数
+    
+    Args:
+        question: 用户问题
+        
+    Returns:
+        RAGResponse: 查询响应
+    """
+    rag_service = get_rag_service()
+    if rag_service is None:
+        return RAGResponse(
+            content="",
+            source_documents=[],
+            success=False,
+            error_message="RAG服务初始化失败"
+        )
+    
+    return rag_service.query_service(question)
+
+
+if __name__ == "__main__":
+    # 测试代码 - 专业客服RAG系统
+    rag = get_rag_service()
+
+
+    # 检测健康状态
+    health = rag.health_check()
+    print(f"健康状态: {health}")
+
+
+    # 测试不同场景的客服问题
+    test_cases = [
+        "帮我介绍一下盗窃罪",  # 其他咨询类
+        "我要退货，怎么办理？",    # 售后服务类
+        "忘记密码了，如何重置？",  # 账户相关类
+        "你们的服务太差了！",      # 投诉建议类
+        "产品无法连接WiFi",       # 技术支持类
+    ]
+
+    print(100 * "=")
+    
+    for i, question in enumerate(test_cases, 1):
+        print(100 * "*")
+        print(f"\n=== 测试案例 {i}: {question} ===")
+        
+        # 测试场景检测
+        scene_info = rag.detect_user_scene(question)
+        print(f"🎯 场景类型: {scene_info['type']}")
+        print(f"🚨 优先级: {scene_info['priority']}")
+        print(f"👤 建议转人工: {'是' if scene_info['suggest_human'] else '否'}")
+        
+        # 测试完整查询
+        response = rag.query_service(question, use_rerank=True)
+        print(f"💬 客服回答: {response.content}")
+        if response.scene_info:
+            print(f"📊 场景信息: {response.scene_info['type']}")
+        if response.token_usage:
+            print(f"💰 Token使用: {response.token_usage.total_tokens}")
+        print(f"📚 参考文档数: {len(response.source_documents)}")
+        
+    print(100 * "=")
+    
+    print("\n=== 测试经典产品咨询 ===")
+    response = rag.query_service("介绍一下老人与海这本书", use_rerank=True)
+    print(f"回答: {response.content}")
+    print(f"场景信息: {response.scene_info}")
+    print(f"源文档数量: {len(response.source_documents)}")
+    if response.token_usage:
+        print(f"Token使用: {response.token_usage.total_tokens}")
+
+    for doc in response.source_documents:
+        print("--------------------------------")
+        print(f"源文档: {doc.doc_name}")
+        print(f"相似度分数: {doc.similarity_score}")
+        print(f"内容预览: {doc.content_preview[:50]}...")
+        

mildoc_wxkf/requirements.txt

@@ -0,0 +1,12 @@
+Flask==3.1.2
+pycryptodome==3.23.0
+requests==2.32.5
+python-dotenv==1.2.1
+
+
+# langchain相关依赖
+langchain==1.1.3
+langchain-openai==1.1.3
+langchain-milvus==0.3.1
+langchain-community==0.4.1
+# langchain-core==1.2.0

mildoc_wxkf/rerank_service.py

@@ -0,0 +1,326 @@
+"""
+重排序服务模块
+
+支持多个重排序服务提供商：
+- 阿里百炼平台 (dashscope)
+- 硅基流动平台 (siliconflow)
+
+作者：开发工程师
+日期：2025年01月
+"""
+
+import logging
+import requests
+import json
+from enum import Enum
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel
+from config import Config
+
+# 配置日志
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class RerankProvider(Enum):
+    """重排序服务提供商"""
+    DASHSCOPE = "dashscope"  # 阿里百炼
+    SILICONFLOW = "siliconflow"  # 硅基流动
+
+
+class RerankDocument(BaseModel):
+    """重排序文档模型"""
+    index: int  # 原始文档索引
+    content: str  # 文档内容
+    relevance_score: float  # 相关性分数
+    metadata: Optional[Dict[str, Any]] = None  # 元数据
+
+
+class RerankResponse(BaseModel):
+    """重排序响应模型"""
+    documents: List[RerankDocument]  # 重排序后的文档
+    success: bool = True
+    error_message: Optional[str] = None
+
+
+class RerankService:
+    """重排序服务类
+    
+    支持多个重排序服务提供商，通过统一接口提供文档重排序功能
+    """
+    
+    def __init__(self, provider: RerankProvider, api_key: str, model_name: str, endpoint: Optional[str] = None):
+        """初始化重排序服务
+        
+        Args:
+            provider: 服务提供商
+            api_key: API密钥
+            model_name: 模型名称
+            endpoint: 自定义API端点（可选）
+        """
+        self.provider = provider
+        self.api_key = api_key
+        self.model_name = model_name
+        self.endpoint = endpoint
+        logger.info(f"重排序服务初始化完成: provider={provider.value}, model={model_name}, endpoint={self.endpoint}")
+    
+    def rerank_documents(self, query: str, documents: List[str], top_n: Optional[int] = None) -> RerankResponse:
+        """重排序文档
+        
+        Args:
+            query: 查询文本
+            documents: 文档列表
+            top_n: 返回前N个文档，默认返回全部
+            
+        Returns:
+            RerankResponse: 重排序结果
+        """
+        try:
+            if not query or not documents:
+                logger.warning("查询或文档列表为空")
+                return RerankResponse(
+                    documents=[],
+                    success=False,
+                    error_message="查询或文档列表为空"
+                )
+            
+            logger.info(f"🔄 开始重排序: 查询='{query[:50]}...', 文档数量={len(documents)}, top_n={top_n}")
+            
+            if self.provider == RerankProvider.DASHSCOPE:
+                response = self._rerank_dashscope(query, documents, top_n)
+            elif self.provider == RerankProvider.SILICONFLOW:
+                response = self._rerank_siliconflow(query, documents, top_n)
+            else:
+                raise ValueError(f"不支持的重排序提供商: {self.provider}")
+            
+            if response.success:
+                logger.info(f"✅ 重排序完成: 返回{len(response.documents)}个文档")
+                for i, doc in enumerate(response.documents): 
+                    logger.info(f"  #{i+1}: 相关性={doc.relevance_score:.4f}, 内容='{doc.content[:50]}...'")
+            
+            return response
+                
+        except Exception as e:
+            logger.error(f"❌ 重排序失败: {e}")
+            return RerankResponse(
+                documents=[],
+                success=False,
+                error_message=str(e)
+            )
+    
+    def _rerank_dashscope(self, query: str, documents: List[str], top_n: Optional[int] = None) -> RerankResponse:
+        """阿里百炼平台重排序"""
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json"
+        }
+        
+        data = {
+            "model": self.model_name,
+            "input": {
+                "query": query,
+                "documents": documents
+            },
+            "parameters": {
+                "return_documents": True,  # 显式设置返回文档内容
+                "top_n": top_n or len(documents)
+            }
+        }
+        
+        logger.debug(f"🌐 发送请求到百炼平台: {self.endpoint}")
+        response = requests.post(self.endpoint, headers=headers, json=data, timeout=30)
+        response.raise_for_status()
+        
+        result = response.json()
+        logger.debug(f"📨 百炼平台响应: {json.dumps(result, ensure_ascii=False)[:200]}...")
+        
+        # 解析响应
+        rerank_docs = []
+        if "output" in result and "results" in result["output"]:
+            for item in result["output"]["results"]:
+                # 百炼平台返回格式: document.text
+                document = item.get("document", {})
+                content = document.get("text", "") if isinstance(document, dict) else str(document)
+                
+                rerank_doc = RerankDocument(
+                    index=item.get("index", 0),
+                    content=content,
+                    relevance_score=float(item.get("relevance_score", 0.0))
+                )
+                rerank_docs.append(rerank_doc)
+        else:
+            logger.warning(f"百炼平台响应格式异常: {result}")
+            return RerankResponse(
+                documents=[],
+                success=False,
+                error_message="响应格式异常"
+            )
+        
+        return RerankResponse(documents=rerank_docs)
+    
+    def _rerank_siliconflow(self, query: str, documents: List[str], top_n: Optional[int] = None) -> RerankResponse:
+        """硅基流动平台重排序"""
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json"
+        }
+        
+        data = {
+            "model": self.model_name,
+            "query": query,
+            "documents": documents,
+            "return_documents": True  # 硅基流动需要明确指定返回文档内容
+        }
+        
+        # 硅基流动支持top_n参数
+        if top_n is not None:
+            data["top_n"] = top_n
+        
+        logger.debug(f"🌐 发送请求到硅基流动: {self.endpoint}")
+        response = requests.post(self.endpoint, headers=headers, json=data, timeout=30)
+        response.raise_for_status()
+        
+        result = response.json()
+        logger.debug(f"📨 硅基流动响应: {json.dumps(result, ensure_ascii=False)[:200]}...")
+        
+        # 解析响应
+        rerank_docs = []
+        if "results" in result:
+            for item in result["results"]:
+                # 硅基流动平台返回格式: document.text
+                document = item.get("document", {})
+                content = document.get("text", "") if isinstance(document, dict) else str(document)
+                
+                rerank_doc = RerankDocument(
+                    index=item.get("index", 0),
+                    content=content,
+                    relevance_score=float(item.get("relevance_score", 0.0))
+                )
+                rerank_docs.append(rerank_doc)
+        else:
+            logger.warning(f"硅基流动响应格式异常: {result}")
+            return RerankResponse(
+                documents=[],
+                success=False,
+                error_message="响应格式异常"
+            )
+        
+        return RerankResponse(documents=rerank_docs)
+    
+    def health_check(self) -> Dict[str, Any]:
+        """健康检查"""
+        status = {
+            "service": "RerankService",
+            "provider": self.provider.value,
+            "model": self.model_name,
+            "endpoint": self.endpoint,
+            "status": "unknown"
+        }
+        
+        try:
+            # 简单的健康检查：用最少的文档进行测试
+            test_response = self.rerank_documents(
+                query="测试",
+                documents=["这是一个测试文档"],
+                top_n=1
+            )
+            
+            if test_response.success:
+                status["status"] = "healthy"
+                status["test_result"] = "ok"
+            else:
+                status["status"] = "error"
+                status["error"] = test_response.error_message
+                
+        except Exception as e:
+            status["status"] = "error"
+            status["error"] = str(e)
+        
+        return status
+
+
+def create_rerank_service() -> Optional[RerankService]:
+    """创建重排序服务实例（工厂函数）
+    
+    Returns:
+        RerankService: 重排序服务实例，失败时返回None
+    """
+    try:
+        # 检查配置
+        if not Config.RERANK_PROVIDER:
+            logger.info("未配置RERANK_PROVIDER，跳过重排序服务初始化")
+            return None
+        
+        if not Config.RERANK_API_KEY:
+            logger.warning("缺少RERANK_API_KEY配置")
+            return None
+            
+        if not Config.RERANK_MODEL_NAME:
+            logger.warning("缺少RERANK_MODEL_NAME配置")
+            return None
+        
+        # 创建提供商枚举
+        try:
+            provider = RerankProvider(Config.RERANK_PROVIDER.lower())
+        except ValueError:
+            logger.error(f"不支持的重排序提供商: {Config.RERANK_PROVIDER}")
+            return None
+        
+        # 创建服务实例
+        service = RerankService(
+            provider=provider,
+            api_key=Config.RERANK_API_KEY,
+            model_name=Config.RERANK_MODEL_NAME,
+            endpoint=Config.RERANK_ENDPOINT
+        )
+        
+        logger.info(f"重排序服务创建成功: {provider.value}")
+        return service
+        
+    except Exception as e:
+        logger.error(f"创建重排序服务失败: {e}")
+        return None
+
+
+# 全局重排序服务实例
+_rerank_service_instance = None
+
+def get_rerank_service() -> Optional[RerankService]:
+    """获取重排序服务实例（单例模式）"""
+    global _rerank_service_instance
+    
+    if _rerank_service_instance is None:
+        _rerank_service_instance = create_rerank_service()
+    
+    return _rerank_service_instance
+
+
+if __name__ == "__main__":
+    # 测试代码
+    rerank_service = get_rerank_service()
+    if rerank_service:
+        # 健康检查
+        health = rerank_service.health_check()
+        print(f"健康检查: {health}")
+        
+        # 测试重排序
+        test_docs = [
+            "苹果是一种水果，含有丰富的维生素",
+            "量子计算是计算科学的前沿领域，苹果也在做这方面的研究",
+            "苹果是一家科技企业，主要生产手机、电脑等产品"
+        ]
+        
+        result = rerank_service.rerank_documents(
+            query="苹果公司",
+            documents=test_docs,
+            top_n=3
+        )
+        
+        if result.success:
+            print("\n重排序结果:")
+            for i, doc in enumerate(result.documents):
+                print(f"{i+1}. [分数: {doc.relevance_score:.4f}] {doc.content}")
+        else:
+            print(f"重排序失败: {result.error_message}")
+    else:
+        print("重排序服务未配置或初始化失败")

mildoc_wxkf/run_mildoc_wxkf

@@ -0,0 +1,8 @@
+#!/bin/bash
+echo "Mildoc KF Starting... "
+
+uv sync
+
+nohup uv run gunicorn --workers 1 --bind 0.0.0.0:8890 wxkf_callback_app:app >> mildoc_wxkf.log 2>&1 &
+
+# uv run gunicorn --workers 1 --bind 0.0.0.0:8890 wxkf_callback_app:app

mildoc_wxkf/wecom_api.py

@@ -0,0 +1,321 @@
+"""
+企业微信API调用模块
+支持客服消息的读取和发送功能
+
+作者：开发工程师
+日期：2025年01月
+"""
+
+import time
+import logging
+import requests
+from typing import Dict, Optional
+from config import Config
+
+logger = logging.getLogger(__name__)
+
+class WeComAPI:
+    """企业微信API调用类"""
+    
+    def __init__(self):
+        self.corp_id = Config.CORP_ID
+        self.app_secret = Config.APP_SECRET
+        self.base_url = "https://qyapi.weixin.qq.com/cgi-bin"
+        self._kf_access_token = None
+        self._kf_token_expires_at = 0
+    
+    
+    def get_kf_access_token(self) -> Optional[str]:
+        """获取客服专用access_token"""
+        # 检查token是否过期
+        if self._kf_access_token and time.time() < self._kf_token_expires_at:
+            return self._kf_access_token
+        
+        # 配置客服密钥
+        secret = self.app_secret
+        if not secret:
+            logger.error("缺少APP_SECRET配置，无法获取客服access_token")
+            return None
+        
+        url = f"{self.base_url}/gettoken"
+        params = {
+            'corpid': self.corp_id,
+            'corpsecret': secret
+        }
+        
+        try:
+            response = requests.get(url, params=params, timeout=10)
+            response.raise_for_status()
+            data = response.json()
+            
+            if data.get('errcode') == 0:
+                self._kf_access_token = data.get('access_token')
+                expires_in = data.get('expires_in', 7200)
+                self._kf_token_expires_at = time.time() + expires_in - 300  # 提前5分钟过期
+                logger.info(f"获取客服access_token成功，有效期: {expires_in}秒")
+                return self._kf_access_token
+            else:
+                logger.error(f"获取客服access_token失败: {data.get('errmsg')}")
+                return None
+                
+        except Exception as e:
+            logger.error(f"获取客服access_token异常: {e}")
+            return None
+    
+    def sync_kf_messages(self, token: str, open_kfid: str = "", cursor: str = "", limit: int = 1000) -> Optional[Dict]:
+        """
+        读取客服消息
+        
+        Args:
+            token: 回调事件返回的token字段
+            open_kfid: 指定拉取某个客服账号的消息
+            cursor: 上一次调用时返回的next_cursor
+            limit: 期望请求的数据量，默认1000
+            
+        Returns:
+            消息列表数据
+        """
+        access_token = self.get_kf_access_token()
+        if not access_token:
+            return None
+        
+        url = f"{self.base_url}/kf/sync_msg"
+        params = {'access_token': access_token}
+        
+        data = {
+            'token': token,
+            'limit': limit,
+            'voice_format': 0  # 0-Amr 1-Silk
+        }
+        
+        if cursor:
+            data['cursor'] = cursor
+        if open_kfid:
+            data['open_kfid'] = open_kfid
+        
+        try:
+            response = requests.post(url, params=params, json=data, timeout=10)
+            response.raise_for_status()
+            result = response.json()
+            
+            if result.get('errcode') == 0:
+                logger.info(f"读取客服消息成功，获取到 {len(result.get('msg_list', []))} 条消息")
+                return result
+            else:
+                logger.error(f"读取客服消息失败: {result.get('errmsg')}")
+                return None
+                
+        except Exception as e:
+            logger.error(f"读取客服消息异常: {e}")
+            return None
+    
+    def send_kf_message(self, touser: str, open_kfid: str, msgtype: str, content: Dict, msgid: str = None) -> Optional[Dict]:
+        """
+        发送客服消息
+        
+        Args:
+            touser: 接收消息的客户UserID
+            open_kfid: 发送消息的客服帐号ID
+            msgtype: 消息类型 (text, image, voice, video, file, link, miniprogram, msgmenu, location)
+            content: 消息内容
+            msgid: 指定消息ID
+            
+        Returns:
+            发送结果
+        """
+        access_token = self.get_kf_access_token()
+        if not access_token:
+            return None
+        
+        url = f"{self.base_url}/kf/send_msg"
+        params = {'access_token': access_token}
+        
+        data = {
+            'touser': touser,
+            'open_kfid': open_kfid,
+            'msgtype': msgtype
+        }
+        
+        # 添加消息内容
+        data[msgtype] = content
+        
+        if msgid:
+            data['msgid'] = msgid
+        
+        try:
+            response = requests.post(url, params=params, json=data, timeout=10)
+            response.raise_for_status()
+            result = response.json()
+            
+            if result.get('errcode') == 0:
+                logger.info(f"发送客服消息成功，msgid: {result.get('msgid')}")
+                return result
+            else:
+                logger.error(f"发送客服消息失败: {result.get('errmsg')}")
+                return None
+                
+        except Exception as e:
+            logger.error(f"发送客服消息异常: {e}")
+            return None
+    
+    def send_kf_text_message(self, touser: str, open_kfid: str, content: str, msgid: str = None) -> Optional[Dict]:
+        """发送文本消息"""
+        text_content = {'content': content}
+        return self.send_kf_message(touser, open_kfid, 'text', text_content, msgid)
+    
+    def send_kf_image_message(self, touser: str, open_kfid: str, media_id: str, msgid: str = None) -> Optional[Dict]:
+        """发送图片消息"""
+        image_content = {'media_id': media_id}
+        return self.send_kf_message(touser, open_kfid, 'image', image_content, msgid)
+    
+    def send_kf_link_message(self, touser: str, open_kfid: str, title: str, desc: str, url: str, thumb_media_id: str, msgid: str = None) -> Optional[Dict]:
+        """发送链接消息"""
+        link_content = {
+            'title': title,
+            'desc': desc,
+            'url': url,
+            'thumb_media_id': thumb_media_id
+        }
+        return self.send_kf_message(touser, open_kfid, 'link', link_content, msgid)
+        
+    def send_event_response_message(self, code: str, content: str) -> Optional[Dict]:
+        """
+        发送事件响应消息（如欢迎语、结束语）
+        
+        Args:
+            code: 事件响应码
+            msgtype: 消息类型 (text, msgmenu)
+            content: 消息内容
+            
+        Returns:
+            发送结果
+        """
+        access_token = self.get_kf_access_token()
+        if not access_token:
+            return None
+        
+        url = f"{self.base_url}/kf/send_msg_on_event"
+        params = {'access_token': access_token}
+        
+        data = {
+            'code': code,
+            'msgtype': 'text',
+            'text': {'content': content}
+        }
+        
+        try:
+            response = requests.post(url, params=params, json=data, timeout=10)
+            response.raise_for_status()
+            result = response.json()
+            
+            if result.get('errcode') == 0:
+                logger.info(f"发送事件响应消息成功，code: {code}")
+                return result
+            else:
+                logger.error(f"发送事件响应消息失败: {result.get('errmsg')}")
+                return None
+                
+        except Exception as e:
+            logger.error(f"发送事件响应消息异常: {e}")
+            return None
+    
+    def get_service_session_state(self, external_userid: str, open_kfid: str) -> Optional[Dict]:
+        """
+        获取会话状态
+        
+        Args:
+            open_kfid: 客服帐号ID
+            external_userid: 用户的external_userid
+            
+        Returns:
+            会话状态数据，包含以下字段:
+            - service_state: 会话状态 (0: 未接待, 1: 由智能助手接待, 2: 接待池等待中, 3: 人工接待, 4: 用户已确认接待结束)
+            - service_userid: 接待客服的userid (当 service_state 为3时返回)
+            - service_session_id: 会话ID
+        """
+        access_token = self.get_kf_access_token()
+        if not access_token:
+            return None
+        
+        url = f"{self.base_url}/kf/service_state/get"
+        params = {'access_token': access_token}
+        
+        data = {
+            'open_kfid': open_kfid,
+            'external_userid': external_userid
+        }
+        
+        try:
+            response = requests.post(url, params=params, json=data, timeout=10)
+            response.raise_for_status()
+            result = response.json()
+            
+            if result.get('errcode') == 0:
+                logger.info(f"获取会话状态成功，状态: {result.get('service_state')}")
+                return result
+            else:
+                logger.error(f"获取会话状态失败: {result.get('errmsg')}")
+                return None
+                
+        except Exception as e:
+            logger.error(f"获取会话状态异常: {e}")
+            return None
+    
+    def update_service_session_state(self, external_userid: str, open_kfid: str, service_state: int, service_userid: str = None) -> Optional[Dict]:
+        """
+        变更会话状态
+        
+        Args:
+            open_kfid: 客服帐号ID
+            external_userid: 用户的external_userid
+            service_state: 要变更的会话状态 (0: 结束接待, 1: 开启由智能助手接待, 2: 进入接待池由人工接待)
+            service_userid: 接待客服的userid (当service_state为3时必填)
+            
+        Returns:
+            变更结果
+        """
+        access_token = self.get_kf_access_token()
+        if not access_token:
+            return None
+        
+        url = f"{self.base_url}/kf/service_state/trans"
+        params = {'access_token': access_token}
+        
+        data = {
+            'open_kfid': open_kfid,
+            'external_userid': external_userid,
+            'service_state': service_state
+        }
+        
+        # 当状态为3（开启由人工接待）时，必须指定接待人
+        if service_state == 3:
+            if not service_userid:
+                logger.error("变更会话状态失败: service_state为3时必须指定service_userid")
+                return None
+            data['service_userid'] = service_userid
+        
+        try:
+
+            logger.info(f"变更会话状态请求: {data}")
+            logger.info(f"变更会话状态请求: {url}")
+            logger.info(f"变更会话状态请求: {params}")
+
+            response = requests.post(url, params=params, json=data, timeout=10)
+            response.raise_for_status()
+            result = response.json()
+            
+            if result.get('errcode') == 0:
+                logger.info(f"变更会话状态成功，状态: {service_state}")
+                return result
+            else:
+                logger.error(f"变更会话状态失败: {result.get('errmsg')}")
+                return None
+                
+        except Exception as e:
+            logger.error(f"变更会话状态异常: {e}")
+            return None
+    
+
+
+# 全局API实例
+wecom_api = WeComAPI() 

mildoc_wxkf/wxkf_callback_app.py

@@ -0,0 +1,381 @@
+"""
+企业微信回调服务端
+功能：接收企业微信的回调验证和消息推送
+
+作者：开发工程师
+日期：2025年01月
+"""
+
+import os
+import logging
+import json
+from urllib.parse import unquote
+from flask import Flask, request, abort
+from WXBizMsgCrypt import WXBizMsgCrypt
+from config import Config
+
+# 配置日志
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# 创建Flask应用
+app = Flask(__name__)
+
+def get_wecom_config():
+    """获取企业微信配置，优先从环境变量获取"""
+    corp_id = Config.CORP_ID
+    token = Config.TOKEN
+    encoding_aes_key = Config.ENCODING_AES_KEY
+    
+    if not token:
+        logger.error("缺少TOKEN配置，请设置环境变量TOKEN或修改代码中的TOKEN变量")
+        return None, None, None
+    
+    if not encoding_aes_key:
+        logger.error("缺少ENCODING_AES_KEY配置，请设置环境变量WECOM_ENCODING_AES_KEY或修改代码中的ENCODING_AES_KEY变量")
+        return None, None, None
+    
+    return corp_id, token, encoding_aes_key
+
+
+@app.before_request
+def log_request_info():
+    """记录请求信息"""
+    # 记录请求的基本信息
+    logger.info(f"=== 收到HTTP请求 ===")
+    logger.info(f"请求方法: {request.method}")
+    logger.info(f"请求URL: {request.url}")
+    logger.info(f"请求路径: {request.path}")
+    logger.info(f"客户端IP: {request.remote_addr}")
+    
+    # 记录请求头
+    headers = dict(request.headers)
+    logger.info(f"请求头: {json.dumps(headers, indent=2, ensure_ascii=False)}")
+    
+    # 记录查询参数
+    if request.args:
+        args = dict(request.args)
+        logger.info(f"查询参数: {json.dumps(args, indent=2, ensure_ascii=False)}")
+    
+    # 记录请求体（仅对POST/PUT等方法）
+    if request.method in ['POST', 'PUT', 'PATCH']:
+        try:
+            if request.content_type and 'application/json' in request.content_type:
+                # JSON数据
+                json_data = request.get_json()
+                if json_data:
+                    logger.info(f"请求JSON: {json.dumps(json_data, indent=2, ensure_ascii=False)}")
+            else:
+                # 原始数据
+                raw_data = request.get_data(as_text=True)
+                if raw_data:
+                    logger.info(f"请求体: {raw_data[:500]}...")  # 限制长度避免日志过长
+        except Exception as e:
+            logger.warning(f"读取请求体时出错: {e}")
+
+@app.after_request
+def log_response_info(response):
+    """记录响应信息"""
+    logger.info(f"=== HTTP响应 ===")
+    logger.info(f"响应状态码: {response.status_code}")
+    logger.info(f"响应状态: {response.status}")
+    
+    # 记录响应头
+    headers = dict(response.headers)
+    logger.info(f"响应头: {json.dumps(headers, indent=2, ensure_ascii=False)}")
+    
+    # 记录响应内容
+    try:
+        if response.content_type and 'application/json' in response.content_type:
+            # JSON响应
+            logger.info(f"响应JSON: {response.get_data(as_text=True)}")
+        else:
+            # 其他类型响应
+            response_data = response.get_data(as_text=True)
+            if response_data:
+                if len(response_data) > 500:
+                    logger.info(f"响应内容: {response_data[:500]}...")
+                else:
+                    logger.info(f"响应内容: {response_data}")
+            else:
+                logger.info("响应内容: 无内容")
+    except Exception as e:
+        logger.warning(f"读取响应内容时出错: {e}")
+    
+    logger.info(f"=== 请求处理完成 ===\n")
+    return response
+
+
+def get_wxcrypt():
+    """
+    获取企业微信加解密工具对象
+    """
+    corp_id, token, encoding_aes_key = get_wecom_config()
+    if not all([corp_id, token, encoding_aes_key]):
+        logger.error("企业微信配置不完整")
+        abort(500)
+    return WXBizMsgCrypt(token, encoding_aes_key, corp_id)
+
+
+@app.route('/callback/command', methods=['GET'])
+def wecom_callback_get():
+    """
+    企业微信回调接口
+    GET请求：用于验证回调URL的有效性
+    """
+    # 获取加解密工具对象
+    wxcrypt = get_wxcrypt()
+    
+    # 获取URL参数
+    msg_signature = request.args.get('msg_signature', '')
+    timestamp = request.args.get('timestamp', '')
+    nonce = request.args.get('nonce', '')
+    
+    logger.info(f"收到回调请求 - Method: {request.method}, msg_signature: {msg_signature}, timestamp: {timestamp}, nonce: {nonce}")
+    
+    # URL验证
+    echostr = request.args.get('echostr', '')
+    if not echostr:
+        logger.error("GET请求缺少echostr参数")
+        abort(400)
+    
+    if not all([msg_signature, timestamp, nonce]):
+        logger.error("GET请求缺少必要参数")
+        abort(400)
+    
+    try:
+        
+        # URL解码echostr参数
+        echostr = unquote(echostr)
+        logger.info(f"开始验证URL - echostr: {echostr[:50]}...")
+        
+        logger.info(f"echostr: {echostr}")
+
+        # 验证URL并解密echostr
+        result = wxcrypt.VerifyURL(msg_signature, timestamp, nonce, echostr)
+        
+        logger.info(f"验证URL结果: {result}")
+
+        # 检查返回值类型
+        if isinstance(result, tuple):
+            ret, reply_echostr = result
+        else:
+            ret = result
+            reply_echostr = None
+        
+        logger.info(f"验证结果 - 返回码: {ret}")
+        if ret != 0:
+            logger.error(f"URL验证失败，错误码: {ret}")
+            if ret == -40001:
+                logger.error("签名验证失败 - 请检查Token配置是否与企业微信后台一致")
+            elif ret == -40002:
+                logger.error("AES解密失败或CorpID不匹配 - 请检查EncodingAESKey和CorpID配置")
+            else:
+                logger.error(f"未知错误码: {ret}")
+            abort(403)
+        
+        logger.info("URL验证成功")
+        return reply_echostr
+    except Exception as e:
+        logger.error(f"URL验证过程发生错误: {str(e)}")
+        import traceback
+        logger.error(f"错误详情: {traceback.format_exc()}")
+        abort(500)
+        
+    
+
+
+
+@app.route('/callback/command', methods=['POST'])
+def wecom_callback_post():
+    """
+    企业微信回调接口
+    POST请求：接收企业微信推送的消息和事件
+    """
+    # 获取加解密工具对象
+    wxcrypt = get_wxcrypt()
+    
+    # 获取URL参数
+    msg_signature = request.args.get('msg_signature', '')
+    timestamp = request.args.get('timestamp', '')
+    nonce = request.args.get('nonce', '')
+    
+    logger.info(f"收到回调请求 - Method: {request.method}, msg_signature: {msg_signature}, timestamp: {timestamp}, nonce: {nonce}")
+    
+    # 接收消息
+    if not all([msg_signature, timestamp, nonce]):
+        logger.error("POST请求缺少必要参数")
+        abort(400)
+        
+    try:
+        # 获取POST数据
+        post_data = request.get_data(as_text=True)
+        if not post_data:
+            logger.error("POST请求体为空")
+            abort(400)
+        
+        logger.info(f"收到POST消息: {post_data[:200]}...")
+        
+        # 解密消息
+        ret, msg = wxcrypt.DecryptMsg(post_data, msg_signature, timestamp, nonce)
+        
+        if ret != 0:
+            logger.error(f"消息解密失败，错误码: {ret}")
+            abort(403)
+        
+        logger.info(f"消息解密成功: {msg}")
+        
+        # 处理消息（这里可以根据业务需求进行扩展）
+        response_msg = handle_message(msg)
+        
+        if response_msg:
+            # 加密响应消息
+            ret, encrypted_msg = wxcrypt.EncryptMsg(response_msg, nonce, timestamp)
+            if ret == 0:
+                logger.info("响应消息加密成功")
+                return encrypted_msg
+            else:
+                logger.error(f"响应消息加密失败，错误码: {ret}")
+        
+        # 返回空字符串表示成功接收但不回复
+        return ''
+        
+    except Exception as e:
+        logger.error(f"处理POST请求时发生错误: {str(e)}")
+        abort(500)
+    
+
+def handle_message(msg):
+    """
+    处理解密后的消息
+    
+    Args:
+        msg: 解密后的XML消息
+        
+    Returns:
+        str: 要回复的消息（XML格式），如果不需要回复则返回None
+    """
+    try:
+        import xml.etree.ElementTree as ET
+        import time
+        
+        # 解析XML消息
+        root = ET.fromstring(msg)
+        msg_type = root.find('MsgType').text if root.find('MsgType') is not None else ''
+        msg_id = root.find('MsgId').text if root.find('MsgId') is not None else ''
+        
+        logger.info(f"处理消息 - 类型: {msg_type}, 消息ID: {msg_id}")
+        
+        # 根据消息类型进行处理
+        if msg_type == 'event':
+            # 处理事件消息
+            event = root.find('Event').text if root.find('Event') is not None else ''
+            logger.info(f"收到事件: {event}")
+            
+            return process_event_message(event, root)
+                
+        # 对于其他类型的消息，记录日志但不回复
+        logger.info(f"收到其他类型消息，暂不处理: {msg_type}")
+        return None
+        
+    except Exception as e:
+        logger.error(f"处理消息时发生错误: {str(e)}")
+        import traceback
+        logger.error(f"错误详情: {traceback.format_exc()}")
+        return None
+
+
+
+def process_event_message(event, root):
+    """
+    处理事件消息
+    
+    Args:
+        event: 事件类型
+        from_user: 发送者
+        to_user: 接收者
+        root: XML根节点
+        
+    Returns:
+        str: 回复消息，如果不需要回复则返回None
+    """
+    if event == 'kf_msg_or_event':
+        # 微信客服消息或事件
+        logger.info("收到微信客服事件，开始处理客服消息")
+        
+        # 获取Token和OpenKfId
+        token = root.find('Token').text if root.find('Token') is not None else ''
+        open_kfid = root.find('OpenKfId').text if root.find('OpenKfId') is not None else ''
+        
+        if token and open_kfid:
+            # 异步处理客服消息（避免阻塞回调响应）
+            import threading
+            
+            def process_kf_messages():
+                try:
+                    # 延迟导入避免循环导入
+                    from kf_message_handler import KfMessageHandler
+                    kf_handler = KfMessageHandler()
+                    kf_handler.process_kf_event(token, open_kfid)
+                except Exception as e:
+                    logger.error(f"处理客服消息异常: {e}")
+            
+            # 启动后台线程处理
+            thread = threading.Thread(target=process_kf_messages)
+            thread.daemon = True
+            thread.start()
+            
+            logger.info(f"已启动客服消息处理线程 - OpenKfId: {open_kfid}")
+        else:
+            logger.error("客服事件缺少必要参数 - Token或OpenKfId为空")
+        
+        # 客服事件不需要回复
+        return None
+    
+    # 其他事件类型
+    logger.info(f"收到其他事件类型: {event}")
+    return None
+
+@app.route('/health', methods=['GET'])
+def health_check():
+    """健康检查接口"""
+    return {'status': 'ok', 'message': '企业微信回调服务运行正常'}
+
+@app.route('/', methods=['GET'])
+def index():
+    return '''
+    <h1>企业微信回调服务</h1>
+    <p>服务正在运行...</p>
+    '''
+
+
+
+if __name__ == '__main__':
+    # 检查配置
+    corp_id, token, encoding_aes_key = get_wecom_config()
+    
+    if not all([corp_id, token, encoding_aes_key]):
+        logger.error("配置不完整，请检查企业微信相关配置")
+        logger.info("请设置以下环境变量或修改代码中的配置:")
+        logger.info("- WECOM_CORP_ID: 企业ID")
+        logger.info("- WECOM_TOKEN: 应用Token")
+        logger.info("- WECOM_ENCODING_AES_KEY: 应用EncodingAESKey")
+        exit(1)
+    
+    logger.info("企业微信回调服务启动中...")
+    logger.info(f"企业ID: {corp_id}")
+    logger.info(f"Token: {token[:10]}..." if token else "Token: 未配置")
+    logger.info(f"EncodingAESKey: {encoding_aes_key[:10]}..." if encoding_aes_key else "EncodingAESKey: 未配置")
+
+    # 启动服务
+    port = Config.PORT
+    logger.info(f"启动服务 - 端口: {port}")
+    host = Config.HOST
+    logger.info(f"启动服务 - 主机: {host}")
+    debug = Config.DEBUG
+    logger.info(f"启动服务 - 调试模式: {debug}")
+
+    app.run(host=host, port=port, debug=debug, threaded=True, processes=1) 

pyproject.toml

@@ -0,0 +1,37 @@
+[project]
+name = "mildoc"
+version = "1.1.0"
+description = "mildoc 是一个企业知识库管理系统。"
+readme = "README.md"
+requires-python = "==3.12.*"
+dependencies = [
+    "alibabacloud-oss-v2==1.2.1",
+    "aliyun-mns-sdk==1.3.0",
+    "av==16.0.1",
+    "flask==3.1.2",
+    "flask-login==0.6.3",
+    "gunicorn==23.0.0",
+    "langchain==1.1.3",
+    "langchain-community==0.4.1",
+    "langchain-core==1.2.0",
+    "langchain-milvus==0.3.1",
+    "langchain-openai==1.1.3",
+    "langchain-text-splitters==1.1.0",
+    "langfuse==3.12.0",
+    "markitdown[docx,pdf,pptx,xls,xlsx]==0.1.4",
+    # "mineru[core]==2.7.0",
+    "minio==7.2.20",
+    "openai==2.11.0",
+    "pycryptodome==3.23.0",
+    "pymilvus==2.6.5",
+    "pypdf2==3.0.1",
+    "python-dotenv==1.2.1",
+    "pytz==2025.2",
+    "requests==2.32.5",
+]
+
+
+[[tool.uv.index]]
+name = "aliyun"
+url = "https://mirrors.aliyun.com/pypi/simple/"
+default = true

requirements-all.txt

@@ -0,0 +1,40 @@
+# 环境变量管理
+python-dotenv==1.2.1
+
+# 日期处理
+pytz==2025.2
+
+# Flask Web框架
+Flask==3.1.2
+Flask-Login==0.6.3
+
+pycryptodome==3.23.0
+requests==2.32.5
+
+# PDF文档解析
+PyPDF2==3.0.1
+
+# Office文档解析
+markitdown[all]==0.1.4
+
+# Minio对象存储客户端
+minio==7.2.20
+
+# Milvus向量数据库客户端
+pymilvus==2.6.5
+
+
+# OpenAI API客户端（用于embedding）
+openai==2.11.0
+
+# langchain相关依赖
+langchain==1.1.3
+langchain-openai==1.1.3
+langchain-milvus==0.3.1
+langchain-community==0.4.1
+langchain-core==1.2.0
+langchain-text-splitters==1.1.0
+
+# 阿里云
+alibabacloud-oss-v2==1.2.1
+aliyun-mns-sdk==1.3.0