init: 初始化 dpb 桃育种系统代码库
前后端 + 后端 FastAPI 全量源码、部署脚本与文档。
This commit is contained in:
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
from typing import Any
|
||||
|
||||
from agno.agent import Agent
|
||||
from agno.models.openai.like import OpenAILike
|
||||
from agno.team import Team
|
||||
|
||||
from app.config.setting import settings
|
||||
|
||||
|
||||
class AgnoFactory:
|
||||
"""Agno 工厂类 - 统一管理 Agent、Team 创建逻辑"""
|
||||
|
||||
# 配置常量
|
||||
AGENT_DESCRIPTION = "你是一个有用的AI助手,可以帮助用户回答问题和提供帮助。"
|
||||
AGENT_INSTRUCTIONS = ["保持回答简洁明了", "如果不确定,请说明"]
|
||||
AGENT_EXPECTED_OUTPUT = "中文回答"
|
||||
AGENT_TEMPERATURE = 0.7
|
||||
NUM_HISTORY_RUNS = 3
|
||||
REQUEST_TIMEOUT = 60.0 # LLM 请求总超时(秒),流式响应需放长
|
||||
CONNECT_TIMEOUT = 10.0 # TCP 连接超时(秒)
|
||||
|
||||
def create_agent(
|
||||
self,
|
||||
user_id: str,
|
||||
team_id: str,
|
||||
session_id: str,
|
||||
db: Any | None = None,
|
||||
model_config: dict[str, Any] | None = None,
|
||||
) -> Team:
|
||||
"""创建带 Agent 的 Team 实例。
|
||||
|
||||
参数:
|
||||
- user_id (str): 用户标识。
|
||||
- team_id (str): 团队标识。
|
||||
- session_id (str): 会话 ID。
|
||||
- db (Any | None): Agno 持久化数据库实例,可选。
|
||||
- model_config (dict | None): 运行时模型配置,覆盖系统默认。
|
||||
支持字段:base_url, api_key, model_id, temperature。
|
||||
|
||||
返回:
|
||||
- Team: 配置好的 Team。
|
||||
"""
|
||||
# 优先使用运行时配置,否则 fallback 到系统 settings
|
||||
base_url = settings.OPENAI_BASE_URL
|
||||
api_key = settings.OPENAI_API_KEY
|
||||
model_id = settings.OPENAI_MODEL
|
||||
temperature = self.AGENT_TEMPERATURE
|
||||
|
||||
if model_config:
|
||||
base_url = model_config.get("base_url") or base_url
|
||||
api_key = model_config.get("api_key") or api_key
|
||||
model_id = model_config.get("model_id") or model_id
|
||||
if isinstance(model_config.get("temperature"), (int, float)):
|
||||
temperature = float(model_config["temperature"])
|
||||
|
||||
# 创建 Agent
|
||||
fastapiadmin_agent = Agent(
|
||||
id=user_id,
|
||||
name="fastapiadmin_agent",
|
||||
role="You are a helpful AI assistant",
|
||||
description=self.AGENT_DESCRIPTION,
|
||||
tools=[],
|
||||
)
|
||||
|
||||
# 创建 Team
|
||||
fastapiadmin_team = Team(
|
||||
id=team_id,
|
||||
user_id=user_id,
|
||||
session_id=session_id,
|
||||
model=OpenAILike(
|
||||
id=model_id,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
temperature=temperature,
|
||||
timeout=self.REQUEST_TIMEOUT,
|
||||
),
|
||||
members=[fastapiadmin_agent],
|
||||
instructions=self.AGENT_INSTRUCTIONS,
|
||||
expected_output=self.AGENT_EXPECTED_OUTPUT,
|
||||
add_datetime_to_context=True,
|
||||
add_history_to_context=True,
|
||||
markdown=True,
|
||||
num_history_runs=self.NUM_HISTORY_RUNS,
|
||||
input_schema=None,
|
||||
output_schema=None,
|
||||
parse_response=True,
|
||||
read_chat_history=True,
|
||||
db=db,
|
||||
)
|
||||
|
||||
return fastapiadmin_team
|
||||
@@ -0,0 +1,62 @@
|
||||
"""每日 02:00 数据库备份 job —— 包装 backend/scripts/pg_backup.sh。
|
||||
|
||||
独立脚本可手动/CI 直跑;系统调度经此包装,保证与脚本同一份备份逻辑与轮转策略。
|
||||
Windows 下须显式定位 Git Bash(PATH 里的 bash 可能是 WSL shim,mount/PATH 均不匹配),
|
||||
脚本路径统一转 MSYS /d/... 格式传给子进程。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# backend/app/utils/backup_job.py → parents[2] = backend/
|
||||
_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "pg_backup.sh"
|
||||
|
||||
_GIT_BASH_CANDIDATES = (
|
||||
Path(r"C:\Program Files\Git\bin\bash.exe"),
|
||||
Path(r"C:\Program Files\Git\usr\bin\bash.exe"),
|
||||
)
|
||||
|
||||
|
||||
def _find_git_bash() -> str | None:
|
||||
for cand in _GIT_BASH_CANDIDATES:
|
||||
if cand.exists():
|
||||
return str(cand)
|
||||
return shutil.which("bash")
|
||||
|
||||
|
||||
def _msys_path(p: Path) -> str:
|
||||
"""D:/dpb/... → /d/dpb/...(Git Bash 挂载约定)。"""
|
||||
s = p.as_posix()
|
||||
if len(s) >= 2 and s[1] == ":":
|
||||
return f"/{s[0].lower()}{s[2:]}"
|
||||
return s
|
||||
|
||||
|
||||
async def run_daily_backup() -> bool:
|
||||
"""每日备份:调用 pg_backup.sh(独立脚本,pg_dump 自定义格式 + N 天轮转)。"""
|
||||
script = _SCRIPT
|
||||
if not script.exists():
|
||||
alt = Path.cwd() / "scripts" / "pg_backup.sh"
|
||||
script = alt if alt.exists() else script
|
||||
if not script.exists():
|
||||
logger.error(f"备份脚本不存在: {script}")
|
||||
return False
|
||||
git_bash = _find_git_bash()
|
||||
if not git_bash:
|
||||
logger.error("未找到 Git Bash(C:/Program Files/Git/bin/bash.exe)")
|
||||
return False
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
git_bash, _msys_path(script),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
if proc.returncode == 0:
|
||||
logger.info(f"每日备份完成: {stdout.decode().strip()}")
|
||||
return True
|
||||
logger.error(f"每日备份失败(rc={proc.returncode}): {stderr.decode()[:500]}")
|
||||
return False
|
||||
@@ -0,0 +1,13 @@
|
||||
from app.config.path_conf import BANNER_FILE
|
||||
|
||||
|
||||
def worship() -> str:
|
||||
"""读取启动 Banner(优先 `banner.txt`)。
|
||||
获取地址:https://patorjk.com/software/taag/#p=testall&f=Fire+Font-k&t=fastapiadmin%0A&x=none&v=4&h=4&w=80&we=false
|
||||
|
||||
返回:
|
||||
- str: banner 文本。
|
||||
"""
|
||||
if BANNER_FILE.exists():
|
||||
return BANNER_FILE.read_text(encoding="utf-8")
|
||||
return ""
|
||||
@@ -0,0 +1,459 @@
|
||||
import importlib
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Generator, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from sqlalchemy.engine.row import Row
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
from sqlalchemy.orm.collections import InstrumentedList
|
||||
from sqlalchemy.sql.elements import Null
|
||||
from sqlalchemy.sql.expression import null
|
||||
|
||||
from app.config.path_conf import STATIC_DIR
|
||||
from app.core.exceptions import CustomException
|
||||
from app.core.logger import logger
|
||||
|
||||
|
||||
def import_module(module: str, desc: str) -> Any:
|
||||
"""动态导入模块
|
||||
|
||||
参数:
|
||||
- module (str): 模块名称。
|
||||
- desc (str): 模块描述。
|
||||
|
||||
返回:
|
||||
- Any: 模块对象。
|
||||
"""
|
||||
try:
|
||||
module_path, module_class = module.rsplit(".", 1)
|
||||
module = importlib.import_module(module_path) # pyright: ignore[reportAssignmentType]
|
||||
return getattr(module, module_class)
|
||||
except ModuleNotFoundError:
|
||||
logger.error(f"❗️ 导入{desc}失败,未找到模块:{module}")
|
||||
raise
|
||||
except AttributeError:
|
||||
logger.error(f"❗ ️导入{desc}失败,未找到模块方法:{module}")
|
||||
raise
|
||||
|
||||
|
||||
async def import_modules_async(modules: list, desc: str, **kwargs) -> None:
|
||||
"""异步导入模块列表
|
||||
|
||||
参数:
|
||||
- modules (list[str]): 模块列表。
|
||||
- desc (str): 模块描述。
|
||||
- kwargs: 额外参数。
|
||||
|
||||
返回:
|
||||
- None
|
||||
"""
|
||||
for module in modules:
|
||||
if not module:
|
||||
continue
|
||||
try:
|
||||
module_path = module[0 : module.rindex(".")]
|
||||
module_name = module[module.rindex(".") + 1 :]
|
||||
module_obj = importlib.import_module(module_path)
|
||||
await getattr(module_obj, module_name)(**kwargs)
|
||||
except ModuleNotFoundError:
|
||||
logger.error(f"❌️ 导入{desc}失败,未找到模块:{module}")
|
||||
raise
|
||||
except AttributeError:
|
||||
logger.error(f"❌️ 导入{desc}失败,未找到模块方法:{module}")
|
||||
raise
|
||||
|
||||
|
||||
def get_random_character() -> str:
|
||||
"""生成随机字符串
|
||||
|
||||
返回:
|
||||
- str: 随机字符串。
|
||||
"""
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def uuid4_str() -> str:
|
||||
"""数据库引擎 UUID 类型兼容:返回无连字符的 UUID 字符串。
|
||||
|
||||
返回:
|
||||
- str: UUID 字符串。
|
||||
"""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def search_to_dict(search: Any, default: Any = None) -> dict | None:
|
||||
"""将 Pydantic 查询模型转为参数字典。
|
||||
|
||||
- search 为 None 时返回 default(默认 None)。
|
||||
- search 非空时调用 model_dump() 提取字段字典,排除 None 值。
|
||||
- 自动合并 ``_start`` / ``_end`` 时间范围对为 ``("between", [start, end])``。
|
||||
- 自动将 ``json_schema_extra={"q": "op"}`` 标记的字段转为 ``("op", value)`` 元组。
|
||||
|
||||
参数:
|
||||
- search: Pydantic 查询模型或 None。
|
||||
- default: search 为 None 时的返回值,默认 None。
|
||||
|
||||
返回:
|
||||
- dict | None: 查询参数字典或 None。
|
||||
"""
|
||||
if not search:
|
||||
return default
|
||||
d = search.model_dump(exclude_none=True)
|
||||
|
||||
# 处理数组格式的时间范围参数 → ("between", [start, end])
|
||||
for key in list(d.keys()):
|
||||
if isinstance(d[key], list) and len(d[key]) == 2 and key.endswith("_time"):
|
||||
d[key] = ("between", d[key])
|
||||
|
||||
for field_name, field_info in search.model_fields.items():
|
||||
if field_name not in d:
|
||||
continue
|
||||
q_op = (field_info.json_schema_extra or {}).get("q")
|
||||
if q_op:
|
||||
d[field_name] = (q_op, d[field_name])
|
||||
return d
|
||||
|
||||
|
||||
def get_parent_id_map(model_list: Sequence[DeclarativeBase]) -> dict[int, int]:
|
||||
"""获取父级 ID 映射字典
|
||||
|
||||
参数:
|
||||
- model_list (Sequence[DeclarativeBase]): 模型列表。
|
||||
|
||||
返回:
|
||||
- Dict[int, int]: {id: parent_id} 映射字典。
|
||||
"""
|
||||
return {item.id: item.parent_id for item in model_list} # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
|
||||
def get_parent_recursion(id: int, id_map: dict[int, int], ids: list[int] | None = None) -> list[int]:
|
||||
"""递归获取所有父级 ID
|
||||
|
||||
参数:
|
||||
- id (int): 当前 ID。
|
||||
- id_map (dict[int, int]): ID 映射字典。
|
||||
- ids (list[int] | None): 已收集的 ID 列表。
|
||||
|
||||
返回:
|
||||
- list[int]: 所有父级 ID 列表。
|
||||
"""
|
||||
ids = ids or []
|
||||
if id in ids:
|
||||
raise CustomException(msg="递归获取父级ID失败,不可以自引用")
|
||||
ids.append(id)
|
||||
parent_id = id_map.get(id)
|
||||
if parent_id:
|
||||
get_parent_recursion(parent_id, id_map, ids)
|
||||
return ids
|
||||
|
||||
|
||||
def get_child_id_map(
|
||||
model_list: Sequence[DeclarativeBase],
|
||||
) -> dict[int, list[int]]:
|
||||
"""获取子级 ID 映射字典
|
||||
|
||||
参数:
|
||||
- model_list (Sequence[DeclarativeBase]): 模型列表。
|
||||
|
||||
返回:
|
||||
- Dict[int, List[int]]: {id: [child_ids]} 映射字典。
|
||||
"""
|
||||
data_map = {}
|
||||
for model in model_list:
|
||||
data_map.setdefault(model.id, []) # pyright: ignore[reportAttributeAccessIssue]
|
||||
if model.parent_id: # pyright: ignore[reportAttributeAccessIssue]
|
||||
data_map.setdefault(model.parent_id, []).append(model.id) # pyright: ignore[reportAttributeAccessIssue]
|
||||
return data_map
|
||||
|
||||
|
||||
def get_child_recursion(id: int, id_map: dict[int, list[int]], ids: list[int] | None = None) -> list[int]:
|
||||
"""递归获取所有子级 ID
|
||||
|
||||
参数:
|
||||
- id (int): 当前 ID。
|
||||
- id_map (dict[int, list[int]]): ID 映射字典。
|
||||
- ids (list[int] | None): 已收集的 ID 列表。
|
||||
|
||||
返回:
|
||||
- list[int]: 所有子级 ID 列表。
|
||||
"""
|
||||
ids = ids or []
|
||||
ids.append(id)
|
||||
for child in id_map.get(id, []):
|
||||
get_child_recursion(child, id_map, ids)
|
||||
return ids
|
||||
|
||||
|
||||
def traversal_to_tree(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""通过遍历算法构造树形结构
|
||||
|
||||
参数:
|
||||
- nodes (list[dict[str, Any]]): 树节点列表。
|
||||
|
||||
返回:
|
||||
- list[dict[str, Any]]: 构造后的树形结构列表。
|
||||
"""
|
||||
tree: list[dict[str, Any]] = []
|
||||
node_dict = {node["id"]: node for node in nodes}
|
||||
|
||||
for node in nodes:
|
||||
# 确保每个节点都有children字段,即使没有子节点也设置为null
|
||||
if "children" not in node:
|
||||
node["children"] = None
|
||||
|
||||
parent_id = node["parent_id"]
|
||||
if parent_id is None:
|
||||
tree.append(node)
|
||||
else:
|
||||
parent_node = node_dict.get(parent_id)
|
||||
if parent_node is not None:
|
||||
if "children" not in parent_node or parent_node["children"] is None:
|
||||
parent_node["children"] = []
|
||||
if node not in parent_node["children"]:
|
||||
parent_node["children"].append(node)
|
||||
elif node not in tree:
|
||||
tree.append(node)
|
||||
|
||||
# 确保所有节点都有children字段
|
||||
for node in tree:
|
||||
if "children" not in node:
|
||||
node["children"] = None
|
||||
|
||||
return tree
|
||||
|
||||
|
||||
def recursive_to_tree(nodes: list[dict[str, Any]], *, parent_id: int | None = None) -> list[dict[str, Any]]:
|
||||
"""通过递归算法构造树形结构(性能影响较大)
|
||||
|
||||
参数:
|
||||
- nodes (list[dict[str, Any]]): 树节点列表。
|
||||
- parent_id (int | None): 父节点 ID,默认为 None 表示根节点。
|
||||
|
||||
返回:
|
||||
- list[dict[str, Any]]: 构造后的树形结构列表。
|
||||
"""
|
||||
tree: list[dict[str, Any]] = []
|
||||
for node in nodes:
|
||||
if node["parent_id"] == parent_id:
|
||||
child_nodes = recursive_to_tree(nodes, parent_id=node["id"])
|
||||
if child_nodes:
|
||||
node["children"] = child_nodes
|
||||
tree.append(node)
|
||||
return tree
|
||||
|
||||
|
||||
def bytes2human(n: int, format_str: str = "%(value).1f%(symbol)s") -> str:
|
||||
"""字节数转人类可读格式
|
||||
Used by various scripts. See:
|
||||
http://goo.gl/zeJZl
|
||||
|
||||
>>> bytes2human(10000)
|
||||
'9.8K'
|
||||
>>> bytes2human(100001221)
|
||||
'95.4M'
|
||||
|
||||
参数:
|
||||
- n (int): 字节数。
|
||||
- format_str (str): 格式化字符串,默认 '%(value).1f%(symbol)s'。
|
||||
|
||||
返回:
|
||||
- str: 可读的字节字符串,如 '1.5MB'。
|
||||
"""
|
||||
symbols = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
|
||||
prefix = {s: 1 << (i + 1) * 10 for i, s in enumerate(symbols[1:])}
|
||||
for symbol in reversed(symbols[1:]):
|
||||
if n >= prefix[symbol]:
|
||||
value = float(n) / prefix[symbol]
|
||||
return format_str % locals()
|
||||
return format_str % {"symbol": symbols[0], "value": n}
|
||||
|
||||
|
||||
def bytes2file_response(bytes_info: bytes) -> Generator[bytes, Any, None]:
|
||||
"""将字节内容封装为单块流式生成器,供文件下载响应使用。
|
||||
|
||||
参数:
|
||||
- bytes_info (bytes): 文件二进制内容。
|
||||
|
||||
返回:
|
||||
- Generator[bytes, Any, None]: 仅 yield 一次的字节生成器。
|
||||
"""
|
||||
yield bytes_info
|
||||
|
||||
|
||||
def get_filepath_from_url(url: str) -> Path:
|
||||
"""工具方法:根据请求参数获取文件路径
|
||||
|
||||
参数:
|
||||
- url (str): 请求参数中的 url 参数。
|
||||
|
||||
返回:
|
||||
- Path: 文件路径。
|
||||
"""
|
||||
file_info = url.split("?")[1].split("&")
|
||||
task_id = file_info[0].split("=")[1]
|
||||
file_name = file_info[1].split("=")[1]
|
||||
task_path = file_info[2].split("=")[1]
|
||||
filepath = STATIC_DIR.joinpath(task_path, task_id, file_name)
|
||||
|
||||
return filepath
|
||||
|
||||
|
||||
def compute_menu_route_first_segment(
|
||||
parent_catalog_id: int | None, # noqa: ARG001
|
||||
package_name: str,
|
||||
module_name: str | None, # noqa: ARG001
|
||||
) -> str:
|
||||
"""前端页面路由首段 — 始终使用 ``module_xxx`` 作为路由首段。
|
||||
|
||||
中性位置函数,供 ``Jinja2TemplateUtil`` 和 ``GenTableService``
|
||||
双方模块级安全导入,避免 ``jinja2_template_util ↔ gencode.service`` 循环依赖。
|
||||
"""
|
||||
pn = (package_name or "").strip()
|
||||
if not pn:
|
||||
raise CustomException(msg="包名不能为空")
|
||||
return pn if pn.startswith("module_") else f"module_{pn}"
|
||||
|
||||
|
||||
class SqlalchemyUtil:
|
||||
"""sqlalchemy工具类
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def base_to_dict(
|
||||
cls,
|
||||
obj: DeclarativeBase | dict[str, Any],
|
||||
transform_case: Literal["no_case", "snake_to_camel", "camel_to_snake"] = "no_case",
|
||||
):
|
||||
"""将 SQLAlchemy 模型或字典转为普通 dict,并可做键名大小写转换。
|
||||
|
||||
参数:
|
||||
- obj (DeclarativeBase | dict[str, Any]): 模型实例或字典。
|
||||
- transform_case (Literal[...]): no_case / snake_to_camel / camel_to_snake。
|
||||
|
||||
返回:
|
||||
- dict[str, Any]: 扁平字典结果。
|
||||
"""
|
||||
if isinstance(obj, DeclarativeBase):
|
||||
base_dict = obj.__dict__.copy()
|
||||
base_dict.pop("_sa_instance_state", None)
|
||||
for name, value in base_dict.items():
|
||||
if isinstance(value, InstrumentedList):
|
||||
base_dict[name] = cls.serialize_result(value, "snake_to_camel")
|
||||
elif isinstance(obj, dict):
|
||||
base_dict = obj.copy()
|
||||
if transform_case == "snake_to_camel":
|
||||
return {CamelCaseUtil.snake_to_camel(k): v for k, v in base_dict.items()}
|
||||
if transform_case == "camel_to_snake":
|
||||
return {SnakeCaseUtil.camel_to_snake(k): v for k, v in base_dict.items()}
|
||||
|
||||
return base_dict
|
||||
|
||||
@classmethod
|
||||
def serialize_result(
|
||||
cls,
|
||||
result: Any,
|
||||
transform_case: Literal["no_case", "snake_to_camel", "camel_to_snake"] = "no_case",
|
||||
):
|
||||
"""将 SQLAlchemy 查询结果(模型、列表、Row 等)递归序列化为可 JSON 化结构。
|
||||
|
||||
参数:
|
||||
- result (Any): ORM 对象、列表、Row 等。
|
||||
- transform_case (Literal[...]): 键名转换策略。
|
||||
|
||||
返回:
|
||||
- Any: 序列化后的 Python 内置类型或嵌套结构。
|
||||
"""
|
||||
if isinstance(result, (DeclarativeBase, dict)):
|
||||
return cls.base_to_dict(result, transform_case)
|
||||
if isinstance(result, list):
|
||||
return [cls.serialize_result(row, transform_case) for row in result]
|
||||
if isinstance(result, Row):
|
||||
if all(isinstance(row, DeclarativeBase) for row in result):
|
||||
return [cls.base_to_dict(row, transform_case) for row in result]
|
||||
if any(isinstance(row, DeclarativeBase) for row in result):
|
||||
return [cls.serialize_result(row, transform_case) for row in result]
|
||||
result_dict = result._asdict()
|
||||
if transform_case == "snake_to_camel":
|
||||
return {CamelCaseUtil.snake_to_camel(k): v for k, v in result_dict.items()}
|
||||
if transform_case == "camel_to_snake":
|
||||
return {SnakeCaseUtil.camel_to_snake(k): v for k, v in result_dict.items()}
|
||||
return result_dict
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def get_server_default_null(cls, dialect_name: str, need_explicit_null: bool = True) -> Null | None:
|
||||
"""按方言返回列默认值中的 NULL 表达(PostgreSQL 可显式 DEFAULT NULL)。
|
||||
|
||||
参数:
|
||||
- dialect_name (str): 数据库方言名。
|
||||
- need_explicit_null (bool): 是否生成显式 NULL 默认值。
|
||||
|
||||
返回:
|
||||
- Null | None: SQLAlchemy null() 或 None。
|
||||
"""
|
||||
if need_explicit_null and dialect_name == "postgres":
|
||||
return null()
|
||||
return None
|
||||
|
||||
|
||||
class CamelCaseUtil:
|
||||
"""下划线形式(snake_case)转小驼峰形式(camelCase)工具方法
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def snake_to_camel(cls, snake_str: str):
|
||||
"""下划线形式 (snake_case) 转为大驼峰形式 (PascalCase)。
|
||||
|
||||
参数:
|
||||
- snake_str (str): 下划线分隔字符串。
|
||||
|
||||
返回:
|
||||
- str: 合并首字母大写后的驼峰字符串。
|
||||
"""
|
||||
words = snake_str.split("_")
|
||||
return "".join(word.capitalize() for word in words)
|
||||
|
||||
@classmethod
|
||||
def transform_result(cls, result: Any):
|
||||
"""将查询结果递归序列化并将键名转为小驼峰。
|
||||
|
||||
参数:
|
||||
- result (Any): ORM 查询结果或嵌套结构。
|
||||
|
||||
返回:
|
||||
- Any: 小驼峰键名的序列化结果。
|
||||
"""
|
||||
return SqlalchemyUtil.serialize_result(result=result, transform_case="snake_to_camel")
|
||||
|
||||
|
||||
class SnakeCaseUtil:
|
||||
"""小驼峰形式(camelCase)转下划线形式(snake_case)工具方法
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def camel_to_snake(cls, camel_str: str):
|
||||
"""小驼峰形式 (camelCase) 转为下划线形式 (snake_case)。
|
||||
|
||||
参数:
|
||||
- camel_str (str): 驼峰字符串。
|
||||
|
||||
返回:
|
||||
- str: 下划线分隔且全小写。
|
||||
"""
|
||||
# 在大写字母前添加一个下划线,然后将整个字符串转为小写
|
||||
words = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", camel_str)
|
||||
return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", words).lower()
|
||||
|
||||
@classmethod
|
||||
def transform_result(cls, result: Any):
|
||||
"""将查询结果递归序列化并将键名转为下划线形式。
|
||||
|
||||
参数:
|
||||
- result (Any): ORM 查询结果或嵌套结构。
|
||||
|
||||
返回:
|
||||
- Any: 下划线键名的序列化结果。
|
||||
"""
|
||||
return SqlalchemyUtil.serialize_result(result=result, transform_case="camel_to_snake")
|
||||
@@ -0,0 +1,122 @@
|
||||
from datetime import datetime
|
||||
|
||||
from rich import box, get_console
|
||||
from rich.console import Group
|
||||
from rich.panel import Panel
|
||||
from rich.rule import Rule
|
||||
from rich.table import Table
|
||||
from rich.text import Text
|
||||
|
||||
from app.config.setting import settings
|
||||
|
||||
console = get_console()
|
||||
|
||||
|
||||
def console_start(
|
||||
host: str,
|
||||
port: int,
|
||||
reload: bool,
|
||||
*,
|
||||
database_ready: bool | None = None,
|
||||
redis_ready: bool | None = None,
|
||||
scheduler_ready: bool | None = None,
|
||||
) -> None:
|
||||
"""在终端输出 Rich 面板:服务信息、组件就绪状态与文档链接。
|
||||
|
||||
参数:
|
||||
- host (str): 监听主机。
|
||||
- port (int): 监听端口。
|
||||
- reload (bool): 是否开启热重载。
|
||||
- database_ready (bool | None): 数据库是否就绪。
|
||||
- redis_ready (bool | None): Redis 是否就绪。
|
||||
- scheduler_ready (bool | None): 调度器是否就绪。
|
||||
|
||||
返回:
|
||||
- None
|
||||
"""
|
||||
env_label = settings.ENVIRONMENT.value if hasattr(settings.ENVIRONMENT, 'value') else settings.ENVIRONMENT
|
||||
url = f"http://{host}:{port}"
|
||||
base_url = f"{url}{settings.ROOT_PATH}"
|
||||
docs_url = base_url + settings.DOCS_URL
|
||||
frontend_url = base_url + settings.WEB_URL
|
||||
|
||||
def _status_text(ready: bool | None) -> str:
|
||||
return "✅ 就绪" if ready else "❌ 失败"
|
||||
|
||||
# 标题
|
||||
title_text = Text(f"\n{settings.TITLE} v{settings.VERSION}", style="bold green")
|
||||
|
||||
# 服务信息
|
||||
info_grid = Table.grid(padding=(0, 1))
|
||||
info_grid.add_column(justify="right")
|
||||
info_grid.add_column()
|
||||
info_grid.add_row("服务地址", url, style="bold blue")
|
||||
info_grid.add_row("运行环境", env_label, style="bold yellow")
|
||||
info_grid.add_row("重载配置", "✅ 启动" if reload else "❌ 关闭")
|
||||
|
||||
# 组件状态 — 一行四个,│ 分隔分组
|
||||
sep = Text(" │ ", style="dim")
|
||||
status_grid = Table.grid(padding=(0, 1))
|
||||
status_grid.add_column(justify="right")
|
||||
status_grid.add_column()
|
||||
status_grid.add_column(justify="right")
|
||||
status_grid.add_column()
|
||||
status_grid.add_column(justify="right")
|
||||
status_grid.add_column()
|
||||
status_grid.add_column(justify="right")
|
||||
status_grid.add_column()
|
||||
status_grid.add_row(
|
||||
"PostgreSQL", _status_text(database_ready),
|
||||
sep,
|
||||
"Redis", _status_text(redis_ready),
|
||||
sep,
|
||||
"调度器", _status_text(scheduler_ready),
|
||||
)
|
||||
|
||||
# 文档链接
|
||||
docs_grid = Table.grid(padding=(0, 1))
|
||||
docs_grid.add_column(justify="right")
|
||||
docs_grid.add_column()
|
||||
docs_grid.add_row("Swagger", Text(docs_url, style=f"blue link {docs_url}"))
|
||||
docs_grid.add_row("前端", Text(frontend_url, style=f"blue link {frontend_url}"))
|
||||
|
||||
final_content = Group(
|
||||
title_text,
|
||||
info_grid,
|
||||
Rule(style="dim"),
|
||||
status_grid,
|
||||
Rule(style="dim"),
|
||||
docs_grid,
|
||||
)
|
||||
|
||||
result = Panel(
|
||||
renderable=final_content,
|
||||
title=f"[bold purple]🚀 FastapiAdmin v{settings.VERSION}[/]",
|
||||
border_style="green",
|
||||
box=box.HEAVY,
|
||||
padding=(0, 2),
|
||||
)
|
||||
|
||||
console.print(result)
|
||||
|
||||
|
||||
def console_end() -> None:
|
||||
"""在终端输出服务关闭提示面板。
|
||||
|
||||
返回:
|
||||
- None
|
||||
"""
|
||||
shutdown_content = Text()
|
||||
shutdown_content.append("🛑 ", style="bold red")
|
||||
shutdown_content.append("FastapiAdmin 服务关闭")
|
||||
shutdown_content.append(f"\n⏰ {datetime.now().strftime('%H:%M:%S')}")
|
||||
shutdown_content.append("\n👋 感谢使用!", style="dim")
|
||||
|
||||
result = Panel(
|
||||
shutdown_content,
|
||||
title="[bold red]服务关闭[/]",
|
||||
border_style="red",
|
||||
padding=(1, 2),
|
||||
)
|
||||
|
||||
console.print(result)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""导入 Excel 时把中文 dict_label 翻译为 sys_dict 的英码 dict_value。
|
||||
|
||||
背景:breeding 模块的枚举字段在方案 B 下统一以英码(dict_value)落库、以中文(dict_label)
|
||||
展示。Excel 模板里用户填的是中文 label,写入前必须经 sys_dict 翻译为英码,否则库里会混进
|
||||
中文、与字典对不齐,下拉/回显也会错乱。
|
||||
|
||||
本解析器按 auth+db 直接查 sys_dict_data(带 label->value 缓存,单次导入只查一次),
|
||||
避免了 redis 依赖、也无需改动 import 控制器。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.base_schema import AuthSchema
|
||||
from app.core.exceptions import CustomException
|
||||
|
||||
|
||||
_value2label_cache: dict[str, dict[str, str]] | None = None
|
||||
|
||||
|
||||
def dict_value_to_label(dict_type: str, value: Any) -> Any:
|
||||
"""把英码 dict_value 翻译为中文 dict_label(导出用)。
|
||||
|
||||
与 DictLabelResolver 方向相反:导出时把库内的英码还原为中文,便于用户阅读;
|
||||
值空或无法识别时原样返回,保证"导出的文件再导入"仍可解析。
|
||||
使用同步引擎读取 sys_dict_data 并进程内缓存,避免每次导出都查库。
|
||||
"""
|
||||
if value is None:
|
||||
return value
|
||||
text = str(value)
|
||||
if text == "":
|
||||
return value
|
||||
global _value2label_cache
|
||||
if _value2label_cache is None:
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.api.v1.module_system.dict.model import DictDataModel
|
||||
from app.core.database import db_session
|
||||
|
||||
mapping: dict[str, dict[str, str]] = {}
|
||||
with db_session() as session:
|
||||
rows = session.execute(
|
||||
select(DictDataModel.dict_type, DictDataModel.dict_value, DictDataModel.dict_label)
|
||||
.where(DictDataModel.is_deleted.is_(False))
|
||||
).all()
|
||||
for dt, dv, dl in rows:
|
||||
mapping.setdefault(dt, {})[dv] = dl
|
||||
_value2label_cache = mapping
|
||||
return _value2label_cache.get(dict_type, {}).get(text, value)
|
||||
|
||||
|
||||
class DictLabelResolver:
|
||||
"""把一组字典类型的中文 label 解析为英码 value,支持批量复用。"""
|
||||
|
||||
def __init__(self, auth: AuthSchema, db: AsyncSession, dict_types: list[str]) -> None:
|
||||
self._auth = auth
|
||||
self._db = db
|
||||
self._dict_types = dict_types
|
||||
self._loaded = False
|
||||
# dict_type -> {label: value}
|
||||
self._label2value: dict[str, dict[str, str]] = {}
|
||||
# dict_type -> {value} (支持再导入"导出文件"——文件里已是英码)
|
||||
self._values: dict[str, set[str]] = {}
|
||||
|
||||
async def _ensure_loaded(self) -> None:
|
||||
if self._loaded:
|
||||
return
|
||||
from app.api.v1.module_system.dict.crud import DictDataCRUD
|
||||
|
||||
crud = DictDataCRUD(self._auth, self._db)
|
||||
for dt in self._dict_types:
|
||||
rows = await crud.get_list(search={"dict_type": dt})
|
||||
self._label2value[dt] = {row.dict_label: row.dict_value for row in rows}
|
||||
self._values[dt] = {row.dict_value for row in rows}
|
||||
self._loaded = True
|
||||
|
||||
async def resolve(self, dict_type: str, label: Any, *, required: bool = True) -> Any:
|
||||
"""把中文 label 解析为英码 value;空值返回 None,已为英码则原样返回。
|
||||
|
||||
Args:
|
||||
dict_type: 字典类型,如 "cross_method"
|
||||
label: Excel 单元格原始值(可能为中文 label,也可能已是英码)
|
||||
required: 非空白且无法识别时是否抛错(默认 True,便于发现脏数据)
|
||||
"""
|
||||
if label is None:
|
||||
return None
|
||||
text = str(label).strip()
|
||||
if text == "":
|
||||
return None
|
||||
await self._ensure_loaded()
|
||||
mapping = self._label2value.get(dict_type, {})
|
||||
if text in mapping:
|
||||
return mapping[text]
|
||||
if text in self._values.get(dict_type, set()):
|
||||
# 已是英码(如重新导入导出的文件),直接落库
|
||||
return text
|
||||
if required:
|
||||
raise CustomException(
|
||||
msg=f"字典类型「{dict_type}」中不存在标签或编码「{text}」,请核对导入模板"
|
||||
)
|
||||
return text
|
||||
@@ -0,0 +1,142 @@
|
||||
import io
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from openpyxl import Workbook, load_workbook
|
||||
from openpyxl.styles import Alignment, PatternFill
|
||||
from openpyxl.utils import get_column_letter
|
||||
from openpyxl.worksheet.datavalidation import DataValidation
|
||||
|
||||
|
||||
def _xlsx_safe_value(value: Any) -> Any:
|
||||
"""把 openpyxl 无法直接序列化的值归一为可写值。
|
||||
|
||||
- 带时区的 datetime(如审计列 created_time/updated_time,DB 为 timezone=True)
|
||||
归一为 UTC naive,否则 openpyxl 保存时抛 TypeError,导致导出整表失败。
|
||||
"""
|
||||
if isinstance(value, datetime) and value.tzinfo is not None:
|
||||
return value.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
return value
|
||||
|
||||
|
||||
class ExcelUtil:
|
||||
"""Excel 模板生成与列表导出(openpyxl)。"""
|
||||
|
||||
@staticmethod
|
||||
def read_excel_to_dicts(contents: bytes) -> list[dict[str, Any]]:
|
||||
"""读取 Excel 文件字节,返回字典列表(首行为列名)。"""
|
||||
wb = load_workbook(io.BytesIO(contents), read_only=True, data_only=True)
|
||||
ws = wb.active
|
||||
if not ws:
|
||||
raise ValueError("工作簿没有活动工作表")
|
||||
headers = [cell.value for cell in next(ws.iter_rows(min_row=1, max_row=1))]
|
||||
result: list[dict[str, Any]] = []
|
||||
for row in ws.iter_rows(min_row=2, values_only=True):
|
||||
if all(cell is None for cell in row):
|
||||
continue
|
||||
row_dict: dict[str, Any] = {}
|
||||
for i, val in enumerate(row):
|
||||
if i < len(headers) and headers[i] is not None:
|
||||
row_dict[str(headers[i])] = val
|
||||
if row_dict:
|
||||
result.append(row_dict)
|
||||
wb.close()
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def __mapping_list(cls, list_data: list[dict[str, Any]], mapping_dict: dict) -> list[dict[str, Any]]:
|
||||
"""将列表数据中的字段名映射为对应的中文字段名。
|
||||
|
||||
参数:
|
||||
- list_data: 数据列表。
|
||||
- mapping_dict: 字段名映射字典 {英文key: 中文表头}。
|
||||
|
||||
返回:
|
||||
- list[dict]: 映射后的数据列表 [{中文表头: value}]。
|
||||
"""
|
||||
return [{str(mapping_dict.get(key)): item.get(key) for key in mapping_dict} for item in list_data]
|
||||
|
||||
@classmethod
|
||||
def get_excel_template(
|
||||
cls,
|
||||
header_list: list[str],
|
||||
selector_header_list: list[str],
|
||||
option_list: list[dict[str, list[str]]],
|
||||
) -> bytes:
|
||||
"""生成 Excel 模板文件。
|
||||
|
||||
参数:
|
||||
- header_list: 表头列表。
|
||||
- selector_header_list: 需要设置下拉选择的表头列表。
|
||||
- option_list: 下拉选项配置列表。
|
||||
|
||||
返回:
|
||||
- bytes: Excel 文件的二进制数据。
|
||||
"""
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
if not ws:
|
||||
raise ValueError("不存在活动工作表")
|
||||
|
||||
header_fill = PatternFill(start_color="ababab", end_color="ababab", fill_type="solid")
|
||||
|
||||
for col_num, header in enumerate(header_list, 1):
|
||||
cell = ws.cell(row=1, column=col_num)
|
||||
cell.value = header # pyright: ignore[reportAttributeAccessIssue]
|
||||
cell.fill = header_fill
|
||||
cell.alignment = Alignment(horizontal="center")
|
||||
ws.column_dimensions[get_column_letter(col_num)].width = 12
|
||||
|
||||
for selector_header in selector_header_list:
|
||||
col_idx = header_list.index(selector_header) + 1
|
||||
header_options = next(
|
||||
(opt.get(selector_header) for opt in option_list if selector_header in opt),
|
||||
[],
|
||||
)
|
||||
if header_options:
|
||||
dv = DataValidation(type="list", formula1=f'"{",".join(header_options)}"')
|
||||
dv.add(f"{get_column_letter(col_idx)}2:{get_column_letter(col_idx)}1048576")
|
||||
ws.add_data_validation(dv)
|
||||
|
||||
buffer = io.BytesIO()
|
||||
wb.save(buffer)
|
||||
buffer.seek(0)
|
||||
return buffer.getvalue()
|
||||
|
||||
@classmethod
|
||||
def export_list2excel(cls, list_data: list[dict[str, Any]], mapping_dict: dict) -> bytes:
|
||||
"""将列表数据导出为 Excel 文件。
|
||||
|
||||
参数:
|
||||
- list_data: 要导出的数据列表。
|
||||
- mapping_dict: 字段名映射字典 {英文key: 中文表头}。
|
||||
|
||||
返回:
|
||||
- bytes: Excel 文件的二进制数据。
|
||||
|
||||
限制:
|
||||
- 最多导出 100000 条记录,超出部分截断。
|
||||
"""
|
||||
max_rows = 100000
|
||||
if len(list_data) > max_rows:
|
||||
list_data = list_data[:max_rows]
|
||||
|
||||
mapping_data = cls.__mapping_list(list_data, mapping_dict)
|
||||
headers = list(mapping_dict.values())
|
||||
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
if not ws:
|
||||
raise ValueError("不存在活动工作表")
|
||||
|
||||
for col_num, header in enumerate(headers, 1):
|
||||
ws.cell(row=1, column=col_num, value=header)
|
||||
|
||||
for row_num, row_data in enumerate(mapping_data, 2):
|
||||
for col_num, header in enumerate(headers, 1):
|
||||
ws.cell(row=row_num, column=col_num, value=_xlsx_safe_value(row_data.get(header)))
|
||||
|
||||
buffer = io.BytesIO()
|
||||
wb.save(buffer)
|
||||
buffer.seek(0)
|
||||
return buffer.getvalue()
|
||||
@@ -0,0 +1,208 @@
|
||||
import importlib
|
||||
import inspect
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
from app.config.path_conf import BASE_DIR
|
||||
from app.core.exceptions import CustomException
|
||||
|
||||
|
||||
class ImportUtil:
|
||||
"""扫描工程中的 ORM 模型文件并做有效性校验的辅助类。
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def find_project_root(cls) -> Path:
|
||||
"""返回项目根目录(与配置中的 `BASE_DIR` 一致)。
|
||||
|
||||
返回:
|
||||
- Path: 项目根路径。
|
||||
"""
|
||||
return BASE_DIR
|
||||
|
||||
@classmethod
|
||||
def is_valid_model(cls, obj: Any, base_class: type) -> bool:
|
||||
"""判断是否为可映射的 SQLAlchemy 模型类(含表名与非空列)。
|
||||
|
||||
参数:
|
||||
- obj (Any): 待验证对象(一般为类)。
|
||||
- base_class (type): ORM 声明基类。
|
||||
|
||||
返回:
|
||||
- bool: 是否为有效模型类。
|
||||
"""
|
||||
# 必须继承自base_class且不是base_class本身
|
||||
if not (inspect.isclass(obj) and issubclass(obj, base_class) and obj is not base_class):
|
||||
return False
|
||||
|
||||
# 必须有表名定义(排除抽象基类)
|
||||
if not hasattr(obj, "__tablename__") or getattr(obj, "__tablename__", None) is None:
|
||||
return False
|
||||
|
||||
# 必须有至少一个列定义
|
||||
try:
|
||||
inspected = sa_inspect(obj)
|
||||
return inspected is not None and len(inspected.columns) > 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
@lru_cache(maxsize=256)
|
||||
def find_models(cls, base_class: type) -> list[Any]:
|
||||
"""遍历工程内 `model.py` / `models.py`,收集去重后的有效模型类。
|
||||
|
||||
参数:
|
||||
- base_class (type): SQLAlchemy 声明基类。
|
||||
|
||||
返回:
|
||||
- list[Any]: 模型类列表。
|
||||
|
||||
异常:
|
||||
- ImportError: 模块导入失败(非「无法从某名导入」类警告)。
|
||||
- CustomException: 处理模块时发生未预期错误。
|
||||
"""
|
||||
models = []
|
||||
# 按类对象去重
|
||||
seen_models = set()
|
||||
# 按表名去重(防止同表名冲突)
|
||||
seen_tables = set()
|
||||
# 记录已经处理过的model.py文件路径
|
||||
processed_model_files = set()
|
||||
|
||||
project_root = cls.find_project_root()
|
||||
|
||||
# 排除目录扩展
|
||||
exclude_dirs = {
|
||||
"venv",
|
||||
".env",
|
||||
".git",
|
||||
"__pycache__",
|
||||
"migrations",
|
||||
"alembic",
|
||||
"tests",
|
||||
"test",
|
||||
"docs",
|
||||
"examples",
|
||||
"scripts",
|
||||
".venv",
|
||||
"static",
|
||||
"templates",
|
||||
"sql",
|
||||
"env",
|
||||
}
|
||||
|
||||
# 定义要搜索的模型目录模式
|
||||
model_dir_patterns = ["model.py", "models.py"]
|
||||
|
||||
# 使用一个更高效的方法来查找所有model.py文件
|
||||
model_files = []
|
||||
for root, dirs, files in os.walk(project_root):
|
||||
# 过滤排除目录
|
||||
dirs[:] = [d for d in dirs if d not in exclude_dirs]
|
||||
|
||||
for file in files:
|
||||
if file in model_dir_patterns:
|
||||
file_path = Path(root) / file
|
||||
# 构建相对于项目根的模块路径
|
||||
relative_path = file_path.relative_to(project_root)
|
||||
model_files.append((file_path, relative_path))
|
||||
|
||||
# 按模块路径排序,确保先导入基础模块
|
||||
model_files.sort(key=lambda x: str(x[1]))
|
||||
|
||||
for file_path, relative_path in model_files:
|
||||
# 确保文件路径没有被处理过
|
||||
if str(file_path) in processed_model_files:
|
||||
continue
|
||||
|
||||
processed_model_files.add(str(file_path))
|
||||
|
||||
# 构建模块名(将路径分隔符转换为点)
|
||||
module_parts = (*relative_path.parts[:-1], relative_path.stem)
|
||||
module_name = ".".join(module_parts)
|
||||
|
||||
try:
|
||||
# 导入模块
|
||||
module = importlib.import_module(module_name)
|
||||
|
||||
# 获取模块中的所有类
|
||||
for _name, obj in inspect.getmembers(module, inspect.isclass):
|
||||
# 验证模型有效性
|
||||
if not cls.is_valid_model(obj, base_class):
|
||||
continue
|
||||
|
||||
# 检查类对象重复
|
||||
if obj in seen_models:
|
||||
continue
|
||||
|
||||
# 检查表名重复
|
||||
table_name = getattr(obj, "__tablename__", None)
|
||||
if table_name is None:
|
||||
continue
|
||||
if table_name in seen_tables:
|
||||
continue
|
||||
|
||||
# 添加到已处理集合
|
||||
seen_models.add(obj)
|
||||
seen_tables.add(table_name)
|
||||
models.append(obj)
|
||||
except ImportError as e:
|
||||
if "cannot import name" not in str(e):
|
||||
raise ImportError(f"❗️ 警告: 无法导入模块 {module_name}: {e}")
|
||||
except Exception as e:
|
||||
raise CustomException(f"❌️ 处理模块 {module_name} 时出错: {e}")
|
||||
|
||||
# 查找apscheduler_jobs表的模型(如果存在)
|
||||
cls._find_apscheduler_model(base_class, models, seen_models, seen_tables)
|
||||
|
||||
return models
|
||||
|
||||
@classmethod
|
||||
def _find_apscheduler_model(
|
||||
cls,
|
||||
base_class: type,
|
||||
models: list[Any],
|
||||
seen_models: set[Any],
|
||||
seen_tables: set[str],
|
||||
) -> None:
|
||||
"""尝试从调度相关模块补充 `apscheduler_jobs` 表对应模型。
|
||||
|
||||
参数:
|
||||
- base_class (type): ORM 声明基类。
|
||||
- models (list[Any]): 已收集模型列表(就地追加)。
|
||||
- seen_models (set[Any]): 已见模型对象集合。
|
||||
- seen_tables (set[str]): 已见表名集合。
|
||||
|
||||
返回:
|
||||
- None
|
||||
|
||||
异常:
|
||||
- CustomException: 扫描过程出现未预期错误。
|
||||
"""
|
||||
# 尝试从apscheduler相关模块导入
|
||||
try:
|
||||
# 检查是否有自定义的apscheduler模型
|
||||
for module_name in [
|
||||
"app.core.ap_scheduler",
|
||||
"app.module_task.scheduler_test",
|
||||
]:
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
for _name, obj in inspect.getmembers(module, inspect.isclass):
|
||||
if (cls.is_valid_model(obj, base_class)
|
||||
and getattr(obj, "__tablename__", None) == "apscheduler_jobs"
|
||||
and obj not in seen_models
|
||||
and "apscheduler_jobs" not in seen_tables
|
||||
):
|
||||
seen_models.add(obj)
|
||||
seen_tables.add("apscheduler_jobs")
|
||||
models.append(obj)
|
||||
print(f"✅️ 找到有效模型: {obj.__module__}.{obj.__name__} (表: apscheduler_jobs)")
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception as e:
|
||||
raise CustomException(f"❗️ 查找APScheduler模型时出错: {e}")
|
||||
@@ -0,0 +1,165 @@
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
|
||||
import httpx
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.common.enums import RedisInitKeyConfig, SysParamKey
|
||||
from app.config.setting import settings
|
||||
from app.core.logger import logger
|
||||
from app.core.redis_crud import RedisCURD
|
||||
|
||||
# 归属地缓存:IP 几乎不变化,缓存 7 天可显著减少外网请求
|
||||
_IP_CACHE_TTL: int = settings.IP_LOCATION_CACHE_TTL
|
||||
# 硬超时(秒),避免外网查询阻塞主流程
|
||||
_IP_QUERY_TIMEOUT: float = settings.IP_LOCATION_QUERY_TIMEOUT
|
||||
|
||||
|
||||
def get_client_ip(request: Request) -> str:
|
||||
"""从请求中提取客户端真实 IP(返回空字串表示无法识别)。
|
||||
|
||||
- 仅当 settings.IP_TRUST_PROXY_HEADERS 为 true 时信任 X-Forwarded-For / X-Real-IP;
|
||||
此时要求后端只被可信代理(nginx)访问,否则攻击者可直接伪造该头。
|
||||
- 默认(不信任)返回直连 socket IP,杜绝伪造。
|
||||
"""
|
||||
if settings.IP_TRUST_PROXY_HEADERS:
|
||||
forwarded = request.headers.get("X-Forwarded-For", "")
|
||||
if forwarded:
|
||||
return forwarded.split(",")[0].strip()
|
||||
real_ip = request.headers.get("X-Real-IP", "")
|
||||
if real_ip:
|
||||
return real_ip.strip()
|
||||
if request.client:
|
||||
return request.client.host or ""
|
||||
return ""
|
||||
|
||||
|
||||
class IpLocalUtil:
|
||||
"""获取 IP 归属地工具类(带 Redis 缓存、硬超时、降级)。"""
|
||||
|
||||
@classmethod
|
||||
def is_valid_ip(cls, ip: str) -> bool:
|
||||
try:
|
||||
ipaddress.ip_address(ip)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def is_private_ip(cls, ip: str) -> bool:
|
||||
try:
|
||||
return ipaddress.ip_address(ip).is_private
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
async def _is_location_enabled(cls, redis) -> bool:
|
||||
"""从参数缓存读取 IP 归属地查询开关。"""
|
||||
if not redis:
|
||||
return False
|
||||
redis_key = f"{RedisInitKeyConfig.SYSTEM_CONFIG.key}:{SysParamKey.IP_LOCATION_ENABLE.value}"
|
||||
try:
|
||||
raw = await RedisCURD(redis).get(redis_key)
|
||||
if raw:
|
||||
payload = json.loads(raw)
|
||||
cv = payload.get("config_value", "off")
|
||||
return cv in (True, "true", "1", "yes", "on")
|
||||
except (json.JSONDecodeError, TypeError, Exception):
|
||||
pass
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
async def resolve_location_for_log(cls, redis, ip: str | None) -> str | None:
|
||||
"""登录日志写入入口:仅返回可同步获取的值(内网/缓存/降级),
|
||||
|
||||
外网查询由后台任务异步执行(见 ``resolve_location_async``)。
|
||||
"""
|
||||
if not ip:
|
||||
return None
|
||||
if not await cls._is_location_enabled(redis):
|
||||
return "内网IP" if cls.is_private_ip(ip) else "未解析(已关闭归属地查询)"
|
||||
if cls.is_private_ip(ip):
|
||||
return "内网IP"
|
||||
if redis:
|
||||
cached = await cls._cache_get(redis, ip)
|
||||
if cached is not None:
|
||||
return cached
|
||||
return "归属地查询中"
|
||||
|
||||
@classmethod
|
||||
async def resolve_location_async(cls, redis, ip: str) -> str:
|
||||
"""异步查询归属地(含缓存、降级、硬超时)。"""
|
||||
if not cls.is_valid_ip(ip):
|
||||
return "未知"
|
||||
if not await cls._is_location_enabled(redis):
|
||||
return "未解析(已关闭归属地查询)"
|
||||
if cls.is_private_ip(ip):
|
||||
return "内网IP"
|
||||
|
||||
cached = await cls._cache_get(redis, ip) if redis else None
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
result = await cls._query_with_timeout(ip)
|
||||
if redis:
|
||||
await cls._cache_set(redis, ip, result)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
async def _query_with_timeout(cls, ip: str) -> str:
|
||||
"""在硬超时内依次尝试多个 API,全部失败返回未知。"""
|
||||
apis: list[tuple[str, Callable, dict[str, str]]] = [
|
||||
("http://ip-api.com/json", cls._parse_ipapi, {"lang": "zh-CN"}),
|
||||
("https://whois.pconline.com.cn/ipJson.jsp", cls._parse_pconline, {"ip": ip, "json": "true"}),
|
||||
]
|
||||
async with httpx.AsyncClient(timeout=_IP_QUERY_TIMEOUT) as client:
|
||||
for url, parser, params in apis:
|
||||
try:
|
||||
resp = await client.get(f"{url}/{ip}" if "ip-api" in url else url, params=params)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json() if "ip-api" in url else resp.text
|
||||
location = parser(data)
|
||||
if location:
|
||||
return location
|
||||
except Exception as e:
|
||||
logger.warning(f"IP 归属地 API 失败: {url} - {e}")
|
||||
return "未知"
|
||||
|
||||
@staticmethod
|
||||
def _parse_ipapi(data: dict) -> str | None:
|
||||
if data.get("status") != "success":
|
||||
return None
|
||||
parts = [data.get("country"), data.get("regionName"), data.get("city"), data.get("isp")]
|
||||
joined = "-".join(filter(None, parts))
|
||||
return joined or None
|
||||
|
||||
@staticmethod
|
||||
def _parse_pconline(text: str) -> str | None:
|
||||
"""解析 pconline 返回的 JSONP 文本,格式如 'if( {\"ip\":\"...\",\"pro\":\"省\",\"city\":\"市\"} )'。"""
|
||||
try:
|
||||
match = re.search(r"\{.*\}", text)
|
||||
if not match:
|
||||
return None
|
||||
data = json.loads(match.group())
|
||||
parts = [data.get("pro"), data.get("city"), data.get("addr")]
|
||||
joined = " ".join(filter(None, parts))
|
||||
return joined or None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def _cache_get(redis, ip: str) -> str | None:
|
||||
try:
|
||||
value = await RedisCURD(redis).get(f"ip:location:{ip}")
|
||||
return value.decode("utf-8") if value else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def _cache_set(redis, ip: str, value: str) -> None:
|
||||
try:
|
||||
await RedisCURD(redis).set(f"ip:location:{ip}", value, expire=_IP_CACHE_TTL)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,61 @@
|
||||
"""统一编号服务 —— 原子取号,替代各处分散扫描式生成(§3.3 编号约定)。
|
||||
|
||||
取号基于 bre_sequence(name/current_seq)表 + PostgreSQL 事务级 advisory lock:
|
||||
pg_advisory_xact_lock 在事务内串行化同名取号,锁随事务提交/回滚自动释放,
|
||||
与 CRUD 同事务提交 ⇒ 并发请求不重复、失败自动回滚不留空号。
|
||||
|
||||
存量数据迁移:首次使用某序列名时由 seed_fn(服务端传入)从既有记录推算
|
||||
起始号(原扫描逻辑),此后全部走原子递增,不再扫描。
|
||||
"""
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.v1.module_bre.sequence.model import SequenceModel
|
||||
|
||||
|
||||
class NumberGenService:
|
||||
"""原子序号发生器。实例化于 HTTP/服务层,随当前事务取号。"""
|
||||
|
||||
def __init__(self, db: AsyncSession) -> None:
|
||||
self.db = db
|
||||
|
||||
@staticmethod
|
||||
async def _acquire_lock(db: AsyncSession, name: str) -> None:
|
||||
from app.config.setting import settings
|
||||
|
||||
if settings.DATABASE_TYPE != "postgres":
|
||||
return # sqlite 测试库无 advisory lock,退化为普通自增(单进程无并发竞争)
|
||||
# hashtextextended 对文本做稳定 64 位哈希,同名序列恒映射同一把锁
|
||||
await db.execute(text("SELECT pg_advisory_xact_lock(hashtextextended(:n, 0))"), {"n": name})
|
||||
|
||||
async def next_seq(
|
||||
self,
|
||||
name: str,
|
||||
*,
|
||||
start: int = 1,
|
||||
seed_fn: Callable[[], Awaitable[int | None]] | None = None,
|
||||
) -> int:
|
||||
"""原子取号。返回本序列名下一个应使用的序号。
|
||||
|
||||
name: 序列名(如 "combination:2026" / "tree:3"),按编号维度天然分片;
|
||||
start: 新序列的默认起始值;
|
||||
seed_fn: 仅当序列行尚不存在时调用,从存量数据推算起始号(兼容既有数据迁移)。
|
||||
"""
|
||||
await self._acquire_lock(self.db, name)
|
||||
row = await self.db.scalar(select(SequenceModel).where(SequenceModel.name == name))
|
||||
if row is None:
|
||||
init = await seed_fn() if seed_fn is not None else start
|
||||
if init is None or init < 1:
|
||||
init = start
|
||||
self.db.add(SequenceModel(name=name, current_seq=init))
|
||||
await self.db.flush()
|
||||
return init
|
||||
row.current_seq += 1
|
||||
await self.db.flush()
|
||||
return int(row.current_seq)
|
||||
|
||||
async def exists(self, name: str) -> bool:
|
||||
"""序列行是否已初始化(供服务端判断是否需要 seed_fn 兜底)。"""
|
||||
return (await self.db.scalar(select(SequenceModel.id).where(SequenceModel.name == name))) is not None
|
||||
@@ -0,0 +1,71 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
import secrets
|
||||
import string
|
||||
|
||||
_PBKDF2_ALGO = "sha256"
|
||||
_PBKDF2_ITERATIONS = 600_000
|
||||
_PBKDF2_SALT_LEN = 16
|
||||
_PBKDF2_PREFIX = "$pbkdf2-sha256$"
|
||||
|
||||
_STRONG_PWD_CHARS = string.ascii_letters + string.digits + "!@#$%^&*"
|
||||
|
||||
|
||||
class PwdUtil:
|
||||
@staticmethod
|
||||
def hash_password(password: str) -> str:
|
||||
salt = os.urandom(_PBKDF2_SALT_LEN)
|
||||
dk = hashlib.pbkdf2_hmac(_PBKDF2_ALGO, password.encode(), salt, _PBKDF2_ITERATIONS)
|
||||
return f"{_PBKDF2_PREFIX}{_PBKDF2_ITERATIONS}${base64.b64encode(salt).decode()}${base64.b64encode(dk).decode()}"
|
||||
|
||||
@staticmethod
|
||||
def verify_password(plain_password: str, password_hash: str) -> bool:
|
||||
try:
|
||||
_, _algo, iters_str, salt_b64, hash_b64 = password_hash.split("$")
|
||||
salt = base64.b64decode(salt_b64)
|
||||
expected = base64.b64decode(hash_b64)
|
||||
dk = hashlib.pbkdf2_hmac(_PBKDF2_ALGO, plain_password.encode(), salt, int(iters_str))
|
||||
return dk == expected
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def check_password_strength(password: str) -> str | None:
|
||||
if len(password) < 6:
|
||||
return "密码长度至少6位"
|
||||
if not any(c.isupper() for c in password):
|
||||
return "密码需要包含大写字母"
|
||||
if not any(c.islower() for c in password):
|
||||
return "密码需要包含小写字母"
|
||||
if not any(c.isdigit() for c in password):
|
||||
return "密码需要包含数字"
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def generate_strong_password(length: int = 12) -> str:
|
||||
"""生成符合强度要求的强随机密码(大写+小写+数字+特殊符号)。
|
||||
|
||||
使用 ``secrets`` 而非 ``random``,避免伪随机带来的安全风险。
|
||||
|
||||
参数:
|
||||
- length (int): 密码长度,默认 12,最小 8。
|
||||
|
||||
返回:
|
||||
- str: 生成的明文密码。
|
||||
"""
|
||||
if length < 8:
|
||||
raise ValueError("密码长度至少 8 位")
|
||||
|
||||
# 保证每类字符至少出现一次
|
||||
uppercase = secrets.choice(string.ascii_uppercase)
|
||||
lowercase = secrets.choice(string.ascii_lowercase)
|
||||
digit = secrets.choice(string.digits)
|
||||
special = secrets.choice("!@#$%^&*")
|
||||
|
||||
remaining_length = length - 4
|
||||
rest = [secrets.choice(_STRONG_PWD_CHARS) for _ in range(remaining_length)]
|
||||
|
||||
chars = list(rest) + [uppercase, lowercase, digit, special]
|
||||
secrets.SystemRandom().shuffle(chars)
|
||||
return "".join(chars)
|
||||
@@ -0,0 +1,201 @@
|
||||
from app.common.constant import CommonConstant
|
||||
|
||||
|
||||
class StringUtil:
|
||||
"""字符串工具类
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def is_blank(cls, string: str) -> bool:
|
||||
"""校验字符串是否为''或全空格
|
||||
|
||||
参数:
|
||||
- string (str): 需要校验的字符串。
|
||||
|
||||
返回:
|
||||
- bool: 校验结果。
|
||||
"""
|
||||
str_len = len(string)
|
||||
if str_len == 0:
|
||||
return True
|
||||
return all(string[i] == " " for i in range(str_len))
|
||||
|
||||
@classmethod
|
||||
def is_empty(cls, string: str | None) -> bool:
|
||||
"""校验字符串是否为''或None
|
||||
|
||||
参数:
|
||||
- string (str | None): 需要校验的字符串。
|
||||
|
||||
返回:
|
||||
- bool: 校验结果。
|
||||
"""
|
||||
return string is None or len(string) == 0
|
||||
|
||||
@classmethod
|
||||
def is_not_empty(cls, string: str) -> bool:
|
||||
"""校验字符串是否不是''和None
|
||||
|
||||
参数:
|
||||
- string (str): 需要校验的字符串。
|
||||
|
||||
返回:
|
||||
- bool: 校验结果。
|
||||
"""
|
||||
return not cls.is_empty(string)
|
||||
|
||||
@classmethod
|
||||
def is_http(cls, link: str):
|
||||
"""判断是否为 http(s):// 开头
|
||||
|
||||
参数:
|
||||
- link (str): 链接。
|
||||
|
||||
返回:
|
||||
- bool: 是否为 http(s):// 开头。
|
||||
"""
|
||||
return link.startswith((CommonConstant.HTTP, CommonConstant.HTTPS))
|
||||
|
||||
@classmethod
|
||||
def contains_ignore_case(cls, search_str: str, compare_str: str):
|
||||
"""查找指定字符串是否包含指定字符串同时忽略大小写
|
||||
|
||||
参数:
|
||||
- search_str (str): 查找的字符串。
|
||||
- compare_str (str): 比对的字符串。
|
||||
|
||||
返回:
|
||||
- bool: 查找结果。
|
||||
"""
|
||||
if compare_str and search_str:
|
||||
return compare_str.lower() in search_str.lower()
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def contains_any_ignore_case(cls, search_str: str, compare_str_list: list[str]):
|
||||
"""查找指定字符串是否包含列表中的任意一个字符串(忽略大小写)
|
||||
|
||||
参数:
|
||||
- search_str (str): 查找的字符串。
|
||||
- compare_str_list (list[str]): 比对的字符串列表。
|
||||
|
||||
返回:
|
||||
- bool: 查找结果。
|
||||
"""
|
||||
if search_str and compare_str_list:
|
||||
return any(cls.contains_ignore_case(search_str, compare_str) for compare_str in compare_str_list)
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def equals_ignore_case(cls, search_str: str, compare_str: str):
|
||||
"""比较两个字符串是否相等(忽略大小写)
|
||||
|
||||
参数:
|
||||
- search_str (str): 查找的字符串。
|
||||
- compare_str (str): 比对的字符串。
|
||||
|
||||
返回:
|
||||
- bool: 比较结果。
|
||||
"""
|
||||
if search_str and compare_str:
|
||||
return search_str.lower() == compare_str.lower()
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def equals_any_ignore_case(cls, search_str: str, compare_str_list: list[str]):
|
||||
"""判断指定字符串是否与列表中任意一个字符串相等(忽略大小写)
|
||||
|
||||
参数:
|
||||
- search_str (str): 查找的字符串。
|
||||
- compare_str_list (list[str]): 比对的字符串列表。
|
||||
|
||||
返回:
|
||||
- bool: 比较结果。
|
||||
"""
|
||||
if search_str and compare_str_list:
|
||||
return any(cls.equals_ignore_case(search_str, compare_str) for compare_str in compare_str_list)
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def startswith_case(cls, search_str: str, compare_str: str):
|
||||
"""查找指定字符串是否以指定字符串开头
|
||||
|
||||
参数:
|
||||
- search_str (str): 查找的字符串。
|
||||
- compare_str (str): 比对的字符串。
|
||||
|
||||
返回:
|
||||
- bool: 查找结果。
|
||||
"""
|
||||
if compare_str and search_str:
|
||||
return search_str.startswith(compare_str)
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def startswith_any_case(cls, search_str: str, compare_str_list: list[str]):
|
||||
"""查找指定字符串是否以列表中任意一个字符串开头
|
||||
|
||||
参数:
|
||||
- search_str (str): 查找的字符串。
|
||||
- compare_str_list (list[str]): 比对的字符串列表。
|
||||
|
||||
返回:
|
||||
- bool: 查找结果。
|
||||
"""
|
||||
if search_str and compare_str_list:
|
||||
return any(cls.startswith_case(search_str, compare_str) for compare_str in compare_str_list)
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def convert_to_camel_case(cls, name: str) -> str:
|
||||
"""将下划线命名的字符串转换为大驼峰(PascalCase);若输入为空则返回空字符串。
|
||||
|
||||
参数:
|
||||
- name (str): 下划线命名的字符串。
|
||||
|
||||
返回:
|
||||
- str: 转换后的大驼峰字符串。
|
||||
"""
|
||||
if not name:
|
||||
return ""
|
||||
if "_" not in name:
|
||||
return name[0].upper() + name[1:]
|
||||
parts = name.split("_")
|
||||
result = []
|
||||
for part in parts:
|
||||
if not part:
|
||||
continue
|
||||
result.append(part[0].upper() + part[1:].lower())
|
||||
return "".join(result)
|
||||
|
||||
@classmethod
|
||||
def to_lower_camel_case(cls, text: str) -> str:
|
||||
"""将下划线命名的字符串转换为小驼峰(lower camelCase);若输入为空则返回空字符串。
|
||||
|
||||
参数:
|
||||
- text (str): 下划线命名的字符串。
|
||||
|
||||
返回:
|
||||
- str: 转换后的小驼峰字符串。
|
||||
"""
|
||||
if not text:
|
||||
return ""
|
||||
parts = text.split("_")
|
||||
return parts[0] + "".join(word.capitalize() for word in parts[1:])
|
||||
|
||||
@classmethod
|
||||
def get_mapping_value_by_key_ignore_case(cls, mapping: dict[str, str], key: str) -> str:
|
||||
"""根据忽略大小写的键获取字典中的对应的值
|
||||
|
||||
参数:
|
||||
- mapping (dict[str, str]): 字典。
|
||||
- key (str): 字典的键。
|
||||
|
||||
返回:
|
||||
- str: 字典键对应的值,未匹配则返回空字符串。
|
||||
"""
|
||||
for k, v in mapping.items():
|
||||
if key.lower() == k.lower():
|
||||
return v
|
||||
|
||||
return ""
|
||||
@@ -0,0 +1,511 @@
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import aiofiles
|
||||
from fastapi import UploadFile
|
||||
|
||||
from app.config.setting import settings
|
||||
from app.core.exceptions import CustomException
|
||||
from app.core.logger import logger
|
||||
|
||||
DANGEROUS_EXTENSIONS = {
|
||||
".py",
|
||||
".pyc",
|
||||
".pyo",
|
||||
".php",
|
||||
".php3",
|
||||
".php4",
|
||||
".php5",
|
||||
".phtml",
|
||||
".exe",
|
||||
".bat",
|
||||
".cmd",
|
||||
".sh",
|
||||
".bash",
|
||||
".zsh",
|
||||
".ps1",
|
||||
".ps2",
|
||||
".psm1",
|
||||
".psd1",
|
||||
".vbs",
|
||||
".vbe",
|
||||
".js",
|
||||
".jse",
|
||||
".wsf",
|
||||
".wsh",
|
||||
".msi",
|
||||
".dll",
|
||||
".so",
|
||||
".dylib",
|
||||
".jar",
|
||||
".class",
|
||||
".jsp",
|
||||
".jspx",
|
||||
".asp",
|
||||
".aspx",
|
||||
".asa",
|
||||
".asax",
|
||||
".cer",
|
||||
".cdx",
|
||||
".config",
|
||||
".htaccess",
|
||||
".htpasswd",
|
||||
".sql",
|
||||
".db",
|
||||
".sqlite",
|
||||
".sqlite3",
|
||||
}
|
||||
|
||||
MIME_TYPE_MAPPING = {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/gif": ".gif",
|
||||
"image/webp": ".webp",
|
||||
"image/svg+xml": ".svg",
|
||||
"image/x-icon": ".ico",
|
||||
"image/bmp": ".bmp",
|
||||
"application/vnd.ms-excel": ".xls",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
|
||||
"application/msword": ".doc",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
|
||||
"application/pdf": ".pdf",
|
||||
"text/plain": ".txt",
|
||||
"text/csv": ".csv",
|
||||
}
|
||||
|
||||
|
||||
class UploadUtil:
|
||||
"""上传工具类
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def generate_random_number() -> str:
|
||||
"""生成3位随机数字字符串。
|
||||
|
||||
返回:
|
||||
- str: 三位随机数字字符串。
|
||||
"""
|
||||
return f"{random.randint(1, 999):03}"
|
||||
|
||||
@staticmethod
|
||||
def check_file_exists(filepath: str) -> bool:
|
||||
"""检查文件是否存在。
|
||||
|
||||
参数:
|
||||
- filepath (str): 文件路径。
|
||||
|
||||
返回:
|
||||
- bool: 文件是否存在。
|
||||
"""
|
||||
return Path(filepath).exists()
|
||||
|
||||
@staticmethod
|
||||
def sanitize_filename(filename: str) -> str:
|
||||
"""清理文件名,移除危险字符和路径穿越。
|
||||
|
||||
参数:
|
||||
- filename (str): 原始文件名。
|
||||
|
||||
返回:
|
||||
- str: 安全的文件名。
|
||||
"""
|
||||
if not filename:
|
||||
return ""
|
||||
filename = os.path.basename(filename)
|
||||
filename = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "", filename)
|
||||
filename = re.sub(r"\.{2,}", ".", filename)
|
||||
filename = filename.strip(". ")
|
||||
if not filename:
|
||||
filename = f"file_{datetime.now().strftime('%Y%m%d%H%M%S')}"
|
||||
return filename
|
||||
|
||||
@staticmethod
|
||||
def check_path_traversal(filename: str) -> bool:
|
||||
"""检查文件名是否包含路径穿越。
|
||||
|
||||
参数:
|
||||
- filename (str): 文件名。
|
||||
|
||||
返回:
|
||||
- bool: 是否安全(True 表示安全,False 表示存在路径穿越)。
|
||||
"""
|
||||
dangerous_patterns = ["../", "..\\", "/", "\\", "\0"]
|
||||
for pattern in dangerous_patterns:
|
||||
if pattern in filename:
|
||||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_extension_from_filename(filename: str) -> str:
|
||||
"""从文件名获取扩展名。
|
||||
|
||||
参数:
|
||||
- filename (str): 文件名。
|
||||
|
||||
返回:
|
||||
- str: 扩展名(小写,包含点),如 ".jpg"。
|
||||
"""
|
||||
if not filename or "." not in filename:
|
||||
return ""
|
||||
ext = filename.rsplit(".", 1)[-1].lower()
|
||||
return f".{ext}" if ext else ""
|
||||
|
||||
@staticmethod
|
||||
def is_dangerous_extension(extension: str) -> bool:
|
||||
"""检查扩展名是否为危险类型。
|
||||
|
||||
参数:
|
||||
- extension (str): 文件扩展名。
|
||||
|
||||
返回:
|
||||
- bool: 是否为危险扩展名。
|
||||
"""
|
||||
return extension.lower() in DANGEROUS_EXTENSIONS
|
||||
|
||||
@staticmethod
|
||||
def detect_file_type(content: bytes) -> str | None:
|
||||
"""通过文件内容检测真实文件类型。
|
||||
|
||||
参数:
|
||||
- content (bytes): 文件内容(前几字节即可)。
|
||||
|
||||
返回:
|
||||
- str | None: 检测到的 MIME 类型,无法识别返回 None。
|
||||
"""
|
||||
if content.startswith(b"\xff\xd8\xff"):
|
||||
return "image/jpeg"
|
||||
if content.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return "image/png"
|
||||
if content.startswith(b"GIF87a") or content.startswith(b"GIF89a"):
|
||||
return "image/gif"
|
||||
if content.startswith(b"PK\x03\x04"):
|
||||
if b"[Content_Types].xml" in content[:1000]:
|
||||
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
return "application/zip"
|
||||
if content.startswith(b"%PDF"):
|
||||
return "application/pdf"
|
||||
if content.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):
|
||||
return "application/msword"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def validate_file_extension(cls, extension: str) -> bool:
|
||||
"""验证文件扩展名是否在允许列表中。
|
||||
|
||||
参数:
|
||||
- extension (str): 文件扩展名。
|
||||
|
||||
返回:
|
||||
- bool: 是否允许。
|
||||
|
||||
异常:
|
||||
- CustomException: 扩展名不允许时抛出。
|
||||
"""
|
||||
ext_lower = extension.lower()
|
||||
if cls.is_dangerous_extension(ext_lower):
|
||||
raise CustomException(msg=f"不允许上传此类型的文件: {extension}")
|
||||
if ext_lower not in settings.ALLOWED_EXTENSIONS:
|
||||
raise CustomException(msg=f"文件类型不支持,允许的类型: {', '.join(settings.ALLOWED_EXTENSIONS)}")
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def validate_file_content_type(cls, content: bytes, claimed_extension: str) -> bool:
|
||||
"""验证文件内容类型与声明的扩展名是否匹配。
|
||||
|
||||
参数:
|
||||
- content (bytes): 文件内容。
|
||||
- claimed_extension (str): 声明的文件扩展名。
|
||||
|
||||
返回:
|
||||
- bool: 是否匹配。
|
||||
|
||||
异常:
|
||||
- CustomException: 类型不匹配时抛出。
|
||||
"""
|
||||
detected_type = cls.detect_file_type(content)
|
||||
if detected_type:
|
||||
expected_ext = MIME_TYPE_MAPPING.get(detected_type, "")
|
||||
if expected_ext and expected_ext != claimed_extension.lower():
|
||||
logger.warning(f"文件类型不匹配: 声明扩展名={claimed_extension}, 检测类型={detected_type}")
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def check_file_size(file: UploadFile) -> bool:
|
||||
"""校验文件大小是否合法。
|
||||
|
||||
参数:
|
||||
- file (UploadFile): 上传的文件对象。
|
||||
|
||||
返回:
|
||||
- bool: 文件大小是否合法。
|
||||
|
||||
异常:
|
||||
- CustomException: 文件过大时抛出。
|
||||
"""
|
||||
if file.size and file.size > settings.MAX_FILE_SIZE:
|
||||
raise CustomException(msg=f"文件大小超过限制,最大允许 {settings.MAX_FILE_SIZE // (1024 * 1024)}MB")
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def generate_safe_filename(cls, original_filename: str, extension: str) -> str:
|
||||
"""生成安全的文件名。
|
||||
|
||||
参数:
|
||||
- original_filename (str): 原始文件名。
|
||||
- extension (str): 文件扩展名。
|
||||
|
||||
返回:
|
||||
- str: 安全的文件名。
|
||||
"""
|
||||
safe_name = cls.sanitize_filename(original_filename)
|
||||
if safe_name and "." in safe_name:
|
||||
name_part = safe_name.rsplit(".", 1)[0]
|
||||
else:
|
||||
name_part = safe_name or "file"
|
||||
name_part = re.sub(r"[^a-zA-Z0-9_\-\u4e00-\u9fa5]", "", name_part)
|
||||
if len(name_part) > 50:
|
||||
name_part = name_part[:50]
|
||||
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
random_suffix = cls.generate_random_number()
|
||||
return f"{name_part}_{timestamp}{settings.UPLOAD_MACHINE}{random_suffix}{extension}"
|
||||
|
||||
@staticmethod
|
||||
def check_file_timestamp(filename: str) -> bool:
|
||||
"""校验文件时间戳是否合法。
|
||||
|
||||
参数:
|
||||
- filename (str): 文件名(包含时间戳片段)。
|
||||
|
||||
返回:
|
||||
- bool: 时间戳是否合法。
|
||||
"""
|
||||
try:
|
||||
name_parts = filename.rsplit(".", 1)[0].split("_")
|
||||
timestamp = name_parts[-1].split(settings.UPLOAD_MACHINE)[0]
|
||||
datetime.strptime(timestamp, "%Y%m%d%H%M%S")
|
||||
return True
|
||||
except (ValueError, IndexError):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def check_file_machine(filename: str) -> bool:
|
||||
"""校验文件机器码是否合法。
|
||||
|
||||
参数:
|
||||
- filename (str): 文件名。
|
||||
|
||||
返回:
|
||||
- bool: 机器码是否合法。
|
||||
"""
|
||||
try:
|
||||
name_without_ext = filename.rsplit(".", 1)[0]
|
||||
return len(name_without_ext) >= 4 and name_without_ext[-4] == settings.UPLOAD_MACHINE
|
||||
except IndexError:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def check_file_random_code(filename: str) -> bool:
|
||||
"""校验文件随机码是否合法。
|
||||
|
||||
参数:
|
||||
- filename (str): 文件名。
|
||||
|
||||
返回:
|
||||
- bool: 随机码是否合法(000–999)。
|
||||
"""
|
||||
try:
|
||||
code = filename.rsplit(".", 1)[0][-3:]
|
||||
return code.isdigit() and 1 <= int(code) <= 999
|
||||
except IndexError:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def generate_file(filepath: Path, chunk_size: int = 8192):
|
||||
"""根据文件生成二进制数据迭代器。
|
||||
|
||||
参数:
|
||||
- filepath (Path): 文件路径。
|
||||
- chunk_size (int): 分块大小,默认 8192 字节。
|
||||
|
||||
返回:
|
||||
- Iterator[bytes]: 文件二进制数据分块迭代器。
|
||||
"""
|
||||
with filepath.open("rb") as f:
|
||||
while chunk := f.read(chunk_size):
|
||||
yield chunk
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_target_path(target_path: str) -> str:
|
||||
"""清理目标路径,移除危险字符和路径穿越。
|
||||
|
||||
参数:
|
||||
- target_path (str): 原始目标路径。
|
||||
|
||||
返回:
|
||||
- str: 安全的相对路径。
|
||||
|
||||
异常:
|
||||
- CustomException: 当路径包含非法字符时抛出。
|
||||
"""
|
||||
if not target_path:
|
||||
return ""
|
||||
|
||||
# 检查路径穿越
|
||||
if ".." in target_path or "\x00" in target_path:
|
||||
logger.error(f"检测到目标路径穿越攻击: {target_path}")
|
||||
raise CustomException(msg="非法的目标路径")
|
||||
|
||||
# 规范化路径:移除多余的斜杠、点号
|
||||
parts = target_path.replace("\\", "/").split("/")
|
||||
safe_parts = []
|
||||
|
||||
for part in parts:
|
||||
# 跳过空字符串和单独的点号
|
||||
if not part or part == ".":
|
||||
continue
|
||||
# 移除危险字符
|
||||
part = re.sub(r'[<>:"|?*\x00-\x1f]', "", part)
|
||||
if part and part != "..":
|
||||
safe_parts.append(part)
|
||||
|
||||
return "/".join(safe_parts)
|
||||
|
||||
@staticmethod
|
||||
def delete_file(filepath: Path) -> bool:
|
||||
"""删除文件。
|
||||
|
||||
参数:
|
||||
- filepath (Path): 文件路径。
|
||||
|
||||
返回:
|
||||
- bool: 删除是否成功。
|
||||
"""
|
||||
try:
|
||||
filepath.unlink(missing_ok=True)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
async def upload_file(
|
||||
cls,
|
||||
file: UploadFile,
|
||||
base_url: str,
|
||||
upload_type: str = "file",
|
||||
target_path: str | None = None,
|
||||
) -> tuple[str, Path, str]:
|
||||
"""安全文件上传。
|
||||
|
||||
参数:
|
||||
- file (UploadFile): 上传的文件对象。
|
||||
- base_url (str): 基础 URL。
|
||||
- upload_type (str): 上传类型,可选值:
|
||||
- "file": 通用文件 (默认)
|
||||
- "avatar": 头像图片
|
||||
- "param": 参数配置
|
||||
- "resource": 监控资源
|
||||
- target_path (str | None): 目标目录路径(相对路径),仅 resource 类型支持。
|
||||
例如: "images", "documents/2024"
|
||||
|
||||
返回:
|
||||
- tuple[str, Path, str]: (文件名, 文件路径, 文件 URL)。
|
||||
|
||||
异常:
|
||||
- CustomException: 当文件校验失败时抛出。
|
||||
"""
|
||||
if not file or not file.filename:
|
||||
raise CustomException(msg="请选择要上传的文件")
|
||||
|
||||
original_filename = file.filename
|
||||
|
||||
if not cls.check_path_traversal(original_filename):
|
||||
raise CustomException(msg="文件名包含非法字符", data=original_filename)
|
||||
|
||||
extension = cls.get_extension_from_filename(original_filename)
|
||||
if not extension:
|
||||
raise CustomException(msg="无法识别文件类型")
|
||||
|
||||
cls.validate_file_extension(extension)
|
||||
|
||||
cls.check_file_size(file)
|
||||
|
||||
content = await file.read()
|
||||
await file.seek(0)
|
||||
|
||||
cls.validate_file_content_type(content, extension)
|
||||
|
||||
safe_filename = cls.generate_safe_filename(original_filename, extension)
|
||||
|
||||
try:
|
||||
# 根据上传类型选择保存目录
|
||||
type_subdir = {
|
||||
"avatar": "avatar",
|
||||
"param": "param",
|
||||
"resource": "resource",
|
||||
}.get(upload_type, "file")
|
||||
|
||||
# 构建目录路径
|
||||
if target_path and upload_type == "resource":
|
||||
# target_path 是相对于 upload/resource 的子目录
|
||||
# 清理路径,防止路径穿越
|
||||
# 资源管理模块:文件直接保存在目标目录,不自动创建日期子目录
|
||||
safe_target = cls._sanitize_target_path(target_path)
|
||||
dir_path = settings.UPLOAD_FILE_PATH.joinpath(type_subdir, safe_target)
|
||||
elif upload_type == "resource":
|
||||
# 资源管理模块根目录上传:直接保存在 upload/resource/ 下
|
||||
dir_path = settings.UPLOAD_FILE_PATH.joinpath(type_subdir)
|
||||
else:
|
||||
# 其他类型:按日期子目录组织
|
||||
dir_path = settings.UPLOAD_FILE_PATH.joinpath(type_subdir, datetime.now().strftime("%Y/%m/%d"))
|
||||
|
||||
dir_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
filepath = dir_path.joinpath(safe_filename)
|
||||
|
||||
if not filepath.resolve().is_relative_to(settings.UPLOAD_FILE_PATH.resolve()):
|
||||
logger.error(f"检测到路径穿越攻击,目标路径: {filepath}")
|
||||
raise CustomException(msg="非法的文件路径")
|
||||
|
||||
file_url = urljoin(base_url, str(filepath))
|
||||
|
||||
chunk_size = 8 * 1024 * 1024
|
||||
async with aiofiles.open(filepath, "wb") as f:
|
||||
while chunk := await file.read(chunk_size):
|
||||
await f.write(chunk)
|
||||
|
||||
return safe_filename, filepath, file_url
|
||||
|
||||
except CustomException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"文件上传失败: {e}")
|
||||
raise CustomException(msg=f"文件上传失败: {e}")
|
||||
|
||||
@staticmethod
|
||||
def get_file_tree(file_path: str) -> list[dict]:
|
||||
"""获取文件树结构。
|
||||
|
||||
参数:
|
||||
- file_path (str): 文件路径。
|
||||
|
||||
返回:
|
||||
- list[dict]: 文件树列表。
|
||||
"""
|
||||
return [{"name": item.name, "is_dir": item.is_dir()} for item in Path(file_path).iterdir()]
|
||||
|
||||
@classmethod
|
||||
async def download_file(cls, file_path: str) -> str:
|
||||
"""下载文件,生成新的文件名。
|
||||
|
||||
参数:
|
||||
- file_path (str): 文件路径。
|
||||
|
||||
返回:
|
||||
- str: 文件下载信息。
|
||||
"""
|
||||
filename = cls.generate_file(Path(file_path))
|
||||
return str(filename)
|
||||
@@ -0,0 +1,118 @@
|
||||
import bleach
|
||||
from bleach.css_sanitizer import CSSSanitizer
|
||||
|
||||
ALLOWED_TAGS = [
|
||||
"a",
|
||||
"abbr",
|
||||
"acronym",
|
||||
"b",
|
||||
"blockquote",
|
||||
"br",
|
||||
"code",
|
||||
"col",
|
||||
"colgroup",
|
||||
"dd",
|
||||
"del",
|
||||
"dl",
|
||||
"dt",
|
||||
"em",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"hr",
|
||||
"i",
|
||||
"img",
|
||||
"li",
|
||||
"ol",
|
||||
"p",
|
||||
"pre",
|
||||
"s",
|
||||
"span",
|
||||
"strike",
|
||||
"strong",
|
||||
"sub",
|
||||
"sup",
|
||||
"table",
|
||||
"tbody",
|
||||
"td",
|
||||
"tfoot",
|
||||
"th",
|
||||
"thead",
|
||||
"tr",
|
||||
"tt",
|
||||
"u",
|
||||
"ul",
|
||||
"video",
|
||||
"source",
|
||||
"div",
|
||||
"font",
|
||||
]
|
||||
|
||||
ALLOWED_ATTRIBUTES = {
|
||||
"*": ["class", "style", "id", "data-*"],
|
||||
"a": ["href", "title", "target", "rel"],
|
||||
"abbr": ["title"],
|
||||
"acronym": ["title"],
|
||||
"img": ["src", "alt", "title", "width", "height"],
|
||||
"video": ["src", "controls", "width", "height", "poster"],
|
||||
"source": ["src", "type"],
|
||||
"font": ["color", "size", "face"],
|
||||
"td": ["width", "height", "colspan", "rowspan"],
|
||||
"th": ["width", "height", "colspan", "rowspan"],
|
||||
"col": ["width", "span"],
|
||||
"colgroup": ["span"],
|
||||
}
|
||||
|
||||
ALLOWED_STYLES = [
|
||||
"color",
|
||||
"background-color",
|
||||
"font-size",
|
||||
"font-family",
|
||||
"font-weight",
|
||||
"font-style",
|
||||
"text-decoration",
|
||||
"text-align",
|
||||
"margin",
|
||||
"margin-left",
|
||||
"margin-right",
|
||||
"margin-top",
|
||||
"margin-bottom",
|
||||
"padding",
|
||||
"padding-left",
|
||||
"padding-right",
|
||||
"padding-top",
|
||||
"padding-bottom",
|
||||
"border",
|
||||
"border-color",
|
||||
"border-width",
|
||||
"border-style",
|
||||
"width",
|
||||
"height",
|
||||
"line-height",
|
||||
"display",
|
||||
]
|
||||
|
||||
|
||||
def sanitize_html(content: str) -> str:
|
||||
"""清理 HTML 内容,移除潜在的 XSS 攻击代码。
|
||||
|
||||
参数:
|
||||
- content (str): 需要清理的 HTML 内容
|
||||
|
||||
返回:
|
||||
- str: 清理后的安全 HTML 内容
|
||||
"""
|
||||
if not content:
|
||||
return content
|
||||
|
||||
return bleach.clean(
|
||||
content,
|
||||
tags=ALLOWED_TAGS,
|
||||
attributes=ALLOWED_ATTRIBUTES,
|
||||
css_sanitizer=CSSSanitizer(allowed_css_properties=ALLOWED_STYLES),
|
||||
strip=True,
|
||||
strip_comments=True,
|
||||
)
|
||||
Reference in New Issue
Block a user