752 lines
31 KiB
Python
752 lines
31 KiB
Python
"""Arena direct model catalog persistence, migration, and dark pool filtering."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
import json
|
||
import os
|
||
import re
|
||
import threading
|
||
from datetime import datetime, timezone, timedelta
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||
|
||
from app.core.config import logger
|
||
from app.utils.site_url import extract_remote_site_domain, route_domain_matches
|
||
|
||
|
||
ARENA_MODEL_CATALOG_CONFIG_PATH = Path(
|
||
os.getenv(
|
||
"ARENA_MODEL_CATALOG_CONFIG_PATH",
|
||
str(Path(__file__).resolve().parents[2] / "config" / "arena_model_catalog.local.json"),
|
||
)
|
||
)
|
||
|
||
ARENA_MODELS_CACHE_PATH = Path(
|
||
os.getenv(
|
||
"ARENA_MODELS_CACHE_PATH",
|
||
str(Path(__file__).resolve().parents[2] / "scripts" / "arena_models_cache.json"),
|
||
)
|
||
)
|
||
|
||
MODEL_CATALOG_SOURCE = "arena_direct"
|
||
|
||
ONLINE_PUBLIC_VISIBLE_MODELS: Set[str] = {
|
||
"gemini-3-flash", "glm-5.1", "qwen3.5-397b-a17b", "claude-sonnet-4-5-20250929",
|
||
"gemini-3.1-pro-preview", "qwen3.7-plus", "minimax-m3", "claude-haiku-4-5-20251001",
|
||
"gemini-2.5-pro", "glm-5v-turbo", "grok-4.20-beta-0309-reasoning", "gpt-5.2-high",
|
||
"gpt-5.5-instant", "gpt-5.1", "gpt-5.2", "gemini-3.6-flash", "claude-sonnet-4-6",
|
||
"grok-4.20-multi-agent-beta-0309", "qwen3.5-max-preview", "gemini-3.5-flash-lite",
|
||
"glm-5", "claude-sonnet-4-5-20250929-thinking-32k", "gpt-5.1-high", "gpt-5.4-mini-high",
|
||
"glm-4.7", "qwen3-max-preview", "gpt-5-high", "kimi-k2.5-instant", "o3-2025-04-16",
|
||
"kimi-k2-thinking-turbo", "gpt-5-chat", "qwen3-max-2025-09-23", "qwen3-235b-a22b-instruct-2507",
|
||
"kimi-k2-0711-preview", "kimi-k2-0905-preview", "qwen3.5-122b-a10b", "minimax-m2.7",
|
||
"qwen3-vl-235b-a22b-instruct", "mistral-large-3", "gpt-4.1-2025-04-14", "gemini-2.5-flash",
|
||
"mistral-medium-2508", "qwen3.5-27b", "inkling-small", "qwen3-235b-a22b-no-thinking",
|
||
"gpt-5.4-nano-high", "longcat-flash-chat", "qwen3-next-80b-a3b-instruct",
|
||
"claude-sonnet-4-20250514-thinking-32k", "qwen3-235b-a22b-thinking-2507", "qwen3.5-flash",
|
||
"qwen3.5-35b-a3b", "hunyuan-vision-1.5-thinking", "qwen3-vl-235b-a22b-thinking",
|
||
"step-3.5-flash", "minimax-m2.5", "o4-mini-2025-04-16", "gpt-5-mini-high",
|
||
"claude-sonnet-4-20250514", "qwen3-coder-480b-a35b-instruct", "minimax-m2.1-preview",
|
||
"qwen3-30b-a3b-instruct-2507", "gpt-4.1-mini-2025-04-14", "trinity-large-preview",
|
||
"qwen3-235b-a22b", "trinity-large-thinking", "qwen3-next-80b-a3b-thinking",
|
||
"gemma-3-27b-it", "minimax-m1", "gemini-2.0-flash-001", "intellect-3",
|
||
"gemma-3-12b-it", "o3-mini-high", "gemma-3-4b-it", "mistral-small-3.2",
|
||
"qwen3-vl-30b-a3b-instruct", "gemini-2.5-flash-lite", "qwen3-30b-a3b-thinking-2507",
|
||
"qwen3-30b-a3b", "deepseek-v3.2", "gemma-3-1b-it", "deepseek-v4-flash",
|
||
"o3-mini-2025-01-31", "glm-4-flash", "gemini-2.0-flash-thinking-exp-01-21",
|
||
"qwen2.5-max-preview", "claude-3.5-haiku-20241022", "claude-3.7-sonnet",
|
||
"step-2-16k-exp", "claude-3.7-sonnet-thinking", "deepseek-r1-distill-qwen-32b",
|
||
"deepseek-v3.1", "deepseek-v4", "claude-3.5-sonnet-20241022", "gpt-4o-2024-11-20",
|
||
"deepseek-v4-pro-max", "claude-sonnet-5-high", "gpt-5.4-mini", "gemini-3.7-pro",
|
||
"gemini-3.7-pro-high", "glm-5.2 (max)", "deepseek-v4-flash-20260731",
|
||
"gpt-5.4-high", "claude-sonnet-5-search", "gemini-3.5-pro-high",
|
||
"gemini-3.5-flash-high", "deepseek-v4-flash-high", "gemini-3.5-flash",
|
||
"gpt-5.4-turbo", "gpt-5.4", "gemini-3.5-pro", "gemini-2.5-pro-high",
|
||
"gemini-3-flash-high", "gemini-3-pro-high", "gemini-3-flash-search",
|
||
"gemini-2.5-ultra", "gemini-3-pro", "gemini-2.5-flash-thinking",
|
||
"gemini-2.5-flash-high", "gemini-3.7-flash-high", "qwen3.5-122b-a10b-thinking",
|
||
"qwen3.5-35b-a3b-thinking", "claude-opus-4.5", "claude-opus-4.5-thinking",
|
||
"claude-opus-4.6", "claude-opus-4.6-thinking", "claude-opus-4.1",
|
||
"claude-opus-4.1-thinking", "claude-opus-4", "claude-opus-4-thinking",
|
||
"o4-mini-high", "o4-mini", "o4-max", "o4-high", "o3-pro-max", "o3-pro",
|
||
"gpt-5.3-high", "gpt-5.3-mini", "gpt-5.3", "gpt-5.2-mini", "gpt-5.2-turbo",
|
||
"gpt-5.1-turbo", "gpt-5.1-mini", "gpt-5.1-codex-max", "gpt-5-turbo", "gpt-5-mini",
|
||
"inkling-low", "qwen3-omni-flash", "inkling-small-low", "inkling-medium",
|
||
"gemini-3.7-flash", "mistral-medium-3.5", "gemini-3-flash (thinking-minimal)",
|
||
"qwen3.7-plus-preview", "grok-4.6-medium", "grok-4.5"
|
||
}
|
||
|
||
_catalog_io_lock = threading.RLock()
|
||
|
||
|
||
def _canonical_modality(value: Any) -> str:
|
||
"""Normalize Arena's page/API terminology to the catalog terminology."""
|
||
modality = str(value or "").strip().lower()
|
||
return {
|
||
"webdev": "code",
|
||
"web": "code",
|
||
"chat": "text",
|
||
}.get(modality, modality)
|
||
|
||
|
||
def _keywords_list(source: Any) -> List[str]:
|
||
if isinstance(source, str):
|
||
source = re.split(r"[\n,]+", source)
|
||
if not isinstance(source, (list, tuple, set)):
|
||
return []
|
||
result: List[str] = []
|
||
seen = set()
|
||
for item in source:
|
||
keyword = str(item or "").strip()
|
||
folded = keyword.casefold()
|
||
if not keyword or folded in seen:
|
||
continue
|
||
seen.add(folded)
|
||
result.append(keyword)
|
||
return result
|
||
|
||
|
||
def normalize_arena_model_catalog_config(value: Any) -> Dict[str, Any]:
|
||
raw = value if isinstance(value, dict) else {}
|
||
|
||
result = {
|
||
"enabled": bool(raw.get("enabled", False)),
|
||
"source": str(raw.get("source") or MODEL_CATALOG_SOURCE).strip() or MODEL_CATALOG_SOURCE,
|
||
"include_keywords": _keywords_list(raw.get("include_keywords")),
|
||
"exclude_keywords": _keywords_list(raw.get("exclude_keywords")),
|
||
"enable_dark_pool": bool(raw.get("enable_dark_pool", False)),
|
||
"dark_pool_since": str(raw.get("dark_pool_since") or "").strip(),
|
||
"dark_pool_whitelist_keywords": _keywords_list(raw.get("dark_pool_whitelist_keywords")),
|
||
"dark_pool_blacklist_keywords": _keywords_list(raw.get("dark_pool_blacklist_keywords")),
|
||
}
|
||
if "modality" in raw and str(raw.get("modality") or "").strip():
|
||
result["modality"] = _canonical_modality(raw.get("modality"))
|
||
return result
|
||
|
||
|
||
def _default_arena_model_catalog_config(domain: str, preset_name: str) -> Optional[Dict[str, Any]]:
|
||
"""Return built-in catalog defaults for presets that should start enabled."""
|
||
normalized_domain = str(domain or "").strip().lower().strip(".")
|
||
normalized_preset = str(preset_name or "").strip()
|
||
if route_domain_matches("arena.ai", normalized_domain) and normalized_preset == "主预设-直连模式":
|
||
return normalize_arena_model_catalog_config(
|
||
{
|
||
"enabled": True,
|
||
"source": MODEL_CATALOG_SOURCE,
|
||
"modality": "text",
|
||
}
|
||
)
|
||
return None
|
||
|
||
|
||
def load_arena_model_catalog_data() -> Dict[str, Dict[str, Any]]:
|
||
"""加载独立配置文件,返回 { domain: { preset_name: catalog_config } }"""
|
||
with _catalog_io_lock:
|
||
if not ARENA_MODEL_CATALOG_CONFIG_PATH.exists():
|
||
return {}
|
||
try:
|
||
content = ARENA_MODEL_CATALOG_CONFIG_PATH.read_text(encoding="utf-8").strip()
|
||
if not content:
|
||
return {}
|
||
data = json.loads(content)
|
||
if not isinstance(data, dict):
|
||
return {}
|
||
normalized: Dict[str, Dict[str, Any]] = {}
|
||
for domain, presets in data.items():
|
||
if not isinstance(presets, dict):
|
||
continue
|
||
d_key = str(domain).strip().lower()
|
||
normalized[d_key] = {}
|
||
for preset_name, cat in presets.items():
|
||
p_key = str(preset_name).strip()
|
||
if isinstance(cat, dict):
|
||
normalized[d_key][p_key] = normalize_arena_model_catalog_config(cat)
|
||
return normalized
|
||
except Exception as e:
|
||
logger.warning(f"读取 Arena 独立模型目录配置失败: {e}")
|
||
return {}
|
||
|
||
|
||
def save_arena_model_catalog_data(data: Dict[str, Any]) -> bool:
|
||
"""原子保存独立配置文件"""
|
||
with _catalog_io_lock:
|
||
try:
|
||
ARENA_MODEL_CATALOG_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||
tmp_file = ARENA_MODEL_CATALOG_CONFIG_PATH.with_suffix(".tmp")
|
||
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
|
||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||
f.flush()
|
||
os.fsync(f.fileno())
|
||
os.replace(tmp_file, ARENA_MODEL_CATALOG_CONFIG_PATH)
|
||
logger.info(f"Arena 独立模型目录配置已保存: {ARENA_MODEL_CATALOG_CONFIG_PATH}")
|
||
return True
|
||
except Exception as e:
|
||
logger.error(f"保存 Arena 独立模型目录配置失败: {e}")
|
||
return False
|
||
|
||
|
||
def get_arena_model_catalog(
|
||
domain: str = "arena.ai",
|
||
preset_name: Optional[str] = None,
|
||
config_engine: Any = None,
|
||
) -> Dict[str, Any]:
|
||
"""获取指定预设的 Arena 模型目录配置。支持旧配置兼容回退。"""
|
||
normalized_domain = str(domain or "arena.ai").strip().lower().strip(".")
|
||
catalog_data = load_arena_model_catalog_data()
|
||
domain_catalogs = catalog_data.get(normalized_domain, {})
|
||
|
||
target_preset = str(preset_name or "").strip()
|
||
if not target_preset and config_engine:
|
||
try:
|
||
target_preset = str(config_engine.get_default_preset(normalized_domain) or "").strip()
|
||
except Exception:
|
||
pass
|
||
if not target_preset:
|
||
target_preset = "主预设"
|
||
|
||
if target_preset in domain_catalogs:
|
||
return copy.deepcopy(domain_catalogs[target_preset])
|
||
|
||
# 兼容期回退:若独立配置中无该预设,检查 config_engine 站点预设中是否有旧 model_catalog
|
||
if config_engine:
|
||
try:
|
||
presets = None
|
||
if hasattr(config_engine, "sites") and isinstance(config_engine.sites, dict):
|
||
site = config_engine.sites.get(normalized_domain, {})
|
||
if isinstance(site, dict):
|
||
presets = site.get("presets")
|
||
if presets is None and hasattr(config_engine, "presets") and isinstance(config_engine.presets, dict):
|
||
presets = config_engine.presets
|
||
|
||
if isinstance(presets, dict):
|
||
resolved_key = config_engine._resolve_preset_alias_key(target_preset, presets) if hasattr(config_engine, "_resolve_preset_alias_key") else target_preset
|
||
preset_dict = presets.get(resolved_key)
|
||
if isinstance(preset_dict, dict) and "model_catalog" in preset_dict:
|
||
return normalize_arena_model_catalog_config(preset_dict.get("model_catalog"))
|
||
|
||
if hasattr(config_engine, "_get_site_data_readonly"):
|
||
preset_dict = config_engine._get_site_data_readonly(normalized_domain, target_preset)
|
||
if isinstance(preset_dict, dict) and "model_catalog" in preset_dict:
|
||
return normalize_arena_model_catalog_config(preset_dict.get("model_catalog"))
|
||
except Exception:
|
||
pass
|
||
|
||
default_catalog = _default_arena_model_catalog_config(normalized_domain, target_preset)
|
||
if default_catalog is not None:
|
||
return default_catalog
|
||
|
||
return normalize_arena_model_catalog_config({})
|
||
|
||
|
||
def set_arena_model_catalog(
|
||
domain: str = "arena.ai",
|
||
preset_name: Optional[str] = None,
|
||
catalog_config: Any = None,
|
||
) -> Dict[str, Any]:
|
||
"""设置并原子保存指定预设的 Arena 模型目录配置"""
|
||
normalized_domain = str(domain or "arena.ai").strip().lower().strip(".")
|
||
target_preset = str(preset_name or "主预设").strip()
|
||
normalized_cat = normalize_arena_model_catalog_config(catalog_config)
|
||
|
||
with _catalog_io_lock:
|
||
catalog_data = load_arena_model_catalog_data()
|
||
if normalized_domain not in catalog_data:
|
||
catalog_data[normalized_domain] = {}
|
||
catalog_data[normalized_domain][target_preset] = normalized_cat
|
||
if not save_arena_model_catalog_data(catalog_data):
|
||
raise IOError(f"保存 Arena 独立模型目录配置失败: {normalized_domain} [{target_preset}]")
|
||
|
||
return normalized_cat
|
||
|
||
|
||
def migrate_and_cleanup_sites_model_catalog(config_engine: Any) -> bool:
|
||
"""扫描 config_engine.sites,将各预设下的 model_catalog 迁移到独立文件,并从 sites.json 中清理旧字段"""
|
||
if not config_engine or not hasattr(config_engine, "sites") or not isinstance(config_engine.sites, dict):
|
||
return False
|
||
|
||
with _catalog_io_lock:
|
||
catalog_data = load_arena_model_catalog_data()
|
||
catalog_data_changed = False
|
||
to_cleanup = []
|
||
|
||
for domain, site_config in config_engine.sites.items():
|
||
if domain.startswith("_") or not isinstance(site_config, dict):
|
||
continue
|
||
d_key = str(domain).strip().lower().strip(".")
|
||
if not route_domain_matches("arena.ai", d_key):
|
||
continue
|
||
|
||
presets = site_config.get("presets")
|
||
if not isinstance(presets, dict):
|
||
continue
|
||
|
||
if d_key not in catalog_data:
|
||
catalog_data[d_key] = {}
|
||
|
||
for p_name, p_data in presets.items():
|
||
if not isinstance(p_data, dict):
|
||
continue
|
||
p_key = str(p_name).strip()
|
||
if "model_catalog" in p_data:
|
||
old_cat = p_data.get("model_catalog")
|
||
# 如果独立文件没有该预设配置,则补全迁移
|
||
if p_key not in catalog_data[d_key]:
|
||
catalog_data[d_key][p_key] = normalize_arena_model_catalog_config(old_cat)
|
||
catalog_data_changed = True
|
||
logger.info(f"已迁移站点 {domain} 预设 [{p_name}] 的 model_catalog 到独立配置文件")
|
||
to_cleanup.append((p_data, d_key, p_key))
|
||
|
||
if catalog_data_changed:
|
||
if not save_arena_model_catalog_data(catalog_data):
|
||
logger.error("保存独立模型目录配置文件失败,取消清理 sites.json 中的 model_catalog")
|
||
return False
|
||
|
||
# 只有在独立配置文件确认已包含该配置时,才安全清理内存与 sites.json
|
||
sites_changed = False
|
||
for p_data, d_key, p_key in to_cleanup:
|
||
if d_key in catalog_data and p_key in catalog_data[d_key]:
|
||
if "model_catalog" in p_data:
|
||
p_data.pop("model_catalog", None)
|
||
sites_changed = True
|
||
|
||
if sites_changed:
|
||
try:
|
||
config_engine.save_config()
|
||
logger.info("已从 sites.json 中清理已迁移的 model_catalog 残留字段")
|
||
except Exception as e:
|
||
logger.warning(f"清理 sites.json 中的 model_catalog 后保存失败: {e}")
|
||
|
||
return catalog_data_changed or sites_changed
|
||
|
||
|
||
def get_model_catalog_preset(config_engine: Any, domain: Any) -> Optional[Dict[str, Any]]:
|
||
"""读取指定域名下已启用 model_catalog 的预设信息"""
|
||
normalized_domain = str(domain or "").strip().lower().strip(".")
|
||
if not normalized_domain or not route_domain_matches("arena.ai", normalized_domain):
|
||
return None
|
||
|
||
try:
|
||
if hasattr(config_engine, "refresh_if_changed"):
|
||
config_engine.refresh_if_changed()
|
||
site = config_engine.sites.get(normalized_domain) if hasattr(config_engine, "sites") else None
|
||
except Exception:
|
||
site = None
|
||
|
||
presets = site.get("presets") if isinstance(site, dict) else (
|
||
getattr(config_engine, "presets", {}) if hasattr(config_engine, "presets") else {}
|
||
)
|
||
catalog_data = load_arena_model_catalog_data()
|
||
domain_catalogs = catalog_data.get(normalized_domain, {})
|
||
|
||
# 优先检查默认预设
|
||
default_preset = ""
|
||
if hasattr(config_engine, "get_default_preset"):
|
||
try:
|
||
default_preset = str(config_engine.get_default_preset(normalized_domain) or "").strip()
|
||
except Exception:
|
||
pass
|
||
|
||
all_preset_names = []
|
||
if default_preset:
|
||
all_preset_names.append(default_preset)
|
||
|
||
if isinstance(presets, dict):
|
||
for p_name in presets.keys():
|
||
if str(p_name) not in all_preset_names:
|
||
all_preset_names.append(str(p_name))
|
||
for p_name in domain_catalogs.keys():
|
||
if str(p_name) not in all_preset_names:
|
||
all_preset_names.append(str(p_name))
|
||
|
||
for preset_name in all_preset_names:
|
||
cat = domain_catalogs.get(preset_name)
|
||
if not cat:
|
||
# 兼容回退
|
||
cat = get_arena_model_catalog(normalized_domain, preset_name, config_engine=config_engine)
|
||
if cat and cat.get("enabled") and cat.get("source") == MODEL_CATALOG_SOURCE:
|
||
preset = presets.get(preset_name, {}) if isinstance(presets, dict) else {}
|
||
return {
|
||
"preset_name": str(preset_name),
|
||
"preset": preset,
|
||
"catalog": cat,
|
||
}
|
||
|
||
return None
|
||
|
||
|
||
def get_arena_direct_catalog_for_tab(
|
||
config_engine: Any,
|
||
tab: Any,
|
||
*,
|
||
preset_name: Any = None,
|
||
) -> Optional[Dict[str, Any]]:
|
||
"""获取当前标签页对应的 Arena 直连目录配置"""
|
||
if not isinstance(tab, dict):
|
||
return None
|
||
status = str(tab.get("status") or "").strip().lower()
|
||
if status not in {"idle", "busy"} or bool(tab.get("terminating")):
|
||
return None
|
||
|
||
current_url = str(tab.get("url") or tab.get("current_url") or "").strip()
|
||
effective_preset_name = str(preset_name or tab.get("preset_name") or "").strip()
|
||
try:
|
||
config_engine.refresh_if_changed()
|
||
if not effective_preset_name:
|
||
effective_preset_name = str(
|
||
config_engine.get_default_preset("arena.ai") or ""
|
||
).strip()
|
||
preset = config_engine._get_site_data_readonly(
|
||
"arena.ai",
|
||
effective_preset_name or None,
|
||
)
|
||
except Exception:
|
||
return None
|
||
if not isinstance(preset, dict):
|
||
return None
|
||
|
||
catalog = get_arena_model_catalog("arena.ai", effective_preset_name, config_engine=config_engine)
|
||
if not catalog.get("enabled") or catalog.get("source") != MODEL_CATALOG_SOURCE:
|
||
return None
|
||
|
||
from app.services.arena_direct_models import _is_arena_direct_url
|
||
if not _is_arena_direct_url(current_url, catalog_preset=preset, catalog_config=catalog):
|
||
return None
|
||
|
||
return {
|
||
"preset_name": effective_preset_name,
|
||
"preset": preset,
|
||
"catalog": catalog,
|
||
}
|
||
|
||
|
||
# ==============================================================================
|
||
# 暗池模型判定、时间戳解析与全量缓存加载
|
||
# ==============================================================================
|
||
|
||
def parse_uuidv7_timestamp(uuid_str: str) -> Optional[float]:
|
||
"""从 UUIDv7 中解析毫秒级时间戳"""
|
||
if not uuid_str or not isinstance(uuid_str, str):
|
||
return None
|
||
clean_hex = uuid_str.replace("-", "").strip().lower()
|
||
if len(clean_hex) != 32 or clean_hex[12] != "7":
|
||
return None
|
||
try:
|
||
ts_ms = int(clean_hex[:12], 16)
|
||
if 1577836800000 <= ts_ms <= 1893456000000:
|
||
return float(ts_ms)
|
||
except Exception:
|
||
pass
|
||
return None
|
||
|
||
|
||
def parse_date_to_timestamp(date_str: str) -> Optional[float]:
|
||
"""将 YYYY-MM-DD 字符串转为 UTC 00:00:00 毫秒时间戳"""
|
||
if not date_str or not isinstance(date_str, str):
|
||
return None
|
||
clean_str = date_str.strip()
|
||
m = re.match(r"^(\d{4})[-_/](\d{1,2})[-_/](\d{1,2})$", clean_str)
|
||
if m:
|
||
try:
|
||
year, month, day = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||
dt = datetime(year, month, day, 0, 0, 0, tzinfo=timezone.utc)
|
||
return dt.timestamp() * 1000.0
|
||
except Exception:
|
||
return None
|
||
return None
|
||
|
||
|
||
def get_model_timestamp(model: Dict[str, Any]) -> Optional[float]:
|
||
"""获取模型入库时间戳(ms)"""
|
||
mid = str(model.get("arena_model_id") or model.get("id") or "")
|
||
ts = parse_uuidv7_timestamp(mid)
|
||
if ts is not None:
|
||
return ts
|
||
|
||
name_str = " ".join(
|
||
str(model.get(k) or "") for k in ("name", "displayName", "display_name", "publicName", "public_name")
|
||
)
|
||
m = re.search(r"202[4-7][_-]?(0[1-9]|1[0-2])[_-]?([0-3][0-9])", name_str)
|
||
if m:
|
||
try:
|
||
match_full = m.group(0).replace("_", "-")
|
||
digits = re.sub(r"\D", "", match_full)
|
||
if len(digits) == 8:
|
||
year, month, day = int(digits[:4]), int(digits[4:6]), int(digits[6:])
|
||
dt = datetime(year, month, day, 0, 0, 0, tzinfo=timezone.utc)
|
||
return dt.timestamp() * 1000.0
|
||
except Exception:
|
||
pass
|
||
return None
|
||
|
||
|
||
def is_dark_pool_model(model: Dict[str, Any]) -> bool:
|
||
"""判定是否属于暗池(在前端模型选择器中隐藏或来自离线暗池缓存)"""
|
||
if "_is_dark_pool" in model:
|
||
return bool(model["_is_dark_pool"])
|
||
if model.get("is_dark_pool") is not None:
|
||
return bool(model.get("is_dark_pool"))
|
||
|
||
disp = str(model.get("display_name") or model.get("displayName") or "").strip().lower()
|
||
name = str(model.get("name") or "").strip().lower()
|
||
pub = str(model.get("public_name") or model.get("publicName") or "").strip().lower()
|
||
search_str = f"{disp} {name} {pub}"
|
||
|
||
# 包含明确内部/测试特征的直接归为暗池
|
||
if any(k in search_str for k in ["internal", "test", "dlp", "fireworks", "node"]):
|
||
return True
|
||
|
||
# 命中公开白名单的必定为明池
|
||
for pub_name in ONLINE_PUBLIC_VISIBLE_MODELS:
|
||
pub_lower = pub_name.lower()
|
||
if disp == pub_lower or pub == pub_lower or name == pub_lower:
|
||
return False
|
||
|
||
# 来自离线暗池全量缓存且不在公开白名单的归为暗池
|
||
if model.get("_is_offline_cache"):
|
||
return True
|
||
|
||
# 实时从页面抓取(或单测传入)的普通模型默认视为明池
|
||
return False
|
||
|
||
|
||
_offline_cache_lock = threading.Lock()
|
||
_offline_cached_models: Optional[List[Dict[str, Any]]] = None
|
||
|
||
|
||
def load_arena_offline_cache_models() -> List[Dict[str, Any]]:
|
||
"""加载本地 scripts/arena_models_cache.json 中的模型并规范化"""
|
||
global _offline_cached_models
|
||
with _offline_cache_lock:
|
||
if _offline_cached_models is not None:
|
||
return _offline_cached_models
|
||
|
||
if not ARENA_MODELS_CACHE_PATH.exists():
|
||
_offline_cached_models = []
|
||
return []
|
||
|
||
try:
|
||
with open(ARENA_MODELS_CACHE_PATH, "r", encoding="utf-8") as f:
|
||
raw_list = json.load(f)
|
||
if not isinstance(raw_list, list):
|
||
_offline_cached_models = []
|
||
return []
|
||
|
||
results = []
|
||
for item in raw_list:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
mid = str(item.get("id") or "").strip()
|
||
if not mid:
|
||
continue
|
||
|
||
caps = item.get("capabilities") or {}
|
||
out_caps = caps.get("outputCapabilities") or {}
|
||
ranks = item.get("rankByModality") or {}
|
||
|
||
def rank_is_finite(key):
|
||
val = ranks.get(key)
|
||
return isinstance(val, (int, float)) and val < 9007199254740991
|
||
|
||
has_text = bool(out_caps.get("text"))
|
||
has_image = bool(out_caps.get("image")) or rank_is_finite("image")
|
||
has_video = bool(out_caps.get("video")) or rank_is_finite("video")
|
||
# `outputCapabilities.web` is browsing/search support, not
|
||
# the WebDev/code modality. Use only the webdev rank here.
|
||
has_web = rank_is_finite("webdev") or bool(out_caps.get("web"))
|
||
has_search = bool(out_caps.get("search")) or rank_is_finite("search")
|
||
|
||
disp_name = str(item.get("displayName") or item.get("publicName") or item.get("name") or mid).strip()
|
||
internal_name = str(item.get("name") or item.get("id") or disp_name).strip()
|
||
name_search = f"{internal_name} {disp_name} {mid}".lower()
|
||
|
||
is_known_image = any(k in name_search for k in [
|
||
"gpt-image", "mona-lisa", "luna-lisa", "lina-alpha", "lina-f-alpha", "silver_halide",
|
||
"flux", "seedream", "seededit", "imagine", "imagen", "z-image", "midjourney", "dall-e", "recraft", "krea"
|
||
])
|
||
|
||
modality = "text"
|
||
if has_video and not has_text and not is_known_image:
|
||
modality = "video"
|
||
elif is_known_image or (has_image and not has_text):
|
||
modality = "image"
|
||
elif has_web and not has_text:
|
||
modality = "code"
|
||
elif has_text:
|
||
modality = "text"
|
||
elif has_web:
|
||
modality = "code"
|
||
elif has_search:
|
||
modality = "search"
|
||
elif has_image:
|
||
modality = "image"
|
||
elif has_video:
|
||
modality = "video"
|
||
|
||
results.append({
|
||
"arena_model_id": mid,
|
||
"name": internal_name,
|
||
"public_name": disp_name,
|
||
"display_name": disp_name,
|
||
"provider": str(item.get("provider") or ""),
|
||
"organization": str(item.get("organization") or item.get("provider") or "arena.ai"),
|
||
"modality": modality,
|
||
"_is_offline_cache": True,
|
||
})
|
||
|
||
_offline_cached_models = results
|
||
return results
|
||
except Exception as e:
|
||
logger.warning(f"读取 Arena 离线模型缓存失败: {e}")
|
||
_offline_cached_models = []
|
||
return []
|
||
|
||
|
||
# ==============================================================================
|
||
# 模型过滤引擎(明池 + 暗池)
|
||
# ==============================================================================
|
||
|
||
def filter_arena_catalog_models(
|
||
models: List[Dict[str, Any]],
|
||
catalog_config: Any,
|
||
) -> List[Dict[str, Any]]:
|
||
"""对模型列表执行明池与暗池的分别过滤与合并去重排序"""
|
||
from app.services.arena_direct_models import (
|
||
_canonical_modality,
|
||
_natural_sort_key,
|
||
get_arena_direct_model_public_id,
|
||
)
|
||
|
||
catalog = normalize_arena_model_catalog_config(catalog_config)
|
||
target_modality = _canonical_modality(catalog.get("modality") or "text")
|
||
enable_dark_pool = bool(catalog.get("enable_dark_pool", False))
|
||
|
||
dark_since_ts = parse_date_to_timestamp(catalog.get("dark_pool_since", ""))
|
||
dark_whitelist = [item.casefold() for item in catalog.get("dark_pool_whitelist_keywords", [])]
|
||
dark_blacklist = [item.casefold() for item in catalog.get("dark_pool_blacklist_keywords", [])]
|
||
|
||
pub_includes = [item.casefold() for item in catalog.get("include_keywords", [])]
|
||
pub_excludes = [item.casefold() for item in catalog.get("exclude_keywords", [])]
|
||
|
||
# 合并来自页面实时抓取与离线缓存的模型(按 arena_model_id 去重)
|
||
all_candidates: List[Dict[str, Any]] = []
|
||
seen_ids = set()
|
||
for m in (models or []) + (load_arena_offline_cache_models() if enable_dark_pool else []):
|
||
if not isinstance(m, dict):
|
||
continue
|
||
mid = str(m.get("arena_model_id") or "").strip().lower()
|
||
if not mid or mid in seen_ids:
|
||
continue
|
||
seen_ids.add(mid)
|
||
all_candidates.append(m)
|
||
|
||
video_keywords = (
|
||
"video", "seedance", "dreamina", "sora", "veo", "gemini-omni", "kling",
|
||
"hailuo", "runway", "pika", "mochi", "ltx", "vidu", "luma", "wan2",
|
||
"wan-2", "wan_2", "wanx", "pixverse", "kandinsky-video", "kandinsky_video", "happyhorse"
|
||
)
|
||
|
||
image_keywords = (
|
||
"image", "mona-lisa", "luna-lisa", "lina-alpha", "lina-f-alpha",
|
||
"silver_halide", "flux", "imagine", "seedream", "seededit", "imagen", "z-image",
|
||
"midjourney", "dall-e", "recraft", "krea"
|
||
)
|
||
|
||
result = []
|
||
result_seen_ids = set()
|
||
|
||
for model in all_candidates:
|
||
search_str = " ".join(
|
||
str(model.get(k) or "") for k in ("name", "public_name", "display_name", "search_name")
|
||
).lower()
|
||
if target_modality != "video":
|
||
if any((k in search_str and (k != "kling" or "inkling" not in search_str)) for k in video_keywords):
|
||
continue
|
||
|
||
model_modality = _canonical_modality(model.get("modality"))
|
||
if target_modality:
|
||
if target_modality in {"code", "search", "video"}:
|
||
if model_modality != target_modality:
|
||
continue
|
||
elif target_modality == "image":
|
||
model_is_image = (model_modality == "image") or (
|
||
not model_modality and any(k in search_str for k in image_keywords)
|
||
)
|
||
if not model_is_image:
|
||
continue
|
||
elif target_modality == "text":
|
||
if model_modality and model_modality != "text":
|
||
continue
|
||
if not model_modality and any(k in search_str for k in image_keywords):
|
||
continue
|
||
else:
|
||
if model_modality and model_modality != target_modality:
|
||
continue
|
||
|
||
searchable = " ".join(
|
||
str(model.get(key) or "")
|
||
for key in (
|
||
"name", "public_name", "display_name", "search_name", "provider", "organization"
|
||
)
|
||
).casefold()
|
||
searchable += " " + " ".join(
|
||
str(alias or "") for alias in (model.get("aliases") or [])
|
||
).casefold()
|
||
|
||
is_dark = is_dark_pool_model(model)
|
||
|
||
if is_dark:
|
||
# 暗池处理规则
|
||
if not enable_dark_pool:
|
||
continue
|
||
|
||
# 1. 黑名单最终否决
|
||
if dark_blacklist and any(kw in searchable for kw in dark_blacklist):
|
||
continue
|
||
|
||
# 2. 白名单与日期条件检查(若设置了白名单或日期,必须满足其中之一)
|
||
if dark_whitelist or dark_since_ts is not None:
|
||
whitelisted = bool(dark_whitelist and any(kw in searchable for kw in dark_whitelist))
|
||
if not whitelisted:
|
||
if dark_since_ts is None:
|
||
continue
|
||
model_ts = get_model_timestamp(model)
|
||
if model_ts is None or model_ts < dark_since_ts:
|
||
continue
|
||
else:
|
||
# 明池处理规则(不受暗池规则影响)
|
||
if pub_includes and not any(keyword in searchable for keyword in pub_includes):
|
||
continue
|
||
if pub_excludes and any(keyword in searchable for keyword in pub_excludes):
|
||
continue
|
||
|
||
mid_lower = str(model.get("arena_model_id") or "").lower()
|
||
if mid_lower in result_seen_ids:
|
||
continue
|
||
result_seen_ids.add(mid_lower)
|
||
result.append(model)
|
||
|
||
result.sort(key=lambda m: _natural_sort_key(get_arena_direct_model_public_id(m)))
|
||
return result
|
||
|
||
|
||
__all__ = [
|
||
"ARENA_MODEL_CATALOG_CONFIG_PATH",
|
||
"MODEL_CATALOG_SOURCE",
|
||
"ONLINE_PUBLIC_VISIBLE_MODELS",
|
||
"filter_arena_catalog_models",
|
||
"get_arena_direct_catalog_for_tab",
|
||
"get_arena_model_catalog",
|
||
"get_model_catalog_preset",
|
||
"get_model_timestamp",
|
||
"is_dark_pool_model",
|
||
"load_arena_model_catalog_data",
|
||
"load_arena_offline_cache_models",
|
||
"migrate_and_cleanup_sites_model_catalog",
|
||
"normalize_arena_model_catalog_config",
|
||
"parse_date_to_timestamp",
|
||
"parse_uuidv7_timestamp",
|
||
"save_arena_model_catalog_data",
|
||
"set_arena_model_catalog",
|
||
]
|