Files
universal-web-api/app/services/arena_model_catalog.py
T
2026-11-02 22:54:14 +08:00

752 lines
31 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Arena direct model catalog persistence, migration, and dark pool filtering."""
from __future__ import annotations
import copy
import json
import os
import re
import threading
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Any, Dict, List, Optional, Set, Tuple
from app.core.config import logger
from app.utils.site_url import extract_remote_site_domain, route_domain_matches
ARENA_MODEL_CATALOG_CONFIG_PATH = Path(
os.getenv(
"ARENA_MODEL_CATALOG_CONFIG_PATH",
str(Path(__file__).resolve().parents[2] / "config" / "arena_model_catalog.local.json"),
)
)
ARENA_MODELS_CACHE_PATH = Path(
os.getenv(
"ARENA_MODELS_CACHE_PATH",
str(Path(__file__).resolve().parents[2] / "scripts" / "arena_models_cache.json"),
)
)
MODEL_CATALOG_SOURCE = "arena_direct"
ONLINE_PUBLIC_VISIBLE_MODELS: Set[str] = {
"gemini-3-flash", "glm-5.1", "qwen3.5-397b-a17b", "claude-sonnet-4-5-20250929",
"gemini-3.1-pro-preview", "qwen3.7-plus", "minimax-m3", "claude-haiku-4-5-20251001",
"gemini-2.5-pro", "glm-5v-turbo", "grok-4.20-beta-0309-reasoning", "gpt-5.2-high",
"gpt-5.5-instant", "gpt-5.1", "gpt-5.2", "gemini-3.6-flash", "claude-sonnet-4-6",
"grok-4.20-multi-agent-beta-0309", "qwen3.5-max-preview", "gemini-3.5-flash-lite",
"glm-5", "claude-sonnet-4-5-20250929-thinking-32k", "gpt-5.1-high", "gpt-5.4-mini-high",
"glm-4.7", "qwen3-max-preview", "gpt-5-high", "kimi-k2.5-instant", "o3-2025-04-16",
"kimi-k2-thinking-turbo", "gpt-5-chat", "qwen3-max-2025-09-23", "qwen3-235b-a22b-instruct-2507",
"kimi-k2-0711-preview", "kimi-k2-0905-preview", "qwen3.5-122b-a10b", "minimax-m2.7",
"qwen3-vl-235b-a22b-instruct", "mistral-large-3", "gpt-4.1-2025-04-14", "gemini-2.5-flash",
"mistral-medium-2508", "qwen3.5-27b", "inkling-small", "qwen3-235b-a22b-no-thinking",
"gpt-5.4-nano-high", "longcat-flash-chat", "qwen3-next-80b-a3b-instruct",
"claude-sonnet-4-20250514-thinking-32k", "qwen3-235b-a22b-thinking-2507", "qwen3.5-flash",
"qwen3.5-35b-a3b", "hunyuan-vision-1.5-thinking", "qwen3-vl-235b-a22b-thinking",
"step-3.5-flash", "minimax-m2.5", "o4-mini-2025-04-16", "gpt-5-mini-high",
"claude-sonnet-4-20250514", "qwen3-coder-480b-a35b-instruct", "minimax-m2.1-preview",
"qwen3-30b-a3b-instruct-2507", "gpt-4.1-mini-2025-04-14", "trinity-large-preview",
"qwen3-235b-a22b", "trinity-large-thinking", "qwen3-next-80b-a3b-thinking",
"gemma-3-27b-it", "minimax-m1", "gemini-2.0-flash-001", "intellect-3",
"gemma-3-12b-it", "o3-mini-high", "gemma-3-4b-it", "mistral-small-3.2",
"qwen3-vl-30b-a3b-instruct", "gemini-2.5-flash-lite", "qwen3-30b-a3b-thinking-2507",
"qwen3-30b-a3b", "deepseek-v3.2", "gemma-3-1b-it", "deepseek-v4-flash",
"o3-mini-2025-01-31", "glm-4-flash", "gemini-2.0-flash-thinking-exp-01-21",
"qwen2.5-max-preview", "claude-3.5-haiku-20241022", "claude-3.7-sonnet",
"step-2-16k-exp", "claude-3.7-sonnet-thinking", "deepseek-r1-distill-qwen-32b",
"deepseek-v3.1", "deepseek-v4", "claude-3.5-sonnet-20241022", "gpt-4o-2024-11-20",
"deepseek-v4-pro-max", "claude-sonnet-5-high", "gpt-5.4-mini", "gemini-3.7-pro",
"gemini-3.7-pro-high", "glm-5.2 (max)", "deepseek-v4-flash-20260731",
"gpt-5.4-high", "claude-sonnet-5-search", "gemini-3.5-pro-high",
"gemini-3.5-flash-high", "deepseek-v4-flash-high", "gemini-3.5-flash",
"gpt-5.4-turbo", "gpt-5.4", "gemini-3.5-pro", "gemini-2.5-pro-high",
"gemini-3-flash-high", "gemini-3-pro-high", "gemini-3-flash-search",
"gemini-2.5-ultra", "gemini-3-pro", "gemini-2.5-flash-thinking",
"gemini-2.5-flash-high", "gemini-3.7-flash-high", "qwen3.5-122b-a10b-thinking",
"qwen3.5-35b-a3b-thinking", "claude-opus-4.5", "claude-opus-4.5-thinking",
"claude-opus-4.6", "claude-opus-4.6-thinking", "claude-opus-4.1",
"claude-opus-4.1-thinking", "claude-opus-4", "claude-opus-4-thinking",
"o4-mini-high", "o4-mini", "o4-max", "o4-high", "o3-pro-max", "o3-pro",
"gpt-5.3-high", "gpt-5.3-mini", "gpt-5.3", "gpt-5.2-mini", "gpt-5.2-turbo",
"gpt-5.1-turbo", "gpt-5.1-mini", "gpt-5.1-codex-max", "gpt-5-turbo", "gpt-5-mini",
"inkling-low", "qwen3-omni-flash", "inkling-small-low", "inkling-medium",
"gemini-3.7-flash", "mistral-medium-3.5", "gemini-3-flash (thinking-minimal)",
"qwen3.7-plus-preview", "grok-4.6-medium", "grok-4.5"
}
_catalog_io_lock = threading.RLock()
def _canonical_modality(value: Any) -> str:
"""Normalize Arena's page/API terminology to the catalog terminology."""
modality = str(value or "").strip().lower()
return {
"webdev": "code",
"web": "code",
"chat": "text",
}.get(modality, modality)
def _keywords_list(source: Any) -> List[str]:
if isinstance(source, str):
source = re.split(r"[\n,]+", source)
if not isinstance(source, (list, tuple, set)):
return []
result: List[str] = []
seen = set()
for item in source:
keyword = str(item or "").strip()
folded = keyword.casefold()
if not keyword or folded in seen:
continue
seen.add(folded)
result.append(keyword)
return result
def normalize_arena_model_catalog_config(value: Any) -> Dict[str, Any]:
raw = value if isinstance(value, dict) else {}
result = {
"enabled": bool(raw.get("enabled", False)),
"source": str(raw.get("source") or MODEL_CATALOG_SOURCE).strip() or MODEL_CATALOG_SOURCE,
"include_keywords": _keywords_list(raw.get("include_keywords")),
"exclude_keywords": _keywords_list(raw.get("exclude_keywords")),
"enable_dark_pool": bool(raw.get("enable_dark_pool", False)),
"dark_pool_since": str(raw.get("dark_pool_since") or "").strip(),
"dark_pool_whitelist_keywords": _keywords_list(raw.get("dark_pool_whitelist_keywords")),
"dark_pool_blacklist_keywords": _keywords_list(raw.get("dark_pool_blacklist_keywords")),
}
if "modality" in raw and str(raw.get("modality") or "").strip():
result["modality"] = _canonical_modality(raw.get("modality"))
return result
def _default_arena_model_catalog_config(domain: str, preset_name: str) -> Optional[Dict[str, Any]]:
"""Return built-in catalog defaults for presets that should start enabled."""
normalized_domain = str(domain or "").strip().lower().strip(".")
normalized_preset = str(preset_name or "").strip()
if route_domain_matches("arena.ai", normalized_domain) and normalized_preset == "主预设-直连模式":
return normalize_arena_model_catalog_config(
{
"enabled": True,
"source": MODEL_CATALOG_SOURCE,
"modality": "text",
}
)
return None
def load_arena_model_catalog_data() -> Dict[str, Dict[str, Any]]:
"""加载独立配置文件,返回 { domain: { preset_name: catalog_config } }"""
with _catalog_io_lock:
if not ARENA_MODEL_CATALOG_CONFIG_PATH.exists():
return {}
try:
content = ARENA_MODEL_CATALOG_CONFIG_PATH.read_text(encoding="utf-8").strip()
if not content:
return {}
data = json.loads(content)
if not isinstance(data, dict):
return {}
normalized: Dict[str, Dict[str, Any]] = {}
for domain, presets in data.items():
if not isinstance(presets, dict):
continue
d_key = str(domain).strip().lower()
normalized[d_key] = {}
for preset_name, cat in presets.items():
p_key = str(preset_name).strip()
if isinstance(cat, dict):
normalized[d_key][p_key] = normalize_arena_model_catalog_config(cat)
return normalized
except Exception as e:
logger.warning(f"读取 Arena 独立模型目录配置失败: {e}")
return {}
def save_arena_model_catalog_data(data: Dict[str, Any]) -> bool:
"""原子保存独立配置文件"""
with _catalog_io_lock:
try:
ARENA_MODEL_CATALOG_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
tmp_file = ARENA_MODEL_CATALOG_CONFIG_PATH.with_suffix(".tmp")
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, ARENA_MODEL_CATALOG_CONFIG_PATH)
logger.info(f"Arena 独立模型目录配置已保存: {ARENA_MODEL_CATALOG_CONFIG_PATH}")
return True
except Exception as e:
logger.error(f"保存 Arena 独立模型目录配置失败: {e}")
return False
def get_arena_model_catalog(
domain: str = "arena.ai",
preset_name: Optional[str] = None,
config_engine: Any = None,
) -> Dict[str, Any]:
"""获取指定预设的 Arena 模型目录配置。支持旧配置兼容回退。"""
normalized_domain = str(domain or "arena.ai").strip().lower().strip(".")
catalog_data = load_arena_model_catalog_data()
domain_catalogs = catalog_data.get(normalized_domain, {})
target_preset = str(preset_name or "").strip()
if not target_preset and config_engine:
try:
target_preset = str(config_engine.get_default_preset(normalized_domain) or "").strip()
except Exception:
pass
if not target_preset:
target_preset = "主预设"
if target_preset in domain_catalogs:
return copy.deepcopy(domain_catalogs[target_preset])
# 兼容期回退:若独立配置中无该预设,检查 config_engine 站点预设中是否有旧 model_catalog
if config_engine:
try:
presets = None
if hasattr(config_engine, "sites") and isinstance(config_engine.sites, dict):
site = config_engine.sites.get(normalized_domain, {})
if isinstance(site, dict):
presets = site.get("presets")
if presets is None and hasattr(config_engine, "presets") and isinstance(config_engine.presets, dict):
presets = config_engine.presets
if isinstance(presets, dict):
resolved_key = config_engine._resolve_preset_alias_key(target_preset, presets) if hasattr(config_engine, "_resolve_preset_alias_key") else target_preset
preset_dict = presets.get(resolved_key)
if isinstance(preset_dict, dict) and "model_catalog" in preset_dict:
return normalize_arena_model_catalog_config(preset_dict.get("model_catalog"))
if hasattr(config_engine, "_get_site_data_readonly"):
preset_dict = config_engine._get_site_data_readonly(normalized_domain, target_preset)
if isinstance(preset_dict, dict) and "model_catalog" in preset_dict:
return normalize_arena_model_catalog_config(preset_dict.get("model_catalog"))
except Exception:
pass
default_catalog = _default_arena_model_catalog_config(normalized_domain, target_preset)
if default_catalog is not None:
return default_catalog
return normalize_arena_model_catalog_config({})
def set_arena_model_catalog(
domain: str = "arena.ai",
preset_name: Optional[str] = None,
catalog_config: Any = None,
) -> Dict[str, Any]:
"""设置并原子保存指定预设的 Arena 模型目录配置"""
normalized_domain = str(domain or "arena.ai").strip().lower().strip(".")
target_preset = str(preset_name or "主预设").strip()
normalized_cat = normalize_arena_model_catalog_config(catalog_config)
with _catalog_io_lock:
catalog_data = load_arena_model_catalog_data()
if normalized_domain not in catalog_data:
catalog_data[normalized_domain] = {}
catalog_data[normalized_domain][target_preset] = normalized_cat
if not save_arena_model_catalog_data(catalog_data):
raise IOError(f"保存 Arena 独立模型目录配置失败: {normalized_domain} [{target_preset}]")
return normalized_cat
def migrate_and_cleanup_sites_model_catalog(config_engine: Any) -> bool:
"""扫描 config_engine.sites,将各预设下的 model_catalog 迁移到独立文件,并从 sites.json 中清理旧字段"""
if not config_engine or not hasattr(config_engine, "sites") or not isinstance(config_engine.sites, dict):
return False
with _catalog_io_lock:
catalog_data = load_arena_model_catalog_data()
catalog_data_changed = False
to_cleanup = []
for domain, site_config in config_engine.sites.items():
if domain.startswith("_") or not isinstance(site_config, dict):
continue
d_key = str(domain).strip().lower().strip(".")
if not route_domain_matches("arena.ai", d_key):
continue
presets = site_config.get("presets")
if not isinstance(presets, dict):
continue
if d_key not in catalog_data:
catalog_data[d_key] = {}
for p_name, p_data in presets.items():
if not isinstance(p_data, dict):
continue
p_key = str(p_name).strip()
if "model_catalog" in p_data:
old_cat = p_data.get("model_catalog")
# 如果独立文件没有该预设配置,则补全迁移
if p_key not in catalog_data[d_key]:
catalog_data[d_key][p_key] = normalize_arena_model_catalog_config(old_cat)
catalog_data_changed = True
logger.info(f"已迁移站点 {domain} 预设 [{p_name}] 的 model_catalog 到独立配置文件")
to_cleanup.append((p_data, d_key, p_key))
if catalog_data_changed:
if not save_arena_model_catalog_data(catalog_data):
logger.error("保存独立模型目录配置文件失败,取消清理 sites.json 中的 model_catalog")
return False
# 只有在独立配置文件确认已包含该配置时,才安全清理内存与 sites.json
sites_changed = False
for p_data, d_key, p_key in to_cleanup:
if d_key in catalog_data and p_key in catalog_data[d_key]:
if "model_catalog" in p_data:
p_data.pop("model_catalog", None)
sites_changed = True
if sites_changed:
try:
config_engine.save_config()
logger.info("已从 sites.json 中清理已迁移的 model_catalog 残留字段")
except Exception as e:
logger.warning(f"清理 sites.json 中的 model_catalog 后保存失败: {e}")
return catalog_data_changed or sites_changed
def get_model_catalog_preset(config_engine: Any, domain: Any) -> Optional[Dict[str, Any]]:
"""读取指定域名下已启用 model_catalog 的预设信息"""
normalized_domain = str(domain or "").strip().lower().strip(".")
if not normalized_domain or not route_domain_matches("arena.ai", normalized_domain):
return None
try:
if hasattr(config_engine, "refresh_if_changed"):
config_engine.refresh_if_changed()
site = config_engine.sites.get(normalized_domain) if hasattr(config_engine, "sites") else None
except Exception:
site = None
presets = site.get("presets") if isinstance(site, dict) else (
getattr(config_engine, "presets", {}) if hasattr(config_engine, "presets") else {}
)
catalog_data = load_arena_model_catalog_data()
domain_catalogs = catalog_data.get(normalized_domain, {})
# 优先检查默认预设
default_preset = ""
if hasattr(config_engine, "get_default_preset"):
try:
default_preset = str(config_engine.get_default_preset(normalized_domain) or "").strip()
except Exception:
pass
all_preset_names = []
if default_preset:
all_preset_names.append(default_preset)
if isinstance(presets, dict):
for p_name in presets.keys():
if str(p_name) not in all_preset_names:
all_preset_names.append(str(p_name))
for p_name in domain_catalogs.keys():
if str(p_name) not in all_preset_names:
all_preset_names.append(str(p_name))
for preset_name in all_preset_names:
cat = domain_catalogs.get(preset_name)
if not cat:
# 兼容回退
cat = get_arena_model_catalog(normalized_domain, preset_name, config_engine=config_engine)
if cat and cat.get("enabled") and cat.get("source") == MODEL_CATALOG_SOURCE:
preset = presets.get(preset_name, {}) if isinstance(presets, dict) else {}
return {
"preset_name": str(preset_name),
"preset": preset,
"catalog": cat,
}
return None
def get_arena_direct_catalog_for_tab(
config_engine: Any,
tab: Any,
*,
preset_name: Any = None,
) -> Optional[Dict[str, Any]]:
"""获取当前标签页对应的 Arena 直连目录配置"""
if not isinstance(tab, dict):
return None
status = str(tab.get("status") or "").strip().lower()
if status not in {"idle", "busy"} or bool(tab.get("terminating")):
return None
current_url = str(tab.get("url") or tab.get("current_url") or "").strip()
effective_preset_name = str(preset_name or tab.get("preset_name") or "").strip()
try:
config_engine.refresh_if_changed()
if not effective_preset_name:
effective_preset_name = str(
config_engine.get_default_preset("arena.ai") or ""
).strip()
preset = config_engine._get_site_data_readonly(
"arena.ai",
effective_preset_name or None,
)
except Exception:
return None
if not isinstance(preset, dict):
return None
catalog = get_arena_model_catalog("arena.ai", effective_preset_name, config_engine=config_engine)
if not catalog.get("enabled") or catalog.get("source") != MODEL_CATALOG_SOURCE:
return None
from app.services.arena_direct_models import _is_arena_direct_url
if not _is_arena_direct_url(current_url, catalog_preset=preset, catalog_config=catalog):
return None
return {
"preset_name": effective_preset_name,
"preset": preset,
"catalog": catalog,
}
# ==============================================================================
# 暗池模型判定、时间戳解析与全量缓存加载
# ==============================================================================
def parse_uuidv7_timestamp(uuid_str: str) -> Optional[float]:
"""从 UUIDv7 中解析毫秒级时间戳"""
if not uuid_str or not isinstance(uuid_str, str):
return None
clean_hex = uuid_str.replace("-", "").strip().lower()
if len(clean_hex) != 32 or clean_hex[12] != "7":
return None
try:
ts_ms = int(clean_hex[:12], 16)
if 1577836800000 <= ts_ms <= 1893456000000:
return float(ts_ms)
except Exception:
pass
return None
def parse_date_to_timestamp(date_str: str) -> Optional[float]:
"""将 YYYY-MM-DD 字符串转为 UTC 00:00:00 毫秒时间戳"""
if not date_str or not isinstance(date_str, str):
return None
clean_str = date_str.strip()
m = re.match(r"^(\d{4})[-_/](\d{1,2})[-_/](\d{1,2})$", clean_str)
if m:
try:
year, month, day = int(m.group(1)), int(m.group(2)), int(m.group(3))
dt = datetime(year, month, day, 0, 0, 0, tzinfo=timezone.utc)
return dt.timestamp() * 1000.0
except Exception:
return None
return None
def get_model_timestamp(model: Dict[str, Any]) -> Optional[float]:
"""获取模型入库时间戳(ms)"""
mid = str(model.get("arena_model_id") or model.get("id") or "")
ts = parse_uuidv7_timestamp(mid)
if ts is not None:
return ts
name_str = " ".join(
str(model.get(k) or "") for k in ("name", "displayName", "display_name", "publicName", "public_name")
)
m = re.search(r"202[4-7][_-]?(0[1-9]|1[0-2])[_-]?([0-3][0-9])", name_str)
if m:
try:
match_full = m.group(0).replace("_", "-")
digits = re.sub(r"\D", "", match_full)
if len(digits) == 8:
year, month, day = int(digits[:4]), int(digits[4:6]), int(digits[6:])
dt = datetime(year, month, day, 0, 0, 0, tzinfo=timezone.utc)
return dt.timestamp() * 1000.0
except Exception:
pass
return None
def is_dark_pool_model(model: Dict[str, Any]) -> bool:
"""判定是否属于暗池(在前端模型选择器中隐藏或来自离线暗池缓存)"""
if "_is_dark_pool" in model:
return bool(model["_is_dark_pool"])
if model.get("is_dark_pool") is not None:
return bool(model.get("is_dark_pool"))
disp = str(model.get("display_name") or model.get("displayName") or "").strip().lower()
name = str(model.get("name") or "").strip().lower()
pub = str(model.get("public_name") or model.get("publicName") or "").strip().lower()
search_str = f"{disp} {name} {pub}"
# 包含明确内部/测试特征的直接归为暗池
if any(k in search_str for k in ["internal", "test", "dlp", "fireworks", "node"]):
return True
# 命中公开白名单的必定为明池
for pub_name in ONLINE_PUBLIC_VISIBLE_MODELS:
pub_lower = pub_name.lower()
if disp == pub_lower or pub == pub_lower or name == pub_lower:
return False
# 来自离线暗池全量缓存且不在公开白名单的归为暗池
if model.get("_is_offline_cache"):
return True
# 实时从页面抓取(或单测传入)的普通模型默认视为明池
return False
_offline_cache_lock = threading.Lock()
_offline_cached_models: Optional[List[Dict[str, Any]]] = None
def load_arena_offline_cache_models() -> List[Dict[str, Any]]:
"""加载本地 scripts/arena_models_cache.json 中的模型并规范化"""
global _offline_cached_models
with _offline_cache_lock:
if _offline_cached_models is not None:
return _offline_cached_models
if not ARENA_MODELS_CACHE_PATH.exists():
_offline_cached_models = []
return []
try:
with open(ARENA_MODELS_CACHE_PATH, "r", encoding="utf-8") as f:
raw_list = json.load(f)
if not isinstance(raw_list, list):
_offline_cached_models = []
return []
results = []
for item in raw_list:
if not isinstance(item, dict):
continue
mid = str(item.get("id") or "").strip()
if not mid:
continue
caps = item.get("capabilities") or {}
out_caps = caps.get("outputCapabilities") or {}
ranks = item.get("rankByModality") or {}
def rank_is_finite(key):
val = ranks.get(key)
return isinstance(val, (int, float)) and val < 9007199254740991
has_text = bool(out_caps.get("text"))
has_image = bool(out_caps.get("image")) or rank_is_finite("image")
has_video = bool(out_caps.get("video")) or rank_is_finite("video")
# `outputCapabilities.web` is browsing/search support, not
# the WebDev/code modality. Use only the webdev rank here.
has_web = rank_is_finite("webdev") or bool(out_caps.get("web"))
has_search = bool(out_caps.get("search")) or rank_is_finite("search")
disp_name = str(item.get("displayName") or item.get("publicName") or item.get("name") or mid).strip()
internal_name = str(item.get("name") or item.get("id") or disp_name).strip()
name_search = f"{internal_name} {disp_name} {mid}".lower()
is_known_image = any(k in name_search for k in [
"gpt-image", "mona-lisa", "luna-lisa", "lina-alpha", "lina-f-alpha", "silver_halide",
"flux", "seedream", "seededit", "imagine", "imagen", "z-image", "midjourney", "dall-e", "recraft", "krea"
])
modality = "text"
if has_video and not has_text and not is_known_image:
modality = "video"
elif is_known_image or (has_image and not has_text):
modality = "image"
elif has_web and not has_text:
modality = "code"
elif has_text:
modality = "text"
elif has_web:
modality = "code"
elif has_search:
modality = "search"
elif has_image:
modality = "image"
elif has_video:
modality = "video"
results.append({
"arena_model_id": mid,
"name": internal_name,
"public_name": disp_name,
"display_name": disp_name,
"provider": str(item.get("provider") or ""),
"organization": str(item.get("organization") or item.get("provider") or "arena.ai"),
"modality": modality,
"_is_offline_cache": True,
})
_offline_cached_models = results
return results
except Exception as e:
logger.warning(f"读取 Arena 离线模型缓存失败: {e}")
_offline_cached_models = []
return []
# ==============================================================================
# 模型过滤引擎(明池 + 暗池)
# ==============================================================================
def filter_arena_catalog_models(
models: List[Dict[str, Any]],
catalog_config: Any,
) -> List[Dict[str, Any]]:
"""对模型列表执行明池与暗池的分别过滤与合并去重排序"""
from app.services.arena_direct_models import (
_canonical_modality,
_natural_sort_key,
get_arena_direct_model_public_id,
)
catalog = normalize_arena_model_catalog_config(catalog_config)
target_modality = _canonical_modality(catalog.get("modality") or "text")
enable_dark_pool = bool(catalog.get("enable_dark_pool", False))
dark_since_ts = parse_date_to_timestamp(catalog.get("dark_pool_since", ""))
dark_whitelist = [item.casefold() for item in catalog.get("dark_pool_whitelist_keywords", [])]
dark_blacklist = [item.casefold() for item in catalog.get("dark_pool_blacklist_keywords", [])]
pub_includes = [item.casefold() for item in catalog.get("include_keywords", [])]
pub_excludes = [item.casefold() for item in catalog.get("exclude_keywords", [])]
# 合并来自页面实时抓取与离线缓存的模型(按 arena_model_id 去重)
all_candidates: List[Dict[str, Any]] = []
seen_ids = set()
for m in (models or []) + (load_arena_offline_cache_models() if enable_dark_pool else []):
if not isinstance(m, dict):
continue
mid = str(m.get("arena_model_id") or "").strip().lower()
if not mid or mid in seen_ids:
continue
seen_ids.add(mid)
all_candidates.append(m)
video_keywords = (
"video", "seedance", "dreamina", "sora", "veo", "gemini-omni", "kling",
"hailuo", "runway", "pika", "mochi", "ltx", "vidu", "luma", "wan2",
"wan-2", "wan_2", "wanx", "pixverse", "kandinsky-video", "kandinsky_video", "happyhorse"
)
image_keywords = (
"image", "mona-lisa", "luna-lisa", "lina-alpha", "lina-f-alpha",
"silver_halide", "flux", "imagine", "seedream", "seededit", "imagen", "z-image",
"midjourney", "dall-e", "recraft", "krea"
)
result = []
result_seen_ids = set()
for model in all_candidates:
search_str = " ".join(
str(model.get(k) or "") for k in ("name", "public_name", "display_name", "search_name")
).lower()
if target_modality != "video":
if any((k in search_str and (k != "kling" or "inkling" not in search_str)) for k in video_keywords):
continue
model_modality = _canonical_modality(model.get("modality"))
if target_modality:
if target_modality in {"code", "search", "video"}:
if model_modality != target_modality:
continue
elif target_modality == "image":
model_is_image = (model_modality == "image") or (
not model_modality and any(k in search_str for k in image_keywords)
)
if not model_is_image:
continue
elif target_modality == "text":
if model_modality and model_modality != "text":
continue
if not model_modality and any(k in search_str for k in image_keywords):
continue
else:
if model_modality and model_modality != target_modality:
continue
searchable = " ".join(
str(model.get(key) or "")
for key in (
"name", "public_name", "display_name", "search_name", "provider", "organization"
)
).casefold()
searchable += " " + " ".join(
str(alias or "") for alias in (model.get("aliases") or [])
).casefold()
is_dark = is_dark_pool_model(model)
if is_dark:
# 暗池处理规则
if not enable_dark_pool:
continue
# 1. 黑名单最终否决
if dark_blacklist and any(kw in searchable for kw in dark_blacklist):
continue
# 2. 白名单与日期条件检查(若设置了白名单或日期,必须满足其中之一)
if dark_whitelist or dark_since_ts is not None:
whitelisted = bool(dark_whitelist and any(kw in searchable for kw in dark_whitelist))
if not whitelisted:
if dark_since_ts is None:
continue
model_ts = get_model_timestamp(model)
if model_ts is None or model_ts < dark_since_ts:
continue
else:
# 明池处理规则(不受暗池规则影响)
if pub_includes and not any(keyword in searchable for keyword in pub_includes):
continue
if pub_excludes and any(keyword in searchable for keyword in pub_excludes):
continue
mid_lower = str(model.get("arena_model_id") or "").lower()
if mid_lower in result_seen_ids:
continue
result_seen_ids.add(mid_lower)
result.append(model)
result.sort(key=lambda m: _natural_sort_key(get_arena_direct_model_public_id(m)))
return result
__all__ = [
"ARENA_MODEL_CATALOG_CONFIG_PATH",
"MODEL_CATALOG_SOURCE",
"ONLINE_PUBLIC_VISIBLE_MODELS",
"filter_arena_catalog_models",
"get_arena_direct_catalog_for_tab",
"get_arena_model_catalog",
"get_model_catalog_preset",
"get_model_timestamp",
"is_dark_pool_model",
"load_arena_model_catalog_data",
"load_arena_offline_cache_models",
"migrate_and_cleanup_sites_model_catalog",
"normalize_arena_model_catalog_config",
"parse_date_to_timestamp",
"parse_uuidv7_timestamp",
"save_arena_model_catalog_data",
"set_arena_model_catalog",
]