feat: crawl full Ollama library catalog

dashboard/plugin_api.py

@@ -1037,6 +1037,22 @@ def _parse_variant_page(html: str, family: str) -> list[dict[str, Any]]:
     return [row for row in rows if not row["is_mlx"]]
 
 
+def _fetch_library_families() -> list[str]:
+    """Return all public model-family slugs from Ollama's library index."""
+    try:
+        html = _text_request(f"{REMOTE_OLLAMA}/library", timeout=30)
+    except (HTTPError, URLError, OSError, ValueError):
+        return []
+    families: set[str] = set()
+    for href in re.findall(r'href=["\'](/library/[^"\']+)["\']', html, re.I):
+        path = unquote(href.split("?", 1)[0]).strip("/")
+        parts = path.split("/")
+        if len(parts) != 2 or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,190}", parts[1]):
+            continue
+        families.add(parts[1])
+    return sorted(families)
+
+
 def _fetch_family_variants(family: str) -> list[dict[str, Any]]:
     try:
         html = _text_request(f"{REMOTE_OLLAMA}/library/{family}/tags", timeout=30)
@@ -1065,8 +1081,25 @@ def _write_catalog(value: dict[str, Any]) -> None:
 
 
 def _refresh_family_variants(rows: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
-    families = sorted({_family_key(str(row.get("name") or row.get("model"))) for row in rows if not _is_mlx(row)})
-    return {family: _fetch_family_variants(family) for family in families if family}
+    families = {
+        _family_key(str(row.get("name") or row.get("model")))
+        for row in rows
+        if not _is_mlx(row)
+    }
+    families.update(_fetch_library_families())
+    families.discard("")
+    # The public library currently contains hundreds of families. Fetch tag
+    # pages concurrently so a daily refresh does not serialize network waits.
+    result: dict[str, list[dict[str, Any]]] = {}
+    with ThreadPoolExecutor(max_workers=8) as executor:
+        futures = {executor.submit(_fetch_family_variants, family): family for family in sorted(families)}
+        for future in as_completed(futures):
+            family = futures[future]
+            try:
+                result[family] = future.result()
+            except Exception:
+                result[family] = []
+    return result
 
 
 def refresh_catalog(force: bool = True) -> dict[str, Any]:
@@ -1077,7 +1110,7 @@ def refresh_catalog(force: bool = True) -> dict[str, Any]:
             rows = [item for item in payload.get("models", []) if isinstance(item, dict) and not _is_mlx(item)]
             catalog = {
                 "fetched_at": datetime.now(timezone.utc).isoformat(),
-                "source": f"{REMOTE_OLLAMA}/api/tags",
+                "source": f"{REMOTE_OLLAMA}/api/tags + {REMOTE_OLLAMA}/library",
                 "models": rows,
                 "families": _refresh_family_variants(_local_tags()),
             }