stage #4561 + stamp v0.51.545 — extension manifest asset bundles

santastabber. HERMES_WEBUI_EXTENSION_MANIFEST: bundle extension assets via a 64KB JSON manifest
instead of long URL lists. Same same-origin allowlist validator, traversal-guarded manifest path,
default-off. Nathan blessed (richer extension system). Co-authored-by: santastabber <santastabber@users.noreply.github.com>

api/extensions.py

@@ -6,10 +6,11 @@ It is disabled by default and never executes or fetches third-party URLs.
 """
 
 import html
+import json
 import logging
 import os
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Tuple
 from urllib.parse import unquote, urlsplit
 
 from api.helpers import _security_headers, j
@@ -22,14 +23,24 @@ _log = logging.getLogger(__name__)
 # avoids rendering tens of thousands of <script> tags into every page load.
 _MAX_URL_LIST = 32
 
+# Keep extension manifests small and auditable. The manifest is a convenience for
+# bundling static assets, not a package manager or dependency lockfile.
+_MAX_MANIFEST_BYTES = 64 * 1024
+
 # Tracks rejected URL strings we've already warned about so a misconfigured env
 # var doesn't spam the log on every request that re-reads it.
 _warned_urls: set = set()
 
+
+class _ManifestTooLarge(ValueError):
+    pass
+
+
 EXTENSION_ROUTE_PREFIX = "/extensions/"
 _EXTENSION_DIR_ENV = "HERMES_WEBUI_EXTENSION_DIR"
 _EXTENSION_SCRIPT_URLS_ENV = "HERMES_WEBUI_EXTENSION_SCRIPT_URLS"
 _EXTENSION_STYLESHEET_URLS_ENV = "HERMES_WEBUI_EXTENSION_STYLESHEET_URLS"
+_EXTENSION_MANIFEST_ENV = "HERMES_WEBUI_EXTENSION_MANIFEST"
 _ALLOWED_ASSET_PREFIXES = ("/extensions/", "/static/")
 
 _EXTENSION_MIME = {
@@ -107,47 +118,185 @@ def _is_safe_asset_url(value: str) -> bool:
     return False
 
 
-def _read_url_list(env_name: str) -> List[str]:
-    raw = os.getenv(env_name, "")
-    urls = []
-    for item in raw.split(","):
-        value = item.strip()
-        if not value:
-            continue
-        if _is_safe_asset_url(value):
-            urls.append(value)
-            if len(urls) >= _MAX_URL_LIST:
-                # Stop accumulating after the cap. Anything past this point
-                # would be silently dropped anyway; logging once makes the
-                # truncation visible to a confused operator.
-                if env_name not in _warned_urls:
-                    _warned_urls.add(env_name)
-                    _log.warning(
-                        "Extension URL list %s truncated at %d entries",
-                        env_name, _MAX_URL_LIST,
-                    )
-                break
-        elif value not in _warned_urls:
-            # First-time-seen invalid URL: log once per process so a typo
-            # in HERMES_WEBUI_EXTENSION_*_URLS doesn't disappear silently.
-            _warned_urls.add(value)
+def _warn_rejected_url(value: str, source: str) -> None:
+    if value in _warned_urls:
+        return
+    _warned_urls.add(value)
+    _log.warning(
+        "Rejected extension URL %r from %s (not a same-origin "
+        "/extensions/ or /static/ path, or contains unsafe chars)",
+        value, source,
+    )
+
+
+def _append_safe_asset_url(
+    urls: List[str], value: str, source: str, *, dedupe: bool = True
+) -> bool:
+    """Append a validated URL while preserving order and the global cap.
+
+    Returns False when the caller should stop accumulating entries for this list.
+    Manifest paths dedupe by default, while env-only lists preserve their legacy
+    behavior unless they are appending after manifest-provided assets.
+    """
+    value = value.strip() if isinstance(value, str) else ""
+    if not value:
+        return True
+    if not _is_safe_asset_url(value):
+        _warn_rejected_url(value, source)
+        return True
+    if dedupe and value in urls:
+        return True
+    if len(urls) >= _MAX_URL_LIST:
+        if source not in _warned_urls:
+            _warned_urls.add(source)
             _log.warning(
-                "Rejected extension URL %r from %s (not a same-origin "
-                "/extensions/ or /static/ path, or contains unsafe chars)",
-                value, env_name,
+                "Extension URL list %s truncated at %d entries",
+                source, _MAX_URL_LIST,
             )
+        return False
+    urls.append(value)
+    return True
+
+
+def _read_url_list(env_name: str, existing: Optional[List[str]] = None) -> List[str]:
+    raw = os.getenv(env_name, "")
+    urls = list(existing or [])
+    # Preserve legacy env-only behavior: duplicate env URLs injected twice before
+    # manifests existed. When a manifest seeds the list, dedupe appended env URLs
+    # so bundle manifests and explicit overrides do not double-load an asset.
+    dedupe = existing is not None
+    for item in raw.split(","):
+        if not _append_safe_asset_url(urls, item, env_name, dedupe=dedupe):
+            break
     return urls
 
 
+def _manifest_path(root: Path) -> Optional[Path]:
+    raw = os.getenv(_EXTENSION_MANIFEST_ENV, "").strip()
+    if not raw:
+        return None
+    if raw.startswith(("/", "~")):
+        _log.warning("Rejected extension manifest path from %s", _EXTENSION_MANIFEST_ENV)
+        return None
+    rel = _fully_unquote_path(raw)
+    if not _is_safe_relative_path(rel):
+        _log.warning("Rejected extension manifest path from %s", _EXTENSION_MANIFEST_ENV)
+        return None
+    manifest = (root / rel).resolve()
+    try:
+        manifest.relative_to(root)
+    except ValueError:
+        _log.warning("Rejected extension manifest path from %s", _EXTENSION_MANIFEST_ENV)
+        return None
+    return manifest
+
+
+def _manifest_asset_url(value: object) -> str:
+    """Normalize a manifest asset entry to the existing same-origin URL format."""
+    if not isinstance(value, str):
+        return ""
+    item = value.strip()
+    if not item:
+        return ""
+    parsed = urlsplit(item)
+    if parsed.scheme or parsed.netloc or item.startswith("//"):
+        return item
+    # Manifests are meant to make bundled local assets less noisy to list, so
+    # bare relative paths resolve under /extensions/. Absolute same-origin paths
+    # are still allowed and go through the same validator as env-configured URLs.
+    if item.startswith("/"):
+        return item
+    return EXTENSION_ROUTE_PREFIX + item
+
+
+def _iter_manifest_entries(manifest: object) -> List[Tuple[str, object]]:
+    entries: List[Tuple[str, object]] = []
+    extension_entries: object = []
+    if isinstance(manifest, dict):
+        entries.append(("manifest", manifest))
+        extension_entries = manifest.get("extensions", [])
+    elif isinstance(manifest, list):
+        extension_entries = manifest
+    if isinstance(extension_entries, list):
+        for index, extension in enumerate(extension_entries):
+            if isinstance(extension, dict) and extension.get("enabled", True) is False:
+                continue
+            entries.append((f"manifest.extensions[{index}]", extension))
+    return entries
+
+
+def _entry_asset_values(entry: Dict[str, object], key: str) -> List[object]:
+    values = entry.get(key, [])
+    return values if isinstance(values, list) else []
+
+
+def _read_manifest_text(manifest_file: Path) -> str:
+    with manifest_file.open("rb") as fh:
+        data = fh.read(_MAX_MANIFEST_BYTES + 1)
+    if len(data) > _MAX_MANIFEST_BYTES:
+        raise _ManifestTooLarge("manifest too large")
+    return data.decode("utf-8")
+
+
+def _read_manifest_urls(root: Path) -> Tuple[List[str], List[str]]:
+    manifest_file = _manifest_path(root)
+    if manifest_file is None:
+        return [], []
+    try:
+        if not manifest_file.exists() or not manifest_file.is_file():
+            _log.warning("Configured extension manifest was not found")
+            return [], []
+        manifest = json.loads(_read_manifest_text(manifest_file))
+    except _ManifestTooLarge:
+        _log.warning("Configured extension manifest exceeds %d bytes", _MAX_MANIFEST_BYTES)
+        return [], []
+    except json.JSONDecodeError:
+        _log.warning("Configured extension manifest is not valid JSON")
+        return [], []
+    except (OSError, UnicodeDecodeError):
+        _log.warning("Configured extension manifest could not be read")
+        return [], []
+
+    scripts: List[str] = []
+    stylesheets: List[str] = []
+    scripts_full = False
+    stylesheets_full = False
+    for _source, entry in _iter_manifest_entries(manifest):
+        if not isinstance(entry, dict):
+            continue
+        if not scripts_full:
+            for value in _entry_asset_values(entry, "scripts"):
+                if not _append_safe_asset_url(
+                    scripts, _manifest_asset_url(value), "manifest:scripts"
+                ):
+                    scripts_full = True
+                    break
+        if not stylesheets_full:
+            for value in _entry_asset_values(entry, "stylesheets"):
+                if not _append_safe_asset_url(
+                    stylesheets, _manifest_asset_url(value), "manifest:stylesheets"
+                ):
+                    stylesheets_full = True
+                    break
+        if scripts_full and stylesheets_full:
+            break
+    return scripts, stylesheets
+
+
 def get_extension_config() -> Dict[str, object]:
     """Return public extension config without exposing filesystem paths."""
-    enabled = _extension_root() is not None
-    if not enabled:
+    root = _extension_root()
+    if root is None:
         return {"enabled": False, "script_urls": [], "stylesheet_urls": []}
+    manifest_scripts, manifest_stylesheets = _read_manifest_urls(root)
     return {
         "enabled": True,
-        "script_urls": _read_url_list(_EXTENSION_SCRIPT_URLS_ENV),
-        "stylesheet_urls": _read_url_list(_EXTENSION_STYLESHEET_URLS_ENV),
+        "script_urls": _read_url_list(
+            _EXTENSION_SCRIPT_URLS_ENV, manifest_scripts or None
+        ),
+        "stylesheet_urls": _read_url_list(
+            _EXTENSION_STYLESHEET_URLS_ENV, manifest_stylesheets or None
+        ),
     }