From f05d4a9b8b56a04bf1181424dd096624eae6ec3a Mon Sep 17 00:00:00 2001 From: Joan Code <172996447+joan-code6@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:43:57 +0200 Subject: [PATCH] feat: add native Dateiverteilung API --- README.md | 2 + api/api.py | 50 +++ api/auth_db.py | 1 + api/documentation.py | 1 + docs/API.md | 11 + .../applets/dateiverteilung/__init__.py | 1 + .../applets/dateiverteilung/api.py | 370 ++++++++++++++++++ schulportal_hessen/base.py | 18 + tests/fixtures/dateiverteilung_overview.html | 27 ++ tests/test_dateiverteilung.py | 210 ++++++++++ 10 files changed, 691 insertions(+) create mode 100644 schulportal_hessen/applets/dateiverteilung/__init__.py create mode 100644 schulportal_hessen/applets/dateiverteilung/api.py create mode 100644 tests/fixtures/dateiverteilung_overview.html create mode 100644 tests/test_dateiverteilung.py diff --git a/README.md b/README.md index e3938d0..86934a1 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ The portal modules are referred to as applets because SPH is built on top of Moo - `stundenplan` — timetable data - `lerngruppen` — study groups - `oberstufenwahl` — student-facing elections and course choices +- `dateispeicher` — school file storage and downloads +- `dateiverteilung` — targeted notices, personal files, and authenticated downloads - `school_list` — school names and IDs for login and school selection diff --git a/api/api.py b/api/api.py index 067378d..f7c1f16 100644 --- a/api/api.py +++ b/api/api.py @@ -340,6 +340,7 @@ def validate_module_preferences(cls, value): "dashboard", "messages", "dateispeicher", + "dateiverteilung", "vertretungsplan", "dsb", "courses", @@ -1995,6 +1996,55 @@ async def download_dateispeicher_file( ) +# --- Dateiverteilung --- + + +@app.get("/dateiverteilung") +async def get_dateiverteilung( + refresh: bool = False, + auth: AuthSession = Depends(client_dependency), +) -> Dict[str, object]: + if not refresh: + cached = await sessions.get_cached(auth.user_id, "/dateiverteilung") + if cached is not None: + return cached + + result = await run_in_threadpool(auth.client.dateiverteilung_get_overview) + if result.get("success"): + await sessions.set_cache(auth.user_id, "/dateiverteilung", result) + elif result.get("error_kind") == "authentication": + await sessions.invalidate_schulportal_client(auth.user_id, auth.client) + return result + + +@app.get("/dateiverteilung/file") +async def download_dateiverteilung_file( + url: str = Query(..., min_length=1, max_length=2048), + auth: AuthSession = Depends(client_dependency), +): + result = await run_in_threadpool(auth.client.dateiverteilung_download_file, url) + stream = result.get("stream") + if not result.get("success") or stream is None: + error_kind = result.get("error_kind") + if error_kind == "validation": + error_status = status.HTTP_400_BAD_REQUEST + elif error_kind == "authentication": + await sessions.invalidate_schulportal_client(auth.user_id, auth.client) + error_status = status.HTTP_401_UNAUTHORIZED + elif result.get("upstream_status") == status.HTTP_404_NOT_FOUND: + error_status = status.HTTP_404_NOT_FOUND + else: + error_status = status.HTTP_502_BAD_GATEWAY + raise HTTPException(status_code=error_status, detail=result.get("error", "File not found")) + + filename = re.sub(r'[\r\n"]', "_", str(result.get("filename") or "dateiverteilung-datei")) + return StreamingResponse( + content=stream, + media_type=result.get("content_type") or "application/octet-stream", + headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename, safe='')}"}, + ) + + # --- Lerngruppen --- diff --git a/api/auth_db.py b/api/auth_db.py index dfed14c..a4ef3a5 100644 --- a/api/auth_db.py +++ b/api/auth_db.py @@ -96,6 +96,7 @@ def _pairing_code_hash(code: str) -> str: "dashboard", "messages", "dateispeicher", + "dateiverteilung", "vertretungsplan", "dsb", "courses", diff --git a/api/documentation.py b/api/documentation.py index 4011e7b..f52b52c 100644 --- a/api/documentation.py +++ b/api/documentation.py @@ -33,6 +33,7 @@ ("/vertretungsplan", "Plans"), ("/stundenplan", "Plans"), ("/dateispeicher", "File Storage"), + ("/dateiverteilung", "File Distribution"), ("/lerngruppen", "Study Groups"), ("/school-list", "School List"), ("/benutzer", "User Info"), diff --git a/docs/API.md b/docs/API.md index 5cadbfd..0d945c9 100644 --- a/docs/API.md +++ b/docs/API.md @@ -327,6 +327,17 @@ Fetch files and folders for a specific dateispeicher node. Search files in the dateispeicher by name. +#### dateiverteilung_get_overview + +Fetch targeted file distributions for the authenticated user. Returns each +distribution's title, provenance, date, unread state, files, and related links. + +#### dateiverteilung_download_file + +Stream a file from `dateiverteilung.php` through the authenticated portal +session. Download URLs are restricted to the configured Schulportal origin, +the Dateiverteilung endpoint, and known download actions. + #### lerngruppen_get_overview Fetch study groups and exam data (lerngruppen.php). diff --git a/schulportal_hessen/applets/dateiverteilung/__init__.py b/schulportal_hessen/applets/dateiverteilung/__init__.py new file mode 100644 index 0000000..3270bb6 --- /dev/null +++ b/schulportal_hessen/applets/dateiverteilung/__init__.py @@ -0,0 +1 @@ +"""Dateiverteilung (targeted file distribution) applet.""" diff --git a/schulportal_hessen/applets/dateiverteilung/api.py b/schulportal_hessen/applets/dateiverteilung/api.py new file mode 100644 index 0000000..666ff10 --- /dev/null +++ b/schulportal_hessen/applets/dateiverteilung/api.py @@ -0,0 +1,370 @@ +from __future__ import annotations + +import re +from collections.abc import Iterator +from typing import Any +from urllib.parse import parse_qs, unquote, urljoin, urlparse + +import requests +from bs4 import BeautifulSoup, Tag + +_DOWNLOAD_ACTIONS = {"download"} + + +def _clean_text(node: Any) -> str: + return " ".join(node.get_text(" ", strip=True).split()) if node else "" + + +def _absolute_portal_url(base_url: str, value: str) -> str: + """Return a safe URL to this applet, or an empty string. + + File links are supplied by upstream HTML and later accepted through a public + API query parameter. Treat both sources as untrusted and pin them to the + configured Schulportal origin and the Dateiverteilung path. + """ + value = (value or "").strip() + if not value or value.startswith(("//", "\\")): + return "" + candidate = urljoin(f"{base_url.rstrip('/')}/", value) + expected = urlparse(base_url) + parsed = urlparse(candidate) + if ( + parsed.scheme != expected.scheme + or parsed.netloc != expected.netloc + or parsed.username is not None + or parsed.password is not None + or parsed.fragment + or parsed.path.rstrip("/") != "/dateiverteilung.php" + ): + return "" + query = parse_qs(parsed.query, keep_blank_values=True) + action = str((query.get("a") or query.get("action") or [""])[0]).lower() + if action not in _DOWNLOAD_ACTIONS: + return "" + return candidate + + +def _file_id(url: str, fallback: str) -> str: + query = parse_qs(urlparse(url).query) + for key in ("f", "file", "id", "d"): + value = str((query.get(key) or [""])[0]).strip() + if value: + return value + return fallback + + +def _parse_file(link: Tag, base_url: str, index: int) -> dict[str, Any] | None: + url = _absolute_portal_url(base_url, str(link.get("href") or "")) + if not url: + return None + name = str(link.get("download") or "").strip() or _clean_text(link) + if not name: + name = f"Datei {index + 1}" + parent_text = _clean_text(link.parent) + size_match = re.search( + r"(?:\(|\b)(\d+(?:[.,]\d+)?\s*(?:B|KB|MB|GB))(?:\)|\b)", + parent_text, + re.IGNORECASE, + ) + return { + "id": _file_id(url, str(index + 1)), + "name": name, + "size": size_match.group(1) if size_match else "", + "download_url": url, + } + + +def _container_for(link: Tag) -> Tag: + return ( + link.find_parent(["article", "section"]) + or link.find_parent( + class_=re.compile(r"(?:panel|card|distribution|verteilung)", re.IGNORECASE) + ) + or link.find_parent("tr") + or link.parent + ) + + +def _parse_distribution( + container: Tag, files: list[dict[str, Any]], index: int, base_url: str +) -> dict[str, Any]: + heading = container.select_one( + "h1, h2, h3, h4, h5, h6, .panel-title, .card-title, .title, strong" + ) + title = _clean_text(heading) or f"Verteilung {index + 1}" + + description_node = container.select_one( + ".description, .beschreibung, .hinweis, .markup, .card-text, .panel-body p" + ) + description = _clean_text(description_node) + if description == title: + description = "" + + source_node = container.select_one( + ".course, .kurs, .sender, .source, .herkunft, [data-course]" + ) + source = _clean_text(source_node) or str(container.get("data-course") or "").strip() + + text = _clean_text(container) + date_match = re.search( + r"\b(\d{1,2}[.]\d{1,2}[.]\d{2,4}(?:\s+(?:um\s+)?\d{1,2}:\d{2})?)\b", text + ) + distribution_id = str(container.get("data-id") or container.get("id") or "").strip() + if not distribution_id: + for file in files: + query = parse_qs(urlparse(file["download_url"]).query) + distribution_id = str( + ( + query.get("v") + or query.get("distribution") + or query.get("id") + or [""] + )[0] + ).strip() + if distribution_id: + break + + external_links: list[dict[str, str]] = [] + for link in container.select("a[href]"): + href = str(link.get("href") or "").strip() + absolute = urljoin(f"{base_url.rstrip('/')}/", href) + if _absolute_portal_url(base_url, href) or not absolute.startswith( + ("http://", "https://") + ): + continue + label = _clean_text(link) or absolute + if not any(item["url"] == absolute for item in external_links): + external_links.append({"label": label, "url": absolute}) + + classes = " ".join(container.get("class") or []) + unread = bool( + container.select_one(".badge, .label-new, .neu, [data-new='1']") + or re.search(r"(?:^|\s)(?:new|unread|neu)(?:\s|$)", classes, re.IGNORECASE) + ) + return { + "id": distribution_id or f"distribution-{index + 1}", + "title": title, + "description": description, + "source": source or "Schulportal", + "created_at": date_match.group(1) if date_match else "", + "unread": unread, + "files": files, + "links": external_links, + } + + +def parse_dateiverteilung_html(html: str, base_url: str) -> list[dict[str, Any]]: + """Parse recipient distributions across current and legacy portal markup.""" + soup = BeautifulSoup(html, "html.parser") + grouped: dict[int, tuple[Tag, list[dict[str, Any]]]] = {} + seen_urls: set[str] = set() + + for link in soup.select("a[href]"): + parsed = _parse_file(link, base_url, len(seen_urls)) + if not parsed or parsed["download_url"] in seen_urls: + continue + seen_urls.add(parsed["download_url"]) + container = _container_for(link) + key = id(container) + if key not in grouped: + grouped[key] = (container, []) + grouped[key][1].append(parsed) + + # Text/link-only distributions have no download anchors. Prefer explicit + # semantic containers so unrelated portal chrome is never returned. + explicit = soup.select( + "article, section[data-id], .distribution, .dateiverteilung, [data-distribution]" + ) + for container in explicit: + if id(container) not in grouped and _clean_text(container): + grouped[id(container)] = (container, []) + + return [ + _parse_distribution(container, files, index, base_url) + for index, (container, files) in enumerate(grouped.values()) + ] + + +def _looks_like_login(response: Any) -> bool: + content_type = str( + (getattr(response, "headers", {}) or {}).get("Content-Type") or "" + ).lower() + prefix = str(getattr(response, "text", "") or "")[:8192].lower() + response_host = ( + urlparse(str(getattr(response, "url", "") or "")).hostname or "" + ).lower() + looks_html = "text/html" in content_type or bool( + re.match(r"\s*<(?:!doctype\s+html|html|head|body|form)\b", prefix) + ) + return "login.schulportal" in response_host or ( + looks_html + and ( + "login.schulportal" in prefix + or ( + "
", + ) + + assert _looks_like_login(response) is True + + +class FakeResponse: + status_code = 200 + + def __init__(self): + self.closed = False + self.headers = { + "Content-Disposition": "attachment; filename*=UTF-8''Elternbrief%20Anna.pdf", + "Content-Type": "application/pdf", + } + + def raise_for_status(self): + return None + + def iter_content(self, chunk_size=8192): + yield b"personal document" + + def close(self): + self.closed = True + + +class FakeSession: + def __init__(self): + self.response = FakeResponse() + self.calls = [] + + def get(self, *args, **kwargs): + self.calls.append((args, kwargs)) + return self.response + + +def test_download_streams_through_authenticated_session(): + client = SimpleNamespace( + logged_in=True, BASE_START_URL=BASE_URL, session=FakeSession() + ) + url = "/dateiverteilung.php?a=download&v=73&f=brief-anna.pdf" + + result = dateiverteilung_download_file(client, url) + + assert result["success"] is True + assert result["filename"] == "Elternbrief Anna.pdf" + assert b"".join(result["stream"]) == b"personal document" + assert client.session.response.closed is True + assert client.session.calls[0][1]["allow_redirects"] is False + + +def test_routes_are_published(): + routes = { + (route.path, method) + for route in api_module.app.routes + for method in getattr(route, "methods", set()) + } + assert ("/dateiverteilung", "GET") in routes + assert ("/dateiverteilung/file", "GET") in routes + + +def test_download_route_maps_validation_errors(monkeypatch): + auth = AuthSession( + client=SimpleNamespace(dateiverteilung_download_file=lambda _url: None), + user_id="user-a", + school_id="school", + username="user", + ) + + async def run_in_threadpool(_func, _url): + return {"success": False, "error": "unsafe", "error_kind": "validation"} + + monkeypatch.setattr(api_module, "run_in_threadpool", run_in_threadpool) + with pytest.raises(HTTPException) as error: + asyncio.run( + api_module.download_dateiverteilung_file( + "https://evil.example/file", auth=auth + ) + ) + assert error.value.status_code == 400 + + +def test_refresh_bypasses_overview_cache(monkeypatch): + client = SimpleNamespace( + dateiverteilung_get_overview=lambda: { + "success": True, + "distributions": [], + "distribution_count": 0, + "file_count": 0, + "unread_count": 0, + } + ) + auth = AuthSession( + client=client, user_id="user-a", school_id="school", username="user" + ) + + class FakeSessions: + def __init__(self): + self.cached = [] + + async def get_cached(self, *_args, **_kwargs): + raise AssertionError("refresh must bypass the response cache") + + async def set_cache(self, *args): + self.cached.append(args) + + fake_sessions = FakeSessions() + monkeypatch.setattr(api_module, "sessions", fake_sessions) + + result = asyncio.run(api_module.get_dateiverteilung(refresh=True, auth=auth)) + + assert result["success"] is True + assert len(fake_sessions.cached) == 1 + + +def test_overview_cache_is_scoped_to_authenticated_user(monkeypatch): + auth = AuthSession( + client=SimpleNamespace(dateiverteilung_get_overview=lambda: None), + user_id="school:user-a", + school_id="school", + username="user-a", + ) + + class FakeSessions: + def __init__(self): + self.lookups = [] + + async def get_cached(self, *args): + self.lookups.append(args) + return {"success": True, "distributions": [{"id": "only-user-a"}]} + + fake_sessions = FakeSessions() + monkeypatch.setattr(api_module, "sessions", fake_sessions) + + result = asyncio.run(api_module.get_dateiverteilung(auth=auth)) + + assert result["distributions"][0]["id"] == "only-user-a" + assert fake_sessions.lookups == [("school:user-a", "/dateiverteilung")]