AntoineVe/college-infos
1
0
Fork 0
Code Issues 11 Pull Requests 6 Actions Packages Projects Releases 3 Wiki Activity

fix(ical): fiabiliser le parsing Pronote 2026

Browse Source 
Revue Codex validée. Corrections de parsing iCal, conservation du contexte des devoirs et fixtures anonymisées.

Co-authored-by: Codex <codex@antoineve.me>
Co-committed-by: Codex <codex@antoineve.me>
This commit was merged in pull request #31.
This commit is contained in:
Codex
2026-09-12 14:54:55 +02:00
committed by Codex
parent f194ed985d
commit d45d38d365
9 changed files with 399 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pronote_sync/sources/pronote/ical.py
+175 -72
View File
@@ -14,13 +14,15 @@ from __future__ import annotations
import hashlib
import re
import unicodedata
import urllib.parse
from datetime import date, datetime
from html import unescape
from pathlib import Path
from typing import TypedDict
from typing import NamedTuple, TypedDict
import requests
from bs4 import BeautifulSoup
from icalendar import Calendar
from ...models.agenda import (
@@ -34,21 +36,13 @@ from ...models.homework import Homework
from ...utils.redaction import redact_exception, redact_url
from ...utils.uid import generate_deterministic_uid, normalize_pronote_uid
_HEADER_LABEL_PATTERN = re.compile(r"\b(Matière|Professeurs?|Salles?|Groupe)\s*:\s*")
_HEADER_LABEL_PATTERN = re.compile(
r"(?P<label>Mati(?:ère|ere)|Professeur(?:s|\(s\))?|Salle(?:s|\(s\))?"
r"|Groupe|Partie(?:s|\(s\))?\s+de\s+classe)\s*:\s*",
re.IGNORECASE,
)
_CALNAME_PATTERN = re.compile(r"^X-WR-CALNAME(?:;[^:]*)?:([^\r\n]*)", re.MULTILINE)
_TAG_PATTERN = re.compile(r"<[^>]+>")
_CONTENT_PATTERN = re.compile(
r"<strong>Contenu pédagogique\s*:\s*</strong>(.*?)(?=<strong>|</div>\s*$|\Z)",
re.DOTALL,
)
_DUE_PATTERN = re.compile(
r"<strong>Pour le (\d{2}/\d{2}/\d{4})\s*:\s*</strong>(.*?)(?=<strong>|</div>\s*$|\Z)",
re.DOTALL,
)
_ASSIGNED_PATTERN = re.compile(
r"<strong>Donné le (\d{2}/\d{2}/\d{4})\s*:\s*</strong>(.*?)(?=<strong>|</div>\s*$|\Z)",
re.DOTALL,
)
_STRONG_PATTERN = re.compile(r"<strong\b[^>]*>(?P<label>.*?)</strong>", re.IGNORECASE | re.DOTALL)
_HEADERS = {
"accept": "text/calendar",
@@ -63,6 +57,15 @@ class HeaderInfo(TypedDict):
teachers: list[str]
rooms: list[str]
group: str | None
class_part: str | None
class ParsedHomeworkBlock(NamedTuple):
"""Bloc de devoir parsé avec son texte nettoyé et son HTML sûr."""
date: date
text: str
html: str
def fetch_ical(url: str, timeout: int = 20) -> str:
@@ -156,11 +159,11 @@ def split_header_and_body(description: str) -> tuple[str, str]:
:return: Tuple ``(en-tête, corps)`` ; le corps est vide si aucun ``<strong>``.
:rtype: tuple[str, str]
"""
strong_start = description.find("<strong>")
if strong_start == -1:
strong_match = _STRONG_PATTERN.search(description)
if strong_match is None:
return description.strip(), ""
header = description[:strong_start].strip()
body = description[strong_start:]
header = description[: strong_match.start()].strip()
body = description[strong_match.start() :]
return header, body
@@ -168,7 +171,7 @@ def parse_header(header: str) -> HeaderInfo:
"""Parse l'en-tête texte pour extraire les métadonnées du cours.
Les labels reconnus sont : ``Matière :``, ``Professeur(s) :``,
``Salle(s) :`` et ``Groupe :``. La recherche se fait par position
``Salle(s) :``, ``Groupe :`` et ``Partie(s) de classe :``. La recherche se fait par position
des labels, ce qui supporte aussi bien un en-tête multi-lignes
qu'un en-tête dont les lignes sont jointes sur une seule ligne.
@@ -176,14 +179,20 @@ def parse_header(header: str) -> HeaderInfo:
:return: Dictionnaire typé avec les champs subject, teachers, rooms, group.
:rtype: HeaderInfo
"""
info: HeaderInfo = {"subject": "", "teachers": [], "rooms": [], "group": None}
info: HeaderInfo = {
"subject": "",
"teachers": [],
"rooms": [],
"group": None,
"class_part": None,
}
matches = list(_HEADER_LABEL_PATTERN.finditer(header))
for index, match in enumerate(matches):
value_start = match.end()
value_end = matches[index + 1].start() if index + 1 < len(matches) else len(header)
value = unescape(header[value_start:value_end].strip())
label = match.group(1).lower()
if label == "matière":
label = _normalize_label(match.group("label")).replace("(s)", "s")
if label == "matiere":
info["subject"] = value
elif label in ("professeur", "professeurs"):
info["teachers"] = [part.strip() for part in value.split(",") if part.strip()]
@@ -191,9 +200,44 @@ def parse_header(header: str) -> HeaderInfo:
info["rooms"] = [part.strip() for part in value.split(",") if part.strip()]
elif label == "groupe":
info["group"] = value
elif label in ("partie de classe", "parties de classe"):
info["class_part"] = value
return info
def _normalize_label(value: str) -> str:
"""Normalise un libellé iCal pour comparer des variantes contrôlées.
:param value: Libellé à normaliser.
:return: Libellé minuscule sans accents et avec des espaces unifiés.
:rtype: str
"""
decomposed = unicodedata.normalize("NFKD", value)
without_accents = "".join(char for char in decomposed if not unicodedata.combining(char))
return re.sub(r"\s+", " ", without_accents).strip().lower()
def _sanitize_html(fragment: str) -> str:
"""Nettoie un fragment HTML de description sans exécuter de contenu.
:param fragment: Fragment HTML extrait d'une section de devoir.
:return: HTML conservé sans scripts, styles ni attributs exécutables.
:rtype: str
"""
soup = BeautifulSoup(fragment, "html.parser")
for tag in soup.find_all(("script", "style")):
tag.decompose()
for tag in soup.find_all(True):
for attribute in list(tag.attrs):
lowered = attribute.lower()
value = tag.attrs[attribute]
if lowered.startswith("on") or (
lowered in ("href", "src") and str(value).lower().strip().startswith("javascript:")
):
del tag.attrs[attribute]
return soup.decode_contents().strip()
def _strip_html(text: str) -> str:
"""Retire les balises HTML d'un texte et nettoie les espaces.
@@ -201,8 +245,10 @@ def _strip_html(text: str) -> str:
:return: Texte brut sans balises, entités HTML décodées.
:rtype: str
"""
cleaned = _TAG_PATTERN.sub("", text)
return unescape(cleaned).strip()
soup = BeautifulSoup(text, "html.parser")
for tag in soup.find_all(("script", "style")):
tag.decompose()
return " ".join(unescape(soup.get_text(" ", strip=True)).split())
def _parse_french_date(value: str) -> date | None:
@@ -218,44 +264,57 @@ def _parse_french_date(value: str) -> date | None:
return None
def parse_body(body: str) -> tuple[str | None, list[tuple[date, str]], list[tuple[date, str]]]:
def parse_body(
body: str,
) -> tuple[str | None, list[ParsedHomeworkBlock], list[ParsedHomeworkBlock]]:
"""Parse le corps HTML pour extraire contenu pédagogique et devoirs.
Le contenu est extrait de la section ``<strong>Contenu pédagogique :</strong>``.
Les devoirs à faire sont extraits des sections ``<strong>Pour le JJ/MM/AAAA :</strong>``
(liste de tuples ``(date, texte)`` dans l'ordre du flux) et les devoirs donnés
des sections ``<strong>Donné le JJ/MM/AAAA :</strong>`` (liste de tuples
``(date, texte)``). Les listes préservent tous les blocs, même lorsque plusieurs
sections partagent la même date.
et les devoirs donnés des sections ``<strong>Donné le JJ/MM/AAAA :</strong>``.
Les listes préservent tous les blocs, même lorsque plusieurs sections partagent
la même date, avec le texte nettoyé et le HTML sûr de chaque bloc.
:param body: Corps HTML (à partir du premier ``<strong>``).
:return: Tuple ``(contenu pédagogique, devoirs dus, devoirs donnés)``.
:rtype: tuple[str | None, list[tuple[date, str]], list[tuple[date, str]]]
:rtype: tuple[str | None, list[ParsedHomeworkBlock], list[ParsedHomeworkBlock]]
"""
content: str | None = None
due_blocks: list[tuple[date, str]] = []
assigned_blocks: list[tuple[date, str]] = []
due_blocks: list[ParsedHomeworkBlock] = []
assigned_blocks: list[ParsedHomeworkBlock] = []
content_match = _CONTENT_PATTERN.search(body)
if content_match is not None:
content = _strip_html(content_match.group(1))
matches = list(_STRONG_PATTERN.finditer(body))
for index, match in enumerate(matches):
next_start = matches[index + 1].start() if index + 1 < len(matches) else len(body)
heading = _strip_html(match.group("label")).rstrip(":").strip()
fragment = body[match.end() : next_start]
safe_html = _sanitize_html(fragment)
text = _strip_html(fragment)
for match in _DUE_PATTERN.finditer(body):
due_date = _parse_french_date(match.group(1))
if due_date is not None:
due_blocks.append((due_date, _strip_html(match.group(2))))
normalized_heading = _normalize_label(heading)
if normalized_heading == "contenu pedagogique":
content = text
continue
for match in _ASSIGNED_PATTERN.finditer(body):
assigned_date = _parse_french_date(match.group(1))
if assigned_date is not None:
assigned_blocks.append((assigned_date, _strip_html(match.group(2))))
due_match = re.fullmatch(r"Pour\s+le\s+(\d{2}/\d{2}/\d{4})", heading, re.IGNORECASE)
assigned_match = re.fullmatch(
r"Donne\s+le\s+(\d{2}/\d{2}/\d{4})", normalized_heading, re.IGNORECASE
)
if due_match is not None:
due_date = _parse_french_date(due_match.group(1))
if due_date is not None:
due_blocks.append(ParsedHomeworkBlock(due_date, text, safe_html))
elif assigned_match is not None:
assigned_date = _parse_french_date(assigned_match.group(1))
if assigned_date is not None:
assigned_blocks.append(ParsedHomeworkBlock(assigned_date, text, safe_html))
return content, due_blocks, assigned_blocks
def parse_homework_blocks(
due_blocks: list[tuple[date, str]],
assigned_blocks: list[tuple[date, str]],
due_blocks: list[ParsedHomeworkBlock],
assigned_blocks: list[ParsedHomeworkBlock],
) -> tuple[HomeworkBlock, ...]:
"""Construit les :class:`HomeworkBlock` depuis les listes de devoirs.
@@ -269,10 +328,12 @@ def parse_homework_blocks(
:rtype: tuple[HomeworkBlock, ...]
"""
blocks: list[HomeworkBlock] = []
for due_date, text in due_blocks:
blocks.append(HomeworkBlock(kind="due", date=due_date, text=text, html=text))
for assigned_date, text in assigned_blocks:
blocks.append(HomeworkBlock(kind="assigned", date=assigned_date, text=text, html=text))
for block in due_blocks:
blocks.append(HomeworkBlock(kind="due", date=block.date, text=block.text, html=block.html))
for block in assigned_blocks:
blocks.append(
HomeworkBlock(kind="assigned", date=block.date, text=block.text, html=block.html)
)
return tuple(blocks)
@@ -286,24 +347,32 @@ def normalize_homework_text(text: str) -> str:
:return: Texte normalisé.
:rtype: str
"""
normalized = re.sub(r"\s+", " ", text)
normalized = _TAG_PATTERN.sub("", normalized)
return normalized.strip().lower()
return _strip_html(text).casefold()
def generate_homework_id(due_on: date, normalized_text: str) -> str:
def generate_homework_id(
due_on: date,
normalized_text: str,
subject: str = "",
teachers: tuple[str, ...] = (),
) -> str:
"""Génère un ID stable pour un devoir.
L'ID est la clé ``AAAA-MM-JJ|texte_normalisé`` hachée en SHA-1 dont
L'ID est la clé ``AAAA-MM-JJ|matière|enseignants|texte_normalisé`` hachée en SHA-1 dont
on garde les 12 premiers caractères hexadécimaux. Le hachage n'est
pas utilisé à des fins de sécurité (``usedforsecurity=False``).
:param due_on: Date d'échéance du devoir.
:param normalized_text: Texte normalisé du devoir.
:param subject: Matière du devoir, utile pour distinguer les homonymes.
:param teachers: Enseignants du devoir, triés pour garantir la stabilité.
:return: ID stable (12 caractères hexadécimaux).
:rtype: str
"""
payload = f"{due_on.isoformat()}|{normalized_text}".encode()
payload = (
f"{due_on.isoformat()}|{subject.casefold()}|"
f"{','.join(sorted(teacher.casefold() for teacher in teachers))}|{normalized_text}".encode()
)
return hashlib.sha1(payload, usedforsecurity=False).hexdigest()[:12]
@@ -313,7 +382,7 @@ def collect_homeworks(lessons: list[Lesson], target_date: date) -> list[Homework
Passe 1 : les blocs ``due`` (devoirs à faire pour ``target_date``)
de tous les cours. Passe 2 : les blocs ``assigned`` (devoirs donnés
le jour cible) des cours du jour ``target_date``. La déduplication
se fait par texte normalisé (premier venu, premier servi) et le
se fait par texte, matière et enseignants normalisés (premier venu, premier servi) et le
résultat est trié par matière puis texte.
:param lessons: Liste de tous les cours (VEVENT) parsés.
@@ -321,15 +390,22 @@ def collect_homeworks(lessons: list[Lesson], target_date: date) -> list[Homework
:return: Liste unique de devoirs, triée par matière puis texte.
:rtype: list[Homework]
"""
by_text: dict[str, Homework] = {}
by_context: dict[tuple[str, tuple[str, ...], str], Homework] = {}
for lesson in lessons:
for block in lesson.homework_blocks:
if block.kind == "due" and block.date == target_date:
key = normalize_homework_text(block.text)
if key not in by_text:
by_text[key] = Homework(
id=generate_homework_id(target_date, key),
normalized_text = normalize_homework_text(block.text)
key = (
lesson.subject.casefold(),
tuple(sorted(teacher.casefold() for teacher in lesson.teachers)),
normalized_text,
)
if key not in by_context:
by_context[key] = Homework(
id=generate_homework_id(
target_date, normalized_text, lesson.subject, lesson.teachers
),
subject=lesson.subject,
teachers=lesson.teachers,
assigned_on=lesson.start.date(),
@@ -343,10 +419,17 @@ def collect_homeworks(lessons: list[Lesson], target_date: date) -> list[Homework
continue
for block in lesson.homework_blocks:
if block.kind == "assigned":
key = normalize_homework_text(block.text)
if key not in by_text:
by_text[key] = Homework(
id=generate_homework_id(target_date, key),
normalized_text = normalize_homework_text(block.text)
key = (
lesson.subject.casefold(),
tuple(sorted(teacher.casefold() for teacher in lesson.teachers)),
normalized_text,
)
if key not in by_context:
by_context[key] = Homework(
id=generate_homework_id(
target_date, normalized_text, lesson.subject, lesson.teachers
),
subject=lesson.subject,
teachers=lesson.teachers,
assigned_on=block.date,
@@ -355,7 +438,7 @@ def collect_homeworks(lessons: list[Lesson], target_date: date) -> list[Homework
html=block.html,
)
return sorted(by_text.values(), key=lambda hw: (hw.subject.lower(), hw.text.lower()))
return sorted(by_context.values(), key=lambda hw: (hw.subject.lower(), hw.text.lower()))
def parse_ical(raw_ical: str) -> tuple[list[Lesson], list[Homework], list[SchoolEvent]]:
@@ -399,15 +482,28 @@ def parse_ical(raw_ical: str) -> tuple[list[Lesson], list[Homework], list[School
else:
categories = [str(category) for category in categories_obj.cats]
# Événements de type vacances/congés (tout le jour).
if any(cat in ("Congés", "Vacances") for cat in categories):
normalized_categories = [_normalize_label(category) for category in categories]
holiday_kind: SchoolEventKind | None = None
if any(
category in ("conges", "vacances", "vacances scolaires")
for category in normalized_categories
):
holiday_kind = SchoolEventKind.HOLIDAY
elif any(
category in ("jour ferie", "jours feries", "ferie", "feries")
for category in normalized_categories
):
holiday_kind = SchoolEventKind.PUBLIC_HOLIDAY
# Événements de type vacances/congés/jours fériés (tout le jour).
if holiday_kind is not None:
from_date = start.date() if isinstance(start, datetime) else start
to_date = end.date() if isinstance(end, datetime) else end
summary = component.get("summary")
label = str(summary) if summary is not None else ""
school_events.append(
SchoolEvent(
kind=SchoolEventKind.HOLIDAY,
kind=holiday_kind,
label=label,
from_date=from_date,
to_date=to_date,
@@ -422,9 +518,15 @@ def parse_ical(raw_ical: str) -> tuple[list[Lesson], list[Homework], list[School
status_obj = component.get("status")
status_value = str(status_obj).strip().upper() if status_obj is not None else ""
if status_value == "CANCELLED" or "Cours - Cours annulé" in categories:
normalized_status_categories = set(normalized_categories)
if status_value == "CANCELLED" or any(
"annul" in category for category in normalized_status_categories
):
status = LessonStatus.CANCELLED
elif "Cours - Cours déplacé" in categories:
elif any(
any(token in category for token in ("deplac", "changement de salle", "modifi"))
for category in normalized_status_categories
):
status = LessonStatus.MOVED
else:
status = LessonStatus.NORMAL
@@ -458,6 +560,7 @@ def parse_ical(raw_ical: str) -> tuple[list[Lesson], list[Homework], list[School
teachers=tuple(lesson_data["teachers"]),
rooms=tuple(lesson_data["rooms"]),
group=lesson_data["group"],
class_part=lesson_data["class_part"],
status=status,
content=content,
homework_blocks=homework_blocks,