1from collections import deque
2from dataclasses import MISSING, dataclass, field, fields, replace
3from enum import Enum
4from pathlib import Path
5from typing import Any, Literal, TypedDict, cast
6
7from jsonschema import ValidationError, validate
8from sphinx.application import Sphinx
9from sphinx.config import Config as _SphinxConfig
10
11from sphinx_codelinks.source_discover.config import (
12 CommentType,
13 SourceDiscoverConfig,
14 SourceDiscoverSectionConfigType,
15)
16from sphinx_codelinks.source_discover.source_discover import SourceDiscover
17from ub_project import anchor, load_toml, select_table
18
19UNIX_NEWLINE = "\n"
20
21
22COMMENT_MARKERS = {
[docs] 23 # @Support C and C++ style comments, IMPL_C_1, impl, [FE_C_SUPPORT, FE_CPP]
24 CommentType.cpp: ["//", "/*"],
[docs] 25 # @Support Python style comments, IMPL_PY_1, impl, [FE_PY]
26 CommentType.python: ["#"],
27 CommentType.cs: ["//", "/*", "///"],
[docs] 28 # @Support Go style comments, IMPL_GO_2, impl, [FE_GO]
29 CommentType.go: ["//", "/*"],
30}
31ESCAPE = "\\"
32
33# Default C/C++ standard for the standalone/defines parse path when no
34# compile_commands.json entry supplies one.
35DEFAULT_CPP_STD = "c++17"
36
37
38class CommentCategory(str, Enum): # noqa: UP042 # StrEnum changes str(member), which reaches CLI warnings and error messages
39 comment = "comment"
40 docstring = "expression_statement"
41
42
43class NeedIdRefsConfigType(TypedDict):
44 markers: list[str]
45
46
47@dataclass
48class NeedIdRefsConfig:
49 @classmethod
50 def field_names(cls) -> set[str]:
51 return {item.name for item in fields(cls)}
52
53 markers: list[str] = field(
54 default_factory=lambda: ["@need-ids:"],
55 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
56 )
57 """The markers to extract need ids from"""
58
59 @classmethod
60 def get_schema(cls, name: str) -> dict[str, Any] | None:
61 _field = next(_field for _field in fields(cls) if _field.name is name)
62 if _field.metadata and "schema" in _field.metadata:
63 return cast(dict[str, Any], _field.metadata["schema"])
64 return None
65
66 def check_schema(self) -> list[str]:
67 errors = []
68 for _field_name in self.field_names():
69 schema = self.get_schema(_field_name)
70 if schema is None:
71 # no schema to check against; `validate` would raise TypeError
72 continue
73 value = getattr(self, _field_name)
74 try:
75 validate(
76 instance=value,
77 schema=schema,
78 )
79 except ValidationError as e:
80 errors.append(
81 f"Schema validation error in field '{_field_name}': {e.message}"
82 )
83 return errors
84
85
86class MultilineNeedsConfigType(TypedDict, total=False):
87 start_sequence: str
88 end_sequence: str
89 default_markup: str
90 markups: dict[str, str]
91
92
93DEFAULT_MARKUPS: dict[str, str] = {"rst": ".rst", "md": ".md"}
94"""The default ``markups`` table: markup tag -> the ``doctype`` suffix of the need."""
95
96
97@dataclass
98class MultilineNeedsConfig:
99 """The markers and markups of multi-line needs (``[analyse.multiline_needs]``)."""
100
101 @classmethod
102 def field_names(cls) -> set[str]:
103 return {item.name for item in fields(cls)}
104
105 start_sequence: str = field(
106 default="@need", metadata={"schema": {"type": "string", "minLength": 1}}
107 )
108 """The word that opens a multi-line need, at the start of its line."""
109 end_sequence: str = field(
110 default="@endneed", metadata={"schema": {"type": "string", "minLength": 1}}
111 )
112 """The word that closes a multi-line need, alone on its line."""
113 default_markup: str = field(
114 default="rst", metadata={"schema": {"type": "string", "minLength": 1}}
115 )
116 """The markup tag of a block whose open line names none; a key of ``markups``."""
117 markups: dict[str, str] = field(
118 default_factory=lambda: dict(DEFAULT_MARKUPS),
119 metadata={
120 "schema": {
121 "type": "object",
122 "additionalProperties": {"type": "string", "minLength": 1},
123 "minProperties": 1,
124 }
125 },
126 )
127 """Markup tag -> the ``doctype`` suffix the need's content is parsed with. Not
128 checked against any parser: the consumer decides what it can parse."""
129
130 @classmethod
131 def get_schema(cls, name: str) -> dict[str, Any] | None:
132 _field = next(_field for _field in fields(cls) if _field.name is name)
133 if _field.metadata and "schema" in _field.metadata:
134 return cast(dict[str, Any], _field.metadata["schema"])
135 return None
136
137 def check_schema(self) -> list[str]:
138 errors = []
139 for _field_name in self.field_names():
140 schema = self.get_schema(_field_name)
141 if schema is None:
142 # no schema to check against; `validate` would raise TypeError
143 continue
144 value = getattr(self, _field_name)
145 try:
146 validate(
147 instance=value,
148 schema=schema,
149 )
150 except ValidationError as e:
151 errors.append(
152 f"Schema validation error in field '{_field_name}': {e.message}"
153 )
154 return errors
155
156 def check_fields_configuration(self) -> list[str]:
157 errors = self.check_schema()
158 if self.start_sequence == self.end_sequence:
159 errors.append("start_sequence and end_sequence cannot be the same.")
160 if isinstance(self.markups, dict) and self.default_markup not in self.markups:
161 errors.append(
162 f"default_markup {self.default_markup!r} is not a key of markups "
163 f"({', '.join(repr(tag) for tag in sorted(self.markups))})."
164 )
165 return errors
166
167
168@dataclass
169class PreprocessorConfig:
170 """Opt-in libclang engine config. Presence => libclang engine for C/C++."""
171
172 compile_commands: Path | None = field(
173 default=None, metadata={"schema": {"type": ["string", "null"]}}
174 )
175 """Explicit path to compile_commands.json. If None, walk-up auto-discovery."""
176
177 defines: list[str] = field(
178 default_factory=list,
179 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
180 )
181 """Fallback -D defines applied globally when no compile_commands.json applies."""
182
183 includes: list[Path] = field(
184 default_factory=list,
185 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
186 )
187 """Fallback -I include dirs for the defines path."""
188
189 std: str = field(
190 default=DEFAULT_CPP_STD,
191 metadata={"schema": {"type": "string"}},
192 )
193 """C/C++ standard for the standalone/defines parse path (e.g. ``c++17``,
194 ``c++20``, ``c11``); libclang pins ``-x`` to match it. Files resolved from a
195 ``compile_commands.json`` entry use that entry's own ``-std`` instead."""
196
197
198def anchor_preproc_paths(preproc: PreprocessorConfig, base: Path) -> PreprocessorConfig:
199 """Resolve a preprocessor config's ``compile_commands`` and ``includes``
200 against ``base`` (the config file's directory), so a relative path resolves
201 against the TOML file rather than the process CWD — matching ``src_dir`` /
202 ``git_root``. An absolute path is not anchored, but it is resolved too
203 (symlinks followed, ``..`` folded), like every other path here.
204 """
205 return replace(
206 preproc,
207 compile_commands=(
208 anchor(preproc.compile_commands, base).resolve()
209 if preproc.compile_commands is not None
210 else None
211 ),
212 includes=[anchor(inc, base).resolve() for inc in preproc.includes],
213 )
214
215
216class FieldConfig(TypedDict, total=False):
217 name: str
218 type: Literal["str", "list[str]"]
219 default: str | list[str] | None
220
221
222class OneLineCommentStyleType(TypedDict):
223 start_sequence: str
224 end_sequence: str
225 field_split_char: str
226 needs_fields: list[FieldConfig]
227
228
229@dataclass
230class OneLineCommentStyle:
231 def __setattr__(self, name: str, value: Any) -> None:
232 if name == "needs_fields":
233 # apply default to fields
234 self.apply_needs_field_default(value)
235 return super().__setattr__(name, value)
236
237 @classmethod
238 def field_names(cls) -> set[str]:
239 return {item.name for item in fields(cls)}
240
241 start_sequence: str = field(default="@", metadata={"schema": {"type": "string"}})
242 """Chars sequence to indicate the start of the one-line comment."""
243
244 end_sequence: str = field(
245 default=UNIX_NEWLINE, metadata={"schema": {"type": "string"}}
246 )
247 """Chars sequence to indicate the end of the one-line comment."""
248
249 field_split_char: str = field(default=",", metadata={"schema": {"type": "string"}})
250 """Char sequence to split the fields."""
251
252 needs_fields: list[FieldConfig] = field(
253 default_factory=lambda: [
254 {"name": "title"},
255 {"name": "id"},
256 {"name": "type", "default": "impl"},
257 {"name": "links", "type": "list[str]", "default": []},
258 ],
259 metadata={
260 "required_fields": ["title", "type"],
261 "field_default": {
262 "type": "str",
263 },
264 "schema": {
265 "type": "array",
266 "items": {
267 "type": "object",
268 "properties": {
269 "name": {"type": "string"},
270 "type": {
271 "type": "string",
272 "enum": ["str", "list[str]"],
273 "default": "str",
274 },
275 "default": {
276 "anyOf": [
277 {"type": "string"},
278 {"type": "array", "items": {"type": "string"}},
279 ]
280 },
281 },
282 "required": ["name"],
283 "additionalProperties": False,
284 "allOf": [
285 {
286 "if": {"properties": {"type": {"const": "list[str]"}}},
287 "then": {
288 "properties": {
289 "default": {
290 "type": "array",
291 "items": {"type": "string"},
292 }
293 }
294 },
295 },
296 {
297 "if": {"properties": {"type": {"const": "str"}}},
298 "then": {"properties": {"default": {"type": "string"}}},
299 },
300 ],
301 },
302 },
303 },
304 )
305
306 @classmethod
307 def apply_needs_field_default(cls, given_fields: list[FieldConfig]) -> None:
308 field_default = next(
309 _field.metadata["field_default"]
310 for _field in fields(cls)
311 if _field.name == "needs_fields"
312 )
313
314 for _field in given_fields:
315 for _default in field_default:
316 if _default not in _field:
317 _field[_default] = field_default[
318 _default
319 ] # dynamically assign keys
320
321 @classmethod
322 def get_required_fields(cls, name: str) -> list[str] | None:
323 _field = next(_field for _field in fields(cls) if _field.name is name)
324 if _field.metadata:
325 return cast(list[str], _field.metadata["required_fields"])
326 return None
327
328 @classmethod
329 def get_schema(cls, name: str) -> dict[str, Any] | None:
330 _field = next(_field for _field in fields(cls) if _field.name is name)
331 if _field.metadata and "schema" in _field.metadata:
332 return cast(dict[str, Any], _field.metadata["schema"])
333 return None
334
335 def check_schema(self) -> list[str]:
336 errors = []
337 for _field_name in self.field_names():
338 schema = self.get_schema(_field_name)
339 if schema is None:
340 # no schema to check against; `validate` would raise TypeError
341 continue
342 value = getattr(self, _field_name)
343 try:
344 validate(
345 instance=value,
346 schema=schema,
347 ) # validate has no type specified
348 except ValidationError as e:
349 if _field_name == "needs_fields":
350 need_field_name = value[e.path[0]]["name"]
351 errors.append(
352 f"Schema validation error in need_fields '{need_field_name}': {e.message}"
353 )
354 else:
355 errors.append(
356 f"Schema validation error in field '{_field_name}': {e.message}"
357 )
358 return errors
359
360 def check_required_fields(self) -> list[str]:
361 errors = []
362 required_fields = self.get_required_fields("needs_fields")
363 if required_fields is None:
364 errors.append("No required fields specified.")
365 return errors
366 given_field_names = [_field["name"] for _field in self.needs_fields]
367 missing_fields = set(required_fields) - set(given_field_names)
368 if len(missing_fields) != 0:
369 errors.append(f"Missing required fields: {sorted(missing_fields)}")
370
371 return errors
372
373 def check_fields_mutually_exclusive(self) -> list[str]:
374 errors = []
375 needs_field_names = set()
376 for _field in self.needs_fields:
377 if _field["name"] in needs_field_names:
378 errors.append(f"Field '{_field['name']}' is defined multiple times.")
379 needs_field_names.add(_field["name"])
380 return errors
381
382 def check_fields_default_order(self) -> list[str]:
383 errors = []
384 seen_default = False
385 first_default_field = ""
386 for _field in self.needs_fields:
387 has_default = _field.get("default") is not None
388 if has_default and not seen_default:
389 seen_default = True
390 first_default_field = _field["name"]
391 elif not has_default and seen_default:
392 errors.append(
393 f"Field '{_field['name']}' without a default follows "
394 f"field '{first_default_field}' which has a default. "
395 f"Fields without defaults must be defined before fields with defaults."
396 )
397 return errors
398
399 def check_fields_configuration(self) -> list[str]:
400 return (
401 self.check_schema()
402 + self.check_required_fields()
403 + self.check_fields_mutually_exclusive()
404 + self.check_fields_default_order()
405 )
406
407 def get_cnt_required_fields(self) -> int:
408 cnt_required_fields = 0
409 for _field in self.needs_fields:
410 if _field.get("default") is None:
411 cnt_required_fields += 1
412 return cnt_required_fields
413
414 def get_pos_list_str(self) -> list[int]:
415 pos_list_str = []
416 for idx, _field in enumerate(self.needs_fields):
417 if _field["type"] == "list[str]":
418 pos_list_str.append(idx + 1)
419 return pos_list_str
420
421
422class AnalyseSectionConfigType(TypedDict, total=False):
423 """Define typing for loading `analyse` section from the file."""
424
425 get_need_id_refs: bool
426 get_oneline_needs: bool
427 get_multiline_needs: bool
428 outdir: str
429 git_root: str
430 need_id_refs: NeedIdRefsConfigType
431 multiline_needs: MultilineNeedsConfigType
432 oneline_comment_style: OneLineCommentStyleType
433 preprocessor: dict[str, object]
434
435
436class SourceAnalyseConfigType(TypedDict, total=False):
437 """Define typing for its API configuration."""
438
439 src_files: list[Path]
440 src_dir: Path
441 comment_type: CommentType
442 get_need_id_refs: bool
443 get_oneline_needs: bool
444 get_multiline_needs: bool
445 git_root: Path | None
446 need_id_refs_config: NeedIdRefsConfig
447 multiline_needs_config: MultilineNeedsConfig
448 oneline_comment_style: OneLineCommentStyle
449 preprocessor: PreprocessorConfig | None
450
451
452class ProjectsAnalyseConfigType(TypedDict, total=False):
453 projects_config: dict[str, SourceAnalyseConfigType]
454
455
456@dataclass
457class SourceAnalyseConfig:
458 @classmethod
459 def field_names(cls) -> set[str]:
460 return {item.name for item in fields(cls)}
461
462 src_files: list[Path] = field(
463 default_factory=list,
464 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
465 )
466 """A list of source files to be processed."""
467 src_dir: Path = field(
468 default_factory=lambda: Path("./"), metadata={"schema": {"type": "string"}}
469 )
470
471 comment_type: CommentType = field(
472 default=CommentType.cpp, metadata={"schema": {"type": "string"}}
473 )
474 """The type of comment to be processed."""
475
476 get_need_id_refs: bool = field(
477 default=True, metadata={"schema": {"type": "boolean"}}
478 )
479 """Whether to extract need id references from comments"""
480
481 get_oneline_needs: bool = field(
482 default=False, metadata={"schema": {"type": "boolean"}}
483 )
484 """Whether to extract oneline needs from comments"""
485
486 get_multiline_needs: bool = field(
487 default=False, metadata={"schema": {"type": "boolean"}}
488 )
489 """Whether to extract multi-line needs (``@need`` … ``@endneed``) from comments"""
490
491 git_root: Path | None = field(
492 default=None, metadata={"schema": {"type": ["string", "null"]}}
493 )
494 """Explicit path to the Git repository root. If not set, it will be auto-detected
495 by traversing parent directories. Useful for Bazel builds or deeply nested configs."""
496
497 need_id_refs_config: NeedIdRefsConfig = field(default_factory=NeedIdRefsConfig)
498 """Configuration for extracting need id references from comments."""
499
500 multiline_needs_config: MultilineNeedsConfig = field(
501 default_factory=MultilineNeedsConfig
502 )
503 """Configuration for extracting multi-line needs from comments."""
504
505 oneline_comment_style: OneLineCommentStyle = field(
506 default_factory=OneLineCommentStyle
507 )
508 """Configuration for extracting oneline needs from comments."""
509
510 preprocessor: PreprocessorConfig | None = field(default=None)
511 """Opt-in libclang preprocessor engine. None => tree-sitter (default).
512
513 No flat ``metadata["schema"]`` here: this is a nested dataclass, like the
514 sibling ``need_id_refs_config`` / ``multiline_needs_config`` /
515 ``oneline_comment_style`` fields. ``check_schema`` only validates fields that
516 declare a flat schema; giving this field one made it validate the constructed
517 ``PreprocessorConfig`` instance against JSON type ``object`` and fail at
518 ``config-inited``. Its structure is enforced by ``convert_analyse_config``.
519 """
520
521 @classmethod
522 def get_schema(cls, name: str) -> dict[str, Any] | None:
523 _field = next(_field for _field in fields(cls) if _field.name is name)
524 if _field.metadata and "schema" in _field.metadata:
525 return cast(dict[str, Any], _field.metadata["schema"])
526 return None
527
528 def check_schema(self) -> list[str]:
529 errors = []
530 for _field_name in self.field_names():
531 schema = self.get_schema(_field_name)
532 if not schema:
533 continue
534 value = getattr(self, _field_name)
535 if isinstance(value, Path): # adapt to json schema restriction
536 value = str(value)
537 if _field_name == "src_files" and isinstance(
538 value, list
539 ): # adapt to json schema restriction
540 value: list[str] = [
541 str(src_file) for src_file in value
542 ] # only for value adaptation
543 try:
544 validate(instance=value, schema=schema)
545 except ValidationError as e:
546 errors.append(
547 f"Schema validation error in field '{_field_name}': {e.message}"
548 )
549 return errors
550
551 def check_markers_mutually_exclusive(self) -> list[str]:
552 errors = set()
553 markers = set()
554 markers.add(self.oneline_comment_style.start_sequence)
555 markers.add(self.oneline_comment_style.end_sequence)
556 # equality only: the default one-line start ``@`` is a PREFIX of ``@need``, and
557 # what keeps the two apart is that a block's lines are hidden from the one-line
558 # parser, not this check. The two words are checked only when the feature is
559 # on: off, they are no markers of the project.
560 multiline_markers = (
561 (
562 self.multiline_needs_config.start_sequence,
563 self.multiline_needs_config.end_sequence,
564 )
565 if self.get_multiline_needs
566 else ()
567 )
568 for marker in multiline_markers:
569 if marker in markers:
570 errors.add(f"Marker {marker} is defined multiple times")
571 else:
572 markers.add(marker)
573
574 for marker in self.need_id_refs_config.markers:
575 if marker in markers:
576 errors.add(f"Marker {marker} is defined multiple times")
577 else:
578 markers.add(marker)
579 return list(errors)
580
581 def check_fields_configuration(self) -> list[str]:
582 errors: deque[str] = deque()
583 if self.get_need_id_refs:
584 need_id_refs_errors = self.need_id_refs_config.check_schema()
585 if need_id_refs_errors:
586 errors.appendleft("NeedIdRefs configuration errors:")
587 errors.extend(need_id_refs_errors)
588 if self.get_oneline_needs:
589 oneline_needs_errors = (
590 self.oneline_comment_style.check_fields_configuration()
591 )
592 if oneline_needs_errors:
593 errors.appendleft("OneLineCommentStyle configuration errors:")
594 errors.extend(oneline_needs_errors)
595 if self.get_multiline_needs:
596 multiline_errors = self.multiline_needs_config.check_fields_configuration()
597 if multiline_errors:
598 errors.appendleft("MultilineNeeds configuration errors:")
599 errors.extend(multiline_errors)
600 analyse_errors = self.check_markers_mutually_exclusive() + self.check_schema()
601 if analyse_errors:
602 errors.appendleft("analyse configuration errors:")
603 errors.extend(analyse_errors)
604 return list(errors)
605
606
607# Shared ubCode project file, which other useblocks tools (sphinx-needs,
608# ubCode checker, ...) read as well, so all tools see the same projects.
609DEFAULT_CONFIG_TOML: str = "ubproject.toml"
610
611DEFAULT_REF_URL_FIELD: str = "code_url"
612"""The need field ``@need-ids:`` references are attached to, unless a project's
613``ref_url_field`` names another (ubCode's key and default; ``""`` disables the attach)."""
614
615#: The table of the TOML file that both readers, the Sphinx extension and the CLI,
616#: take their configuration from.
617CODELINKS_TABLE: str = "codelinks"
618
619
620def load_codelinks_table(path: Path) -> dict[str, object] | None:
621 """Parse *path* and return its ``[codelinks]`` table, through ub-project.
622
623 The values are returned RAW: relative paths are anchored where they are used
624 (the Sphinx extension anchors at ``confdir / Path(config_from_toml).parent``,
625 unresolved), never here.
626
627 :param path: The TOML file.
628 :return: The table, or ``None`` when the file has no ``codelinks`` key.
629 :raises ub_project.ProjectConfigError: If the file cannot be read, is not UTF-8
630 or not valid TOML, or if ``codelinks`` is not a table.
631 :raises RecursionError: For a pathologically nested file, which ``load_toml``
632 does not wrap -- why both callers also catch ``Exception``.
633 """
634 return select_table(load_toml(path), CODELINKS_TABLE, source=path)
635
636
637class CodeLinksProjectConfigType(TypedDict, total=False):
638 """TypedDict defining the configuration structure for individual SrcTrace projects.
639
640 Contains both user-provided configuration:
641 - source_discover
642 - remote_url_pattern
643 - ref_url_field
644 - analyse
645 and runtime-generated configuration objects
646 - source_discover_config
647 - analyse_config
648 """
649
650 source_discover: SourceDiscoverSectionConfigType
651 remote_url_pattern: str
652 ref_url_field: str
653 analyse: AnalyseSectionConfigType
654 source_discover_config: SourceDiscoverConfig
655 analyse_config: SourceAnalyseConfig
656
657
658class CodeLinksConfigType(TypedDict):
659 config_from_toml: str | None
660 set_local_url: bool
661 local_url_field: str
662 set_remote_url: bool
663 remote_url_field: str
664 outdir: Path
665 projects: dict[str, CodeLinksProjectConfigType]
666 debug_measurement: bool
667 debug_filters: bool
668
669
670@dataclass
671class CodeLinksConfig:
672 @classmethod
673 def from_sphinx(cls, sphinx_config: _SphinxConfig) -> "CodeLinksConfig":
674 obj = cls()
675 src_trace_projects = getattr(sphinx_config, "src_trace_projects", None)
676 if isinstance(src_trace_projects, dict):
677 generate_project_configs(src_trace_projects)
678 super().__setattr__(obj, "_sphinx_config", sphinx_config)
679 return obj
680
681 def __getattribute__(self, name: str) -> Any:
682 if name.startswith("__") or name == "_sphinx_config":
683 return super().__getattribute__(name)
684 sphinx_config = (
685 object.__getattribute__(self, "_sphinx_config")
686 if "_sphinx_config" in self.__dict__
687 else None
688 )
689 if sphinx_config:
690 return getattr(
691 super().__getattribute__("_sphinx_config"), f"src_trace_{name}"
692 )
693
694 return object.__getattribute__(self, name)
695
696 def __setattr__(self, name: str, value: Any) -> None:
697 if name == "_sphinx_config" and "src_trace_projects" in value:
698 src_trace_projects: dict[str, CodeLinksProjectConfigType] = value[
699 "src_trace_projects"
700 ]
701 generate_project_configs(src_trace_projects)
702
703 if name.startswith("__") or name == "_sphinx_config":
704 return super().__setattr__(name, value)
705
706 sphinx_config = (
707 object.__getattribute__(self, "_sphinx_config")
708 if "_sphinx_config" in self.__dict__
709 else None
710 )
711
712 if sphinx_config:
713 setattr(
714 super().__getattribute__("_sphinx_config"), f"src_trace_{name}", value
715 )
716
717 if name == "outdir" and isinstance(value, str):
718 # Ensure outdir is a Path object
719 value = Path(value)
720 return object.__setattr__(self, name, value)
721
722 @classmethod
723 def add_config_values(cls, app: Sphinx) -> None:
724 """Add all config values to Sphinx application"""
725 for item in fields(cls):
726 if item.default_factory is not MISSING:
727 default = item.default_factory()
728 elif item.default is not MISSING:
729 default = item.default
730 else:
731 raise Exception(f"Field {item.name} has no default value or factory")
732
733 name = item.name
734 app.add_config_value(
735 f"src_trace_{name}",
736 default,
737 item.metadata["rebuild"],
738 types=item.metadata["types"],
739 )
740
741 @classmethod
742 def field_names(cls) -> set[str]:
743 return {item.name for item in fields(cls)}
744
745 @classmethod
746 def get_schema(cls, name: str) -> dict[str, Any] | None:
747 """Get the schema for a config item."""
748 _field = next(field for field in fields(cls) if field.name is name)
749 if _field.metadata and "schema" in _field.metadata:
750 return _field.metadata["schema"]
751 return None
752
753 config_from_toml: str | None = field(
754 default=DEFAULT_CONFIG_TOML,
755 metadata={
756 "rebuild": "env",
757 "types": (str, type(None)),
758 "schema": {
759 "type": ["string", "null"],
760 "examples": [DEFAULT_CONFIG_TOML, None],
761 },
762 },
763 )
764 """Path to a TOML file to load configuration from.
765
766 Defaults to ``ubproject.toml`` next to :file:`conf.py`. The default is the
767 value ``ubproject.toml`` exactly -- left unset, or written in conf.py as that
768 string: missing, or without a ``[codelinks]`` table, it is silently ignored.
769 Any other value, ``./ubproject.toml`` included, is an explicit file and warns
770 in both cases. A file that exists but cannot be read or parsed warns
771 (``codelinks.config``) either way, as any configured file did at 1.4.0.
772 """
773
774 set_local_url: bool = field(
775 default=False,
776 metadata={
777 "rebuild": "env",
778 "types": (bool,),
779 "schema": {
780 "type": "boolean",
781 },
782 },
783 )
784 """Set the file URL in the extracted need."""
785
786 local_url_field: str = field(
787 default="local-url",
788 metadata={
789 "rebuild": "env",
790 "types": (str,),
791 "schema": {
792 "type": "string",
793 },
794 },
795 )
796 """The field name for the file URL in the extracted need."""
797
798 set_remote_url: bool = field(
799 default=False,
800 metadata={
801 "rebuild": "env",
802 "types": (bool,),
803 "schema": {
804 "type": "boolean",
805 },
806 },
807 )
808 remote_url_field: str = field(
809 default="remote-url",
810 metadata={
811 "rebuild": "env",
812 "types": (str,),
813 "schema": {
814 "type": "string",
815 },
816 },
817 )
818 """The field name for the remote URL in the extracted need."""
819
820 outdir: Path = field(
821 default=Path("output"),
822 metadata={"rebuild": "env", "types": (str), "schema": {"type": "string"}},
823 )
824 """The directory where the generated artifacts and their caches will be stored."""
825
826 projects: dict[str, CodeLinksProjectConfigType] = field(
827 default_factory=dict,
828 metadata={
829 "rebuild": "env",
830 "types": (),
831 "schema": {
832 "type": "object",
833 "additionalProperties": {
834 "type": "object",
835 "properties": {
836 "source_discover": {},
837 "analyse": {},
838 "remote_url_pattern": {},
839 "ref_url_field": {},
840 "source_discover_config": {},
841 "analyse_config": {},
842 },
843 "additionalProperties": False,
844 },
845 },
846 },
847 )
848 """The configuration for the source tracing projects."""
849
850 debug_measurement: bool = field(
851 default=False, metadata={"rebuild": "html", "types": (bool,)}
852 )
853 """If True, log runtime information for various functions."""
854 debug_filters: bool = field(
855 default=False, metadata={"rebuild": "html", "types": (bool,)}
856 )
857 """If True, log filter processing runtime information."""
858
859
860def check_schema(config: CodeLinksConfig) -> list[str]:
861 """Check only first layer's of schema, so that the nested dict is not validated here."""
862 errors = []
863 for _field_name in CodeLinksConfig.field_names():
864 schema = CodeLinksConfig.get_schema(_field_name)
865 if not schema:
866 continue
867 value = getattr(config, _field_name)
868 if isinstance(value, Path): # adapt to json schema restriction
869 value = str(value)
870 try:
871 validate(instance=value, schema=schema)
872 except ValidationError as e:
873 errors.append(
874 f"Schema validation error in filed '{_field_name}': {e.message}"
875 )
876 return errors
877
878
879def check_project_configuration(config: CodeLinksConfig) -> list[str]:
880 """Check nested project configurations"""
881 errors = []
882
883 for project_name, project_config in config.projects.items():
884 project_errors: list[str] = []
885
886 # validate source_discover config
887 src_discover_config: SourceDiscoverConfig | None = project_config.get(
888 "source_discover_config"
889 )
890 src_discover_errors = []
891 if src_discover_config:
892 src_discover_errors.extend(src_discover_config.check_schema())
893
894 # validate analyse config
895 analyse_config: SourceAnalyseConfig | None = project_config.get(
896 "analyse_config"
897 )
898 analyse_errors = []
899 if analyse_config:
900 analyse_errors = analyse_config.check_fields_configuration()
901
902 # validate src-trace config
903 if config.set_remote_url and "remote_url_pattern" not in project_config:
904 project_errors.append(
905 "remote_url_pattern must be given, as set_remote_url is enabled"
906 )
907
908 if "remote_url_pattern" in project_config and not isinstance(
909 project_config["remote_url_pattern"], str
910 ):
911 project_errors.append("remote_url_pattern must be a string")
912
913 ref_url_field = project_config.get("ref_url_field", DEFAULT_REF_URL_FIELD)
914 if not isinstance(ref_url_field, str):
915 project_errors.append("ref_url_field must be a string")
916 elif ref_url_field and ref_url_field in _url_fields(config):
917 project_errors.append(
918 f"ref_url_field {ref_url_field!r} must differ from local_url_field "
919 "and remote_url_field"
920 )
921
922 if analyse_errors or src_discover_errors or project_errors:
923 errors.append(f"Project '{project_name}' has the following errors:")
924 errors.extend(analyse_errors)
925 errors.extend(src_discover_errors)
926 errors.extend(project_errors)
927
928 return errors
929
930
931def config_base_dir(confdir: str | Path, config: CodeLinksConfig) -> Path:
932 """The directory a project's relative paths are anchored at.
933
934 The configuration file's directory when ``src_trace_config_from_toml`` names one
935 (itself anchored at ``confdir``), else ``confdir``: ``src_dir``, ``git_root`` and
936 the preprocessor paths all resolve against it.
937 """
938 base = Path(confdir)
939 if config.config_from_toml:
940 base = anchor(Path(config.config_from_toml).parent, base)
941 return base
942
943
944def locate_src_dir(
945 confdir: str | Path,
946 config: CodeLinksConfig,
947 discover_config: SourceDiscoverConfig,
948) -> Path:
949 """A project's source directory, anchored (see :func:`config_base_dir`) and resolved."""
950 return anchor(discover_config.src_dir, config_base_dir(confdir, config)).resolve()
951
952
953def git_root_problem(git_root: Path, src_dir: Path) -> str | None:
954 """Why a configured ``git_root`` cannot be a project's git root, or ``None``.
955
956 Every source path is relative to the git root, so it must be ``src_dir`` or a
957 directory above it (#2062). Both paths anchored and resolved by the caller: the
958 Sphinx extension (``project_analysis.configured_git_root``) and ``codelinks
959 analyse`` alike, each treating a value with a problem as unset.
960 """
961 try:
962 if not git_root.exists():
963 return f"git_root {git_root.as_posix()} does not exist"
964 if not git_root.is_dir():
965 return f"git_root {git_root.as_posix()} is not a directory"
966 except OSError:
967 return f"git_root {git_root.as_posix()} cannot be read"
968 if not src_dir.is_relative_to(git_root):
969 return (
970 f"git_root {git_root.as_posix()} does not contain src_dir "
971 f"{src_dir.as_posix()}"
972 )
973 return None
974
975
976def git_root_warning(project: str, problem: str) -> str:
977 """The one warning text for a project's ignored ``git_root`` (``codelinks.git_root``)."""
978 return (
979 f"project {project!r}: {problem}; it is ignored, and the repository root is "
980 "detected from src_dir instead"
981 )
982
983
984def remote_url_pattern_warnings(config: CodeLinksConfig) -> list[str]:
985 """Why a project's remote URL pattern will not render as one link, if it will not.
986
987 The ``remote-url`` value is the pattern filled in, and Sphinx-Needs splits every
988 string-linked value on ``,`` and ``;`` before turning the parts into links. A
989 pattern containing either -- gitweb's ``?p=repo.git;a=blob;f={path}`` is the
990 classic -- therefore renders as several links, none of them right.
991 """
992 if not config.set_remote_url or not isinstance(config.projects, dict):
993 return []
994 warnings = []
995 for name, project_config in config.projects.items():
996 if not isinstance(project_config, dict):
997 continue
998 pattern = project_config.get("remote_url_pattern")
999 if isinstance(pattern, str) and ("," in pattern or ";" in pattern):
1000 warnings.append(
1001 f"Project {name!r}: remote_url_pattern {pattern!r} contains ',' or "
1002 "';'. Sphinx-Needs splits string-linked values on ',' and ';', so this "
1003 "pattern's links will not render as one link."
1004 )
1005 return warnings
1006
1007
1008def _url_fields(config: CodeLinksConfig) -> set[str]:
1009 """The names of the URL fields the extension registers (when switched on)."""
1010 names: set[str] = set()
1011 if config.set_local_url:
1012 names.add(config.local_url_field)
1013 if config.set_remote_url:
1014 names.add(config.remote_url_field)
1015 return names
1016
1017
1018def need_id_refs_field(
1019 config: CodeLinksConfig, project_config: CodeLinksProjectConfigType
1020) -> str | None:
1021 """The field a project's ``@need-ids:`` references are attached to, or ``None``.
1022
1023 ubCode's gate: URLs are on (local or remote), references are extracted, the project
1024 has at least one marker, and its ``ref_url_field`` is not ``""``. A field that is not
1025 a string, or that is one of the URL fields, is a configuration error reported by
1026 :func:`check_project_configuration`, and attaches nothing.
1027 """
1028 if not (config.set_remote_url or config.set_local_url):
1029 return None
1030 field_name = project_config.get("ref_url_field", DEFAULT_REF_URL_FIELD)
1031 if not isinstance(field_name, str) or not field_name:
1032 return None
1033 if field_name in _url_fields(config):
1034 return None
1035 analyse_config = project_config.get("analyse_config")
1036 if analyse_config is None or not analyse_config.get_need_id_refs:
1037 return None
1038 if not analyse_config.need_id_refs_config.markers:
1039 return None
1040 return field_name
1041
1042
1043def need_id_refs_fields(config: CodeLinksConfig) -> dict[str, str]:
1044 """Each project whose references are attached, mapped to its field."""
1045 projects = config.projects
1046 if not isinstance(projects, dict):
1047 return {}
1048 fields_by_project: dict[str, str] = {}
1049 for name, project_config in projects.items():
1050 if not isinstance(project_config, dict):
1051 continue
1052 field_name = need_id_refs_field(config, project_config)
1053 if field_name is not None:
1054 fields_by_project[name] = field_name
1055 return fields_by_project
1056
1057
1058def check_configuration(config: CodeLinksConfig) -> list[str]:
1059 errors = []
1060 errors.extend(check_schema(config))
1061 errors.extend(check_project_configuration(config))
1062 return errors
1063
1064
1065def convert_src_discovery_config(
1066 config_dict: SourceDiscoverSectionConfigType | None,
1067) -> SourceDiscoverConfig:
1068 if config_dict:
1069 src_discover_dict = {
1070 key: (Path(value) if key == "src_dir" and isinstance(value, str) else value)
1071 for key, value in config_dict.items()
1072 }
1073 src_discover_config = SourceDiscoverConfig(**src_discover_dict) # ty: ignore[invalid-argument-type]
1074 else:
1075 src_discover_config = SourceDiscoverConfig()
1076
1077 return src_discover_config
1078
1079
1080def _validate_preprocessor_dict(preproc: dict[str, object]) -> None:
1081 """Validate the schema-less ``[preprocessor]`` TOML section.
1082
1083 The section has no ``TypedDict``, so a mistyped scalar would otherwise be
1084 coerced into garbage instead of reported: e.g. ``defines = "X"`` (a bare
1085 string) becomes ``list("X") == ["X"]`` — or worse, ``defines = "cpp17"``
1086 becomes ``["c", "p", "p", "1", "7"]`` → five bogus ``-D`` flags. Fail loud.
1087
1088 :param preproc: the raw ``[preprocessor]`` mapping from TOML.
1089 :raises TypeError: if a key has the wrong type.
1090 """
1091 for key in ("defines", "includes"):
1092 value = preproc.get(key)
1093 if value is not None and (
1094 not isinstance(value, list) or not all(isinstance(x, str) for x in value)
1095 ):
1096 raise TypeError(
1097 f"[preprocessor] {key} must be a list of strings, "
1098 f"got {type(value).__name__}: {value!r}"
1099 )
1100 for key in ("compile_commands", "std"):
1101 value = preproc.get(key)
1102 if value is not None and not isinstance(value, str):
1103 raise TypeError(
1104 f"[preprocessor] {key} must be a string, "
1105 f"got {type(value).__name__}: {value!r}"
1106 )
1107
1108
1109REMOVED_ANALYSE_KEYS: dict[str, str] = {
1110 "get_rst": "get_multiline_needs",
1111 "marked_rst": "[analyse.multiline_needs]",
1112}
1113"""``[analyse]`` keys of the removed marked-rst blocks -> what replaces each."""
1114
1115
1116def check_removed_analyse_keys(config_dict: AnalyseSectionConfigType) -> None:
1117 """Refuse the keys of the removed ``@rst`` blocks with one message naming the cure.
1118
1119 Any other unknown key keeps failing as before, in the dataclass constructor. The one
1120 exception: ``get_rst = false`` alone asks for nothing the blocks did, and
1121 configuration scaffolds wrote it into ``ubproject.toml`` files ubCode reads too, so
1122 it passes here silently -- this runs on every configuration read -- and the build
1123 and the CLI warn once per project (:func:`get_rst_ignored_warning`).
1124
1125 :raises TypeError: through the channel every other configuration error of the
1126 section takes (``typer.BadParameter`` in the CLI, the ``config-inited`` error of
1127 a Sphinx build).
1128 """
1129 found = [key for key in REMOVED_ANALYSE_KEYS if key in config_dict]
1130 if not found:
1131 return
1132 removed = " and ".join(
1133 f"'{key}'" if key == "get_rst" else f"'[analyse.{key}]'" for key in found
1134 )
1135 replacements = " and ".join(REMOVED_ANALYSE_KEYS[key] for key in found)
1136 if _tolerated_get_rst(config_dict):
1137 return
1138 raise TypeError(
1139 f"analyse: {removed} {'is' if len(found) == 1 else 'are'} no longer supported: "
1140 "the marked-rst blocks were replaced by multi-line needs (the @need and "
1141 f"@endneed markers); use {replacements} instead"
1142 )
1143
1144
1145GET_RST_IGNORED = (
1146 "analyse: 'get_rst' is no longer read: the marked-rst blocks were replaced by "
1147 "multi-line needs (the @need and @endneed markers); get_rst = false asks for "
1148 "nothing and can be removed"
1149)
1150"""The warning for a tolerated ``get_rst = false``."""
1151
1152
1153def _tolerated_get_rst(config_dict: Any) -> bool:
1154 """Whether an ``[analyse]`` section's only removed key is ``get_rst = false``."""
1155 return (
1156 isinstance(config_dict, dict)
1157 and config_dict.get("get_rst") is False
1158 and not any(
1159 key in config_dict for key in REMOVED_ANALYSE_KEYS if key != "get_rst"
1160 )
1161 )
1162
1163
1164def get_rst_ignored_warning(name: str, project_config: Any) -> str | None:
1165 """The warning for a project whose ``[analyse]`` section sets ``get_rst = false``
1166 (tolerated, and dropped, by :func:`check_removed_analyse_keys`), else ``None``.
1167
1168 It names the project, as :func:`remote_url_pattern_warnings` does; the rest is
1169 :data:`GET_RST_IGNORED`. Called once per project where a configuration is loaded --
1170 a Sphinx build's ``config-inited``, the CLI before it analyses -- never per read
1171 of it.
1172 """
1173 if isinstance(project_config, dict) and _tolerated_get_rst(
1174 project_config.get("analyse")
1175 ):
1176 return f"Project {name!r}: {GET_RST_IGNORED}"
1177 return None
1178
1179
1180def convert_analyse_config(
1181 config_dict: AnalyseSectionConfigType | None,
1182 src_discover: SourceDiscover | None = None,
1183) -> SourceAnalyseConfig:
1184 analyse_config_dict: SourceAnalyseConfigType = {}
1185 if config_dict:
1186 check_removed_analyse_keys(config_dict)
1187 for k, v in config_dict.items():
1188 if k not in {
1189 "oneline_comment_style",
1190 "need_id_refs",
1191 "multiline_needs",
1192 "preprocessor",
1193 # tolerated (``check_removed_analyse_keys``) and ignored
1194 "get_rst",
1195 }:
1196 # Convert string paths to Path objects
1197 if k in {"src_dir", "git_root"} and isinstance(v, str):
1198 analyse_config_dict[k] = Path(v)
1199 else:
1200 # dynamical assignment
1201 analyse_config_dict[k] = v # ty: ignore[invalid-key]
1202
1203 # Get oneline_comment_style configuration
1204 oneline_comment_style_dict: OneLineCommentStyleType | None = config_dict.get(
1205 "oneline_comment_style"
1206 )
1207 oneline_comment_style: OneLineCommentStyle = (
1208 convert_oneline_comment_style_config(oneline_comment_style_dict)
1209 )
1210
1211 # Get need_id_refs configuration
1212 need_id_refs_config_dict: NeedIdRefsConfigType | None = config_dict.get(
1213 "need_id_refs"
1214 )
1215 need_id_refs_config = convert_need_id_refs_config(need_id_refs_config_dict)
1216
1217 # Get multiline_needs configuration
1218 multiline_needs_config_dict: MultilineNeedsConfigType | None = config_dict.get(
1219 "multiline_needs"
1220 )
1221 multiline_needs_config = convert_multiline_needs_config(
1222 multiline_needs_config_dict
1223 )
1224
1225 analyse_config_dict["need_id_refs_config"] = need_id_refs_config
1226 analyse_config_dict["multiline_needs_config"] = multiline_needs_config
1227 analyse_config_dict["oneline_comment_style"] = oneline_comment_style
1228
1229 preprocessor_dict = config_dict.get("preprocessor")
1230 if preprocessor_dict is not None:
1231 # The preprocessor section has no TypedDict; its values are dynamic
1232 # TOML (typed ``object``), so validate the shapes up front -- a mistyped
1233 # scalar would otherwise coerce into garbage flags.
1234 _validate_preprocessor_dict(preprocessor_dict)
1235 analyse_config_dict["preprocessor"] = PreprocessorConfig(
1236 compile_commands=(
1237 Path(str(preprocessor_dict["compile_commands"]))
1238 if preprocessor_dict.get("compile_commands")
1239 else None
1240 ),
1241 defines=list(preprocessor_dict.get("defines", [])), # ty: ignore[invalid-argument-type]
1242 includes=[Path(str(p)) for p in preprocessor_dict.get("includes", [])], # ty: ignore[not-iterable]
1243 std=str(preprocessor_dict.get("std", DEFAULT_CPP_STD)),
1244 )
1245
1246 if src_discover:
1247 analyse_config_dict["src_files"] = src_discover.source_paths
1248 analyse_config_dict["src_dir"] = src_discover.src_discover_config.src_dir
1249 try:
1250 analyse_config_dict["comment_type"] = CommentType(
1251 src_discover.src_discover_config.comment_type
1252 )
1253 except ValueError:
1254 # If invalid comment_type, keep the string value
1255 # Validation will catch this error later
1256 comment_type_str: str = src_discover.src_discover_config.comment_type
1257 analyse_config_dict["comment_type"] = comment_type_str # ty: ignore[invalid-assignment]
1258
1259 return SourceAnalyseConfig(**analyse_config_dict)
1260
1261
1262def convert_oneline_comment_style_config(
1263 config_dict: OneLineCommentStyleType | None,
1264) -> OneLineCommentStyle:
1265 if config_dict is None:
1266 oneline_comment_style = OneLineCommentStyle()
1267 else:
1268 try:
1269 oneline_comment_style = OneLineCommentStyle(**config_dict)
1270 except TypeError as e:
1271 raise TypeError(f"Invalid oneline comment style configuration: {e}") from e
1272 return oneline_comment_style
1273
1274
1275def convert_need_id_refs_config(
1276 config_dict: NeedIdRefsConfigType | None,
1277) -> NeedIdRefsConfig:
1278 if not config_dict:
1279 need_id_refs_config = NeedIdRefsConfig()
1280 else:
1281 try:
1282 need_id_refs_config = NeedIdRefsConfig(**config_dict)
1283 except TypeError as e:
1284 raise TypeError(f"Invalid oneline comment style configuration: {e}") from e
1285 return need_id_refs_config
1286
1287
1288def convert_multiline_needs_config(
1289 config_dict: MultilineNeedsConfigType | None,
1290) -> MultilineNeedsConfig:
1291 if not config_dict:
1292 return MultilineNeedsConfig()
1293 try:
1294 return MultilineNeedsConfig(**config_dict)
1295 except TypeError as e:
1296 raise TypeError(f"Invalid multiline_needs configuration: {e}") from e
1297
1298
1299def generate_project_configs(
1300 project_configs: dict[str, CodeLinksProjectConfigType],
1301) -> None:
1302 """Generate configs of source discover and analyse from their classes dynamically."""
1303 for project_config in project_configs.values():
1304 # overwrite the config into different types on purpose
1305 # covert dicts to their own classes
1306 src_discover_section: SourceDiscoverSectionConfigType | None = cast(
1307 SourceDiscoverSectionConfigType,
1308 project_config.get("source_discover"),
1309 )
1310 source_discover_config = convert_src_discovery_config(src_discover_section)
1311 project_config["source_discover_config"] = source_discover_config
1312
1313 analyse_section_config: AnalyseSectionConfigType | None = cast(
1314 AnalyseSectionConfigType, project_config.get("analyse")
1315 )
1316 analyse_config = convert_analyse_config(analyse_section_config)
1317 analyse_config.get_oneline_needs = True # force to get oneline_need
1318 # Copy comment_type from source_discover_config to analyse_config
1319 try:
1320 analyse_config.comment_type = CommentType(
1321 source_discover_config.comment_type
1322 )
1323 except ValueError:
1324 # If invalid comment_type, keep the string value
1325 # Validation will catch this error later
1326 analyse_config.comment_type = source_discover_config.comment_type # ty: ignore[invalid-assignment]
1327 project_config["analyse_config"] = analyse_config
Back to top