1import re
2from dataclasses import dataclass
3
4from sphinx_codelinks.analyse.models import WarningSubTypeEnum
5from sphinx_codelinks.config import ESCAPE, UNIX_NEWLINE, OneLineCommentStyle
6
7# Tags of Python docstring conventions (Epydoc, Doxygen, Sphinx field names)
8# that a one-line start sequence such as ``@`` also matches: ``@param a: the
9# first, thing`` would otherwise become a need with the id ``thing``.
10PYTHON_DOCSTRING_TAGS = frozenset(
11 {
12 "arg",
13 "argument",
14 "attention",
15 "author",
16 "brief",
17 "bug",
18 "cvar",
19 "deprecated",
20 "details",
21 "except",
22 "exception",
23 "invariant",
24 "ivar",
25 "key",
26 "keyword",
27 "kwarg",
28 "kwparam",
29 "note",
30 "param",
31 "parameter",
32 "post",
33 "pre",
34 "raise",
35 "raises",
36 "retval",
37 "return",
38 "returns",
39 "rtype",
40 "see",
41 "since",
42 "summary",
43 "throw",
44 "throws",
45 "todo",
46 "tparam",
47 "type",
48 "var",
49 "vartype",
50 "version",
51 "warning",
52 "yield",
53 "yields",
54 "ytype",
55 }
56)
57
58
59#: What follows the tag of an Epydoc field: ``:`` directly (``@return: text``), or
60#: spaces or tabs, one word of characters other than space, tab and ``:``, and ``:``,
61#: as in ``@param a: text``. Locale-free: ubCode spells the same.
62_FIELD_SHAPE = re.compile(r":|[ \t]+[^ \t:]+:")
63
64
65def docstring_tag(line: str, start_sequence: str) -> str | None:
66 """The docstring tag a docstring line starts with, if any.
67
68 The line starts with one when, after comment decoration and whitespace (no
69 alphanumeric character), it holds ``start_sequence`` directly followed by a
70 tag of :data:`PYTHON_DOCSTRING_TAGS` in the Epydoc field shape: ``:``
71 directly, or spaces or tabs, one word without space, tab or ``:``, and ``:``.
72 A line such as ``@todo fix it, IMPL_1, impl`` has no field shape and is left to
73 the one-line parser.
74 """
75 if not start_sequence:
76 return None
77 idx = line.find(start_sequence)
78 if idx == -1 or any(char.isalnum() for char in line[:idx]):
79 return None
80 rest = line[idx + len(start_sequence) :]
81 word_end = 0
82 while word_end < len(rest) and rest[word_end].isalpha():
83 word_end += 1
84 tag = rest[:word_end]
85 if tag not in PYTHON_DOCSTRING_TAGS or not _FIELD_SHAPE.match(rest, word_end):
86 return None
87 return tag
88
89
90# Block tags of JSDoc, TSDoc and TypeScript's JSDoc support, and the inline tags that can
91# start a line, which a one-line start sequence such as ``@`` also matches: ``@param a
92# the first, thing`` would otherwise become a need with the id ``thing``.
93JSDOC_TAGS = frozenset(
94 {
95 "abstract",
96 "access",
97 "alias",
98 "alpha",
99 "arg",
100 "argument",
101 "async",
102 "augments",
103 "author",
104 "beta",
105 "borrows",
106 "callback",
107 "class",
108 "classdesc",
109 "const",
110 "constant",
111 "constructor",
112 "constructs",
113 "copyright",
114 "decorator",
115 "default",
116 "defaultValue",
117 "defaultvalue",
118 "deprecated",
119 "desc",
120 "description",
121 "emits",
122 "enum",
123 "event",
124 "eventProperty",
125 "example",
126 "exception",
127 "experimental",
128 "exports",
129 "extends",
130 "external",
131 "file",
132 "fileoverview",
133 "fires",
134 "func",
135 "function",
136 "generator",
137 "global",
138 "hideconstructor",
139 "host",
140 "ignore",
141 "implements",
142 "import",
143 "inheritDoc",
144 "inheritdoc",
145 "inner",
146 "instance",
147 "interface",
148 "internal",
149 "jsx",
150 "jsxFrag",
151 "jsxImportSource",
152 "jsxRuntime",
153 "kind",
154 "label",
155 "lends",
156 "license",
157 "link",
158 "linkcode",
159 "linkplain",
160 "listens",
161 "member",
162 "memberof",
163 "method",
164 "mixes",
165 "mixin",
166 "modifies",
167 "module",
168 "name",
169 "namespace",
170 "overload",
171 "override",
172 "overview",
173 "package",
174 "packageDocumentation",
175 "param",
176 "private",
177 "privateRemarks",
178 "prop",
179 "property",
180 "protected",
181 "public",
182 "readonly",
183 "remarks",
184 "requires",
185 "return",
186 "returns",
187 "satisfies",
188 "sealed",
189 "see",
190 "since",
191 "static",
192 "summary",
193 "template",
194 "this",
195 "throws",
196 "todo",
197 "tutorial",
198 "type",
199 "typeParam",
200 "typedef",
201 "var",
202 "variation",
203 "version",
204 "virtual",
205 "yield",
206 "yields",
207 }
208)
209
210
211#: What ends a JSDoc tag: a space, a tab, a ``{type}``, the ``}`` of an inline tag, or the
212#: end of the line. Locale-free: ubCode spells the same.
213_JSDOC_TAG_END = re.compile(r"[ \t\n{}]|$")
214
215
216def jsdoc_tag(line: str, start_sequence: str) -> str | None:
217 """The JSDoc tag a line of a JSDoc comment starts with, if any.
218
219 The line starts with one when, after comment decoration and whitespace (no
220 alphanumeric character), it holds ``start_sequence`` directly followed by a tag of
221 :data:`JSDOC_TAGS` that ends there: at a space, a tab, a ``{``, a ``}`` or the end of
222 the line (``@param a text``, ``@param{number} a``, ``{@link Foo}``, ``@returns``).
223 """
224 if not start_sequence:
225 return None
226 idx = line.find(start_sequence)
227 if idx == -1 or any(char.isalnum() for char in line[:idx]):
228 return None
229 rest = line[idx + len(start_sequence) :]
230 word_end = 0
231 while word_end < len(rest) and rest[word_end].isalpha():
232 word_end += 1
233 tag = rest[:word_end]
234 if tag not in JSDOC_TAGS or not _JSDOC_TAG_END.match(rest, word_end):
235 return None
236 return tag
237
238
239# Doxygen's special commands, by name: a line of a Doxygen comment that a one-line start
240# sequence such as ``@`` also matches (``@param clk Clock input, rising edge``) would
241# otherwise become a need with the id ``rising edge``. The list of the shared corpus file
242# ``doxygen.yaml``; ubCode spells the same.
243DOXYGEN_COMMANDS = frozenset(
244 {
245 "a",
246 "addindex",
247 "addtogroup",
248 "anchor",
249 "arg",
250 "attention",
251 "author",
252 "authors",
253 "b",
254 "brief",
255 "bug",
256 "c",
257 "callergraph",
258 "callgraph",
259 "category",
260 "cite",
261 "class",
262 "code",
263 "collaborationgraph",
264 "concept",
265 "cond",
266 "copybrief",
267 "copydetails",
268 "copydoc",
269 "copyright",
270 "date",
271 "def",
272 "defgroup",
273 "deprecated",
274 "details",
275 "diafile",
276 "dir",
277 "directorygraph",
278 "docbookinclude",
279 "docbookonly",
280 "dontinclude",
281 "dot",
282 "dotfile",
283 "doxyconfig",
284 "e",
285 "else",
286 "elseif",
287 "em",
288 "emoji",
289 "endcode",
290 "endcond",
291 "enddocbookonly",
292 "enddot",
293 "endhtmlonly",
294 "endif",
295 "endinternal",
296 "endlatexonly",
297 "endlink",
298 "endmanonly",
299 "endmsc",
300 "endparblock",
301 "endrtfonly",
302 "endsecreflist",
303 "enduml",
304 "endverbatim",
305 "endxmlonly",
306 "enum",
307 "example",
308 "exception",
309 "extends",
310 "file",
311 "fileinfo",
312 "fn",
313 "groupgraph",
314 "headerfile",
315 "hidecallergraph",
316 "hidecallgraph",
317 "hidecollaborationgraph",
318 "hidedirectorygraph",
319 "hidegroupgraph",
320 "hideincludedbygraph",
321 "hideincludegraph",
322 "hideinheritancegraph",
323 "hideinitializer",
324 "hideinlinesource",
325 "hiderefby",
326 "hiderefs",
327 "htmlinclude",
328 "htmlonly",
329 "idlexcept",
330 "if",
331 "ifnot",
332 "image",
333 "implements",
334 "include",
335 "includedbygraph",
336 "includedoc",
337 "includegraph",
338 "includelineno",
339 "ingroup",
340 "inheritancegraph",
341 "interface",
342 "internal",
343 "invariant",
344 "latexinclude",
345 "latexonly",
346 "li",
347 "line",
348 "lineinfo",
349 "link",
350 "mainpage",
351 "maninclude",
352 "manonly",
353 "memberof",
354 "module",
355 "msc",
356 "mscfile",
357 "n",
358 "name",
359 "namespace",
360 "noop",
361 "nosubgrouping",
362 "note",
363 "overload",
364 "p",
365 "package",
366 "page",
367 "par",
368 "paragraph",
369 "param",
370 "parblock",
371 "plantumlfile",
372 "post",
373 "pre",
374 "private",
375 "privatesection",
376 "property",
377 "protected",
378 "protectedsection",
379 "protocol",
380 "public",
381 "publicsection",
382 "pure",
383 "qualifier",
384 "raisewarning",
385 "ref",
386 "refitem",
387 "related",
388 "relatedalso",
389 "relates",
390 "relatesalso",
391 "remark",
392 "remarks",
393 "result",
394 "return",
395 "returns",
396 "retval",
397 "rtfinclude",
398 "rtfonly",
399 "sa",
400 "secreflist",
401 "section",
402 "see",
403 "short",
404 "showdate",
405 "showinitializer",
406 "showinlinesource",
407 "showrefby",
408 "showrefs",
409 "since",
410 "skip",
411 "skipline",
412 "snippet",
413 "snippetdoc",
414 "snippetlineno",
415 "startuml",
416 "static",
417 "struct",
418 "subpage",
419 "subparagraph",
420 "subsection",
421 "subsubparagraph",
422 "subsubsection",
423 "tableofcontents",
424 "test",
425 "throw",
426 "throws",
427 "todo",
428 "tparam",
429 "typedef",
430 "union",
431 "until",
432 "var",
433 "verbatim",
434 "verbinclude",
435 "version",
436 "vhdlflow",
437 "warning",
438 "weakgroup",
439 "xmlinclude",
440 "xmlonly",
441 "xrefitem",
442 }
443)
444
445
446#: What ends a Doxygen command: a space, a tab, a ``[`` as in ``@param[in]``, a ``{`` as
447#: in ``@code{.vhd}``, or the end of the line. Locale-free: ubCode spells the same.
448_DOXYGEN_COMMAND_END = re.compile(r"[ \t\n\[{]|$")
449
450
451def doxygen_tag(line: str, start_sequence: str) -> str | None:
452 """The Doxygen command a line of a Doxygen comment starts with, if any.
453
454 The line starts with one when, after comment decoration and whitespace (no
455 alphanumeric character), it holds ``start_sequence`` directly followed by a command of
456 :data:`DOXYGEN_COMMANDS` that ends there: at a space, a tab, a ``[``, a ``{`` or the end
457 of the line (``@param clk text``, ``@param[in] clk``, ``@code{.vhd}``, ``@brief``).
458 """
459 if not start_sequence:
460 return None
461 idx = line.find(start_sequence)
462 if idx == -1 or any(char.isalnum() for char in line[:idx]):
463 return None
464 rest = line[idx + len(start_sequence) :]
465 word_end = 0
466 while word_end < len(rest) and rest[word_end].isalpha():
467 word_end += 1
468 tag = rest[:word_end]
469 if tag not in DOXYGEN_COMMANDS or not _DOXYGEN_COMMAND_END.match(rest, word_end):
470 return None
471 return tag
472
473
474@dataclass
475class OnelineParserInvalidWarning:
476 """Invalid oneline comments."""
477
478 sub_type: WarningSubTypeEnum
479 msg: str
480
481
[docs]482# @One-line comment parser for traceability markers, IMPL_OLP_1, impl, [FE_DEF, FE_CMT]
483def oneline_parser( # handel warnings
484 oneline: str, oneline_config: OneLineCommentStyle
485) -> dict[str, str | list[str] | int] | OnelineParserInvalidWarning | None:
486 """
487 Extract the string from the custom one-line comment style with the following steps.
488
489 - Locate the start and end sequences
490 - extract the string between them
491 - return ``None`` for an empty content, whatever the start sequence
492 - on a one-character start sequence, return ``None`` for a content without the
493 field separator where more than one field is required: not a marker, and not a
494 warning either. The rule is ubCode's, and so are its words: "when a
495 single-character start_sequence (like `@`) matches but the content has no field
496 delimiter at all, this is likely a stray match
497 (e.g., Doxygen `@param`, `@brief`, `@return`) rather than a real marker.
498 Treating these as NoMatch avoids noisy TooFewFields warnings on codebases with
499 Doxygen annotations. Multi-character sequences (like `[[`) are specific enough
500 that a missing delimiter should still produce a warning."
501 - apply custom_split to split the strings into a list of fields by `field_split_char`
502 - check the number of required fields and the max number of the given fields
503 - split the strings located in the field with `type: list[str]` to a list of string
504 - introduce the default values to those fields which are not given
505 """
506 # find indices start and end char
507 start_idx = oneline.find(oneline_config.start_sequence)
508 end_idx = oneline.rfind(oneline_config.end_sequence)
509 if start_idx == -1 or end_idx == -1:
510 # start or end sequences do not exist
511 return None
512
513 # A marker whose end sequence is the newline extends to the end of the
514 # line, so an unanchored start sequence would swallow trailing prose. Anchor
515 # such a marker to the start of the comment content: everything preceding
516 # the start sequence must be comment decoration (`//`, `#`, `*`, ...) and
517 # whitespace. A word character before it means the start sequence is part of
518 # free-form prose (e.g. `// see @author, ...`) and the line is ignored
519 # (issue #88). Explicitly-bounded markers (e.g. `[[ ... ]]`) are
520 # self-delimiting and may appear anywhere, so they are exempt.
521 if oneline_config.end_sequence == UNIX_NEWLINE and any(
522 char.isalnum() for char in oneline[:start_idx]
523 ):
524 return None
525
526 # extract the string wrapped by start and end
527 start_idx = start_idx + len(oneline_config.start_sequence)
528 string = oneline[start_idx:end_idx].strip()
529
530 # numbers of needs_fields which are required
531 cnt_required_fields = oneline_config.get_cnt_required_fields()
532
533 if not string:
534 # nothing between the start and end sequences: not a marker either
535 return None
536
537 # a documentation tag such as `@param x`, not a marker (see the docstring)
538 if (
539 len(oneline_config.start_sequence) == 1
540 and oneline_config.field_split_char not in string
541 and cnt_required_fields > 1
542 ):
543 return None
544 # indices of the field which has type:list[str]
545 positions_list_str = oneline_config.get_pos_list_str()
546
547 min_fields = cnt_required_fields
548 max_fields = len(oneline_config.needs_fields)
549
550 string_fields = [
551 _field.strip(" ")
552 for _field in custom_split(
553 string, oneline_config.field_split_char, positions_list_str
554 )
555 ]
556 if len(string_fields) < min_fields:
557 return OnelineParserInvalidWarning(
558 sub_type=WarningSubTypeEnum.too_few_fields,
559 msg=f"{len(string_fields)} given fields, minimum is {min_fields}",
560 )
561
562 if len(string_fields) > max_fields:
563 return OnelineParserInvalidWarning(
564 sub_type=WarningSubTypeEnum.too_many_fields,
565 msg=f"{len(string_fields)} given fields, maximum is {max_fields}",
566 )
567 resolved: dict[str, str | list[str] | int] = {}
568 for idx in range(len(oneline_config.needs_fields)):
569 field_name: str = oneline_config.needs_fields[idx]["name"]
570 if len(string_fields) > idx:
571 # given fields
572 if is_newline_in_field(string_fields[idx]):
573 # the case where the field contains a new line character
574 return OnelineParserInvalidWarning(
575 sub_type=WarningSubTypeEnum.newline_in_field,
576 msg=f"Field '{field_name}' contains a newline character",
577 )
578 if oneline_config.needs_fields[idx]["type"] == "str":
579 resolved[field_name] = string_fields[idx]
580 elif oneline_config.needs_fields[idx]["type"] == "list[str]":
581 # find the indices of "[" and "]"
582 list_start_idx = string_fields[idx].find("[")
583 list_end_idx = string_fields[idx].rfind("]")
584 if list_start_idx == -1 or list_end_idx == -1:
585 # brackets are not found
586 return OnelineParserInvalidWarning(
587 sub_type=WarningSubTypeEnum.missing_square_brackets,
588 msg=f"Field '{field_name}' with 'type': '{oneline_config.needs_fields[idx]['type']}' must be given with '[]' brackets",
589 )
590
591 if list_start_idx != 0 or list_end_idx != len(string_fields[idx]) - 1:
592 # brackets are found but not at the beginning and the end
593 return OnelineParserInvalidWarning(
594 sub_type=WarningSubTypeEnum.not_start_or_end_with_square_brackets,
595 msg=f"Field '{field_name}' with 'type': '{oneline_config.needs_fields[idx]['type']}' must start with '[' and end with ']'",
596 )
597
598 string_items = string_fields[idx][list_start_idx + 1 : list_end_idx]
599
600 if not string_items.strip():
601 # the case where the empty string ("") or only spaces between "[" "]"
602 resolved[field_name] = []
603 else:
604 items = [_item.strip() for _item in custom_split(string_items, ",")]
605 resolved[field_name] = [item.strip() for item in items]
606 else:
607 # for not given fields, introduce the default
608 default = oneline_config.needs_fields[idx].get("default")
609 if default is None:
610 continue
611 resolved[field_name] = default
612
613 resolved["start_column"] = start_idx
614 resolved["end_column"] = end_idx
615 return resolved
616
617
618def custom_split(
619 string: str, delimiter: str, positions_list_str: list[int] | None = None
620) -> list[str]:
621 """
622 A string shall be split with the following conditions:
623
624 - To use special chars in literal , escape ('\') must be used
625 - String shall be split by the given delimiter
626 - In a field with `type: str`:
627 - Special chars are delimiter, '\', '[' and ']'
628 - In a field with `type: list[str]`:
629 - Special chars are only '[' and ']'
630
631 When the string is given without any fields with `type: list[str]` (positions_list_str=None),
632 it's considered as it is in a field with `type: str`.
633 """
634 if positions_list_str is None:
635 positions_list_str = []
636 escape_chars = [delimiter, "[", "]", ESCAPE]
637 field = [] # a list of string for a field
638 fields: list[str] = [] # a list of string which contains
639 leading_escape = False
640 expect_closing_bracket = False
641
642 for char in string:
643 # +1 to locate the current field position
644 current_field_idx = len(fields) + 1
645 is_list_str_field = current_field_idx in positions_list_str
646
647 if leading_escape:
648 if char not in escape_chars:
649 # leading escape is considered as a literal
650 field.append(ESCAPE)
651 field.append(char)
652 leading_escape = False
653 continue
654
655 if char == ESCAPE and not is_list_str_field:
656 leading_escape = True
657 continue
658
659 if char == delimiter:
660 if is_list_str_field and expect_closing_bracket:
661 # delimiter occurs in the field with type:list[str]
662 field.append(char)
663 else:
664 fields.append("".join(field))
665 field = []
666 continue
667
668 if is_list_str_field:
669 if char == "[":
670 expect_closing_bracket = True
671 if char == "]":
672 expect_closing_bracket = False
673
674 field.append(char)
675
676 # add last field
677 fields.append("".join(field))
678 return fields
679
680
681def is_newline_in_field(field: str) -> bool:
682 """
683 Check if the field contains a new line character.
684 """
685 return UNIX_NEWLINE in field
Back to top